Compare commits
83
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
ded6cb197c | ||
|
|
908855644a | ||
|
|
a92df7515a | ||
|
|
10dba0892e | ||
|
|
03955675a8 | ||
|
|
b0f42f14a0 | ||
|
|
3a53c50c45 | ||
|
|
3b386649bf | ||
|
|
eb5c74b4b2 | ||
|
|
d4c72697f2 | ||
|
|
6a4265d81d | ||
|
|
706674272c | ||
|
|
649acd12ee | ||
|
|
ad0bed4585 | ||
|
|
7347a08979 | ||
|
|
3f0af38509 | ||
|
|
7d81f85b38 | ||
|
|
f3863c3720 | ||
|
|
c7bea972d6 | ||
|
|
356cddbb59 | ||
|
|
d8ff6fa229 | ||
|
|
cd1b286378 | ||
|
|
456042ee51 | ||
|
|
7db4101b24 | ||
|
|
01de6cbd8b | ||
|
|
e5996b7c1e | ||
|
|
4261f76dd2 | ||
|
|
3322d68948 | ||
|
|
f1e9cce4ae | ||
|
|
8e02b9961d | ||
|
|
25048d9ea4 | ||
|
|
84b4c11188 | ||
|
|
a0849eec58 | ||
|
|
fe578c5864 | ||
|
|
15d2439915 | ||
|
|
107716fa4f | ||
|
|
499dd348ab | ||
|
|
84165ab755 | ||
|
|
4bead26326 | ||
|
|
b552e569bb | ||
|
|
f8f2dca2ce | ||
|
|
945328fa24 | ||
|
|
57030d5168 | ||
|
|
fb258522ac | ||
|
|
b8f60bdec4 | ||
|
|
8f02e4a284 | ||
|
|
49fe17336b | ||
|
|
f89da6a197 | ||
|
|
96a67c7e76 | ||
|
|
e746ba5504 | ||
|
|
0c5938dd60 | ||
|
|
3473a6c889 | ||
|
|
0bd59b0b6e | ||
|
|
e1937674cd | ||
|
|
cfdab63a3a | ||
|
|
f9f8f65cee | ||
|
|
884ac8a7e9 | ||
|
|
9a5d34e888 | ||
|
|
20b117dabf | ||
|
|
e133c11fde | ||
|
|
951dae26a9 | ||
|
|
46ec532745 | ||
|
|
edd8d3d9c9 | ||
|
|
40b3747434 | ||
|
|
2bdb6c44e4 | ||
|
|
82cf300f50 | ||
|
|
9881d1ff81 | ||
|
|
3aed43c6de | ||
|
|
f822e4737c | ||
|
|
6af65ba5c5 | ||
|
|
6d80343320 | ||
|
|
b91707b707 | ||
|
|
691a7acc77 | ||
|
|
08e95f9ec1 | ||
|
|
b91939e5c4 | ||
|
|
e298bb291e | ||
|
|
d7647b6104 | ||
|
|
7be42654fe | ||
|
|
d7725c274c | ||
|
|
b6bf94f129 | ||
|
|
4be3f247d8 | ||
|
|
28ce8c4dfe | ||
|
|
e46e2cfe68 |
@@ -4,6 +4,13 @@
|
||||
# body-diff, CI). This shim runs the structural validator over the working tree; the body-diff/no-
|
||||
# vanish checks run in CI where a base/head is available. Fail-open on any tooling trouble.
|
||||
set -uo pipefail
|
||||
|
||||
# ersatztv#776 — report that this hook fired. MUST precede any stdin read.
|
||||
# git hook: decides by exit code, and its stdout is live progress text.
|
||||
ETV_HOOK_FIRE_LIB="${CLAUDE_PROJECT_DIR:-$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." 2>/dev/null && pwd)}/scripts/hook-fire-log.sh" || true
|
||||
[ -r "$ETV_HOOK_FIRE_LIB" ] && . "$ETV_HOOK_FIRE_LIB" || true
|
||||
type etv_hook_fire_begin >/dev/null 2>&1 || etv_hook_fire_begin() { :; }
|
||||
etv_hook_fire_begin decisions-guard "" stream || true
|
||||
cd "$(git rev-parse --show-toplevel)" || exit 0
|
||||
command -v python3 >/dev/null 2>&1 || exit 0 # no python -> fail-open
|
||||
PYTHONPATH=. python3 scripts/decisions_validate.py
|
||||
|
||||
@@ -17,6 +17,13 @@
|
||||
# This is a reminder, never a hard gate — `start` only injects context; `finish` is a one-shot Stop nudge.
|
||||
set -euo pipefail
|
||||
|
||||
# ersatztv#776 — report that this hook fired. MUST precede any stdin read.
|
||||
# Claude hook: decides by printed JSON, so stdout is captured.
|
||||
ETV_HOOK_FIRE_LIB="${CLAUDE_PROJECT_DIR:-$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." 2>/dev/null && pwd)}/scripts/hook-fire-log.sh" || true
|
||||
[ -r "$ETV_HOOK_FIRE_LIB" ] && . "$ETV_HOOK_FIRE_LIB" || true
|
||||
type etv_hook_fire_begin >/dev/null 2>&1 || etv_hook_fire_begin() { :; }
|
||||
etv_hook_fire_begin design-sync-reminder "${1:-}" capture || true
|
||||
|
||||
UI_RE='(^|/)web/src/.*\.(tsx|css)$'
|
||||
TEST_RE='\.test\.(tsx|ts)$'
|
||||
|
||||
|
||||
@@ -4,6 +4,13 @@
|
||||
# a sibling worktree another session created apart from this session's own.
|
||||
# Fail-safe: any parse trouble → do nothing (the guard stays fail-open without a marker).
|
||||
set -euo pipefail
|
||||
|
||||
# ersatztv#776 — report that this hook fired. MUST precede any stdin read.
|
||||
# Claude hook: decides by printed JSON, so stdout is captured.
|
||||
ETV_HOOK_FIRE_LIB="${CLAUDE_PROJECT_DIR:-$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." 2>/dev/null && pwd)}/scripts/hook-fire-log.sh" || true
|
||||
[ -r "$ETV_HOOK_FIRE_LIB" ] && . "$ETV_HOOK_FIRE_LIB" || true
|
||||
type etv_hook_fire_begin >/dev/null 2>&1 || etv_hook_fire_begin() { :; }
|
||||
etv_hook_fire_begin posttooluse-worktree-marker "" capture || true
|
||||
input=$(cat)
|
||||
cmd=$(printf '%s' "$input" | jq -r '.tool_input.command // ""' 2>/dev/null || true)
|
||||
cwd=$(printf '%s' "$input" | jq -r '.cwd // ""' 2>/dev/null || true)
|
||||
|
||||
@@ -15,6 +15,13 @@
|
||||
# no origin/main, HEAD unresolved -> allow. Deliberate escape: ETV_ALLOW_DIRTY_PUSH=1.
|
||||
set -uo pipefail
|
||||
|
||||
# ersatztv#776 — report that this hook fired. MUST precede any stdin read.
|
||||
# git hook: decides by exit code, and its stdout is live progress text.
|
||||
ETV_HOOK_FIRE_LIB="${CLAUDE_PROJECT_DIR:-$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." 2>/dev/null && pwd)}/scripts/hook-fire-log.sh" || true
|
||||
[ -r "$ETV_HOOK_FIRE_LIB" ] && . "$ETV_HOOK_FIRE_LIB" || true
|
||||
type etv_hook_fire_begin >/dev/null 2>&1 || etv_hook_fire_begin() { :; }
|
||||
etv_hook_fire_begin prepush-clean-worktree-check "" stream || true
|
||||
|
||||
[ "${ETV_ALLOW_DIRTY_PUSH:-}" = "1" ] && exit 0
|
||||
git rev-parse --git-dir >/dev/null 2>&1 || exit 0
|
||||
|
||||
|
||||
@@ -12,6 +12,13 @@
|
||||
# Auth (never committed): ETV_GITEA_TOKEN or ETV_GITEA_BASICAUTH; ETV_GITEA_URL overrides the base.
|
||||
set -euo pipefail
|
||||
|
||||
# ersatztv#776 — report that this hook fired. MUST precede any stdin read.
|
||||
# git hook: decides by exit code, and its stdout is live progress text.
|
||||
ETV_HOOK_FIRE_LIB="${CLAUDE_PROJECT_DIR:-$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." 2>/dev/null && pwd)}/scripts/hook-fire-log.sh" || true
|
||||
[ -r "$ETV_HOOK_FIRE_LIB" ] && . "$ETV_HOOK_FIRE_LIB" || true
|
||||
type etv_hook_fire_begin >/dev/null 2>&1 || etv_hook_fire_begin() { :; }
|
||||
etv_hook_fire_begin prepush-donewhen "" stream || true
|
||||
|
||||
# git passes "<localref> <localsha> <remoteref> <remotesha>" lines on stdin.
|
||||
refs=$(cat || true)
|
||||
printf '%s\n' "$refs" | grep -q 'refs/heads/main' || exit 0 # only gate pushes to main
|
||||
|
||||
@@ -9,9 +9,48 @@
|
||||
# a positively-proven "behind origin/main". Deliberate exception: ETV_SKIP_REBASE_CHECK=1.
|
||||
set -uo pipefail
|
||||
|
||||
# ersatztv#776 — report that this hook fired. MUST precede any stdin read.
|
||||
# git hook: decides by exit code, and its stdout is live progress text.
|
||||
ETV_HOOK_FIRE_LIB="${CLAUDE_PROJECT_DIR:-$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." 2>/dev/null && pwd)}/scripts/hook-fire-log.sh" || true
|
||||
[ -r "$ETV_HOOK_FIRE_LIB" ] && . "$ETV_HOOK_FIRE_LIB" || true
|
||||
type etv_hook_fire_begin >/dev/null 2>&1 || etv_hook_fire_begin() { :; }
|
||||
etv_hook_fire_begin prepush-rebase-check "" stream || true
|
||||
|
||||
[ "${ETV_SKIP_REBASE_CHECK:-}" = "1" ] && exit 0
|
||||
git rev-parse --git-dir >/dev/null 2>&1 || exit 0
|
||||
|
||||
# Tag-only push exemption (ersatztv#719): the release cut tags a commit on main while the local
|
||||
# branch sits 1 commit behind origin/main, so H11 blocked EVERY release -- and its "rebase first"
|
||||
# advice did not even apply, since no branch was being pushed. A tag push cannot revert anyone's
|
||||
# merged work, which is the failure mode H11 exists to prevent, so skip the freshness check when
|
||||
# EVERY ref being pushed is under refs/tags/. (See #719 for the observed flow.)
|
||||
#
|
||||
# Read pushed refs from stdin: git feeds pre-push hooks one line per ref, "<local ref> <local sha>
|
||||
# <remote ref> <remote sha>" (.husky/pre-push forwards the lines it already captured). Ignore blank
|
||||
# lines. VACUOUS-TRUTH GUARD: "all refs are tags" is trivially true when there are zero ref lines
|
||||
# (hook run manually, stdin not forwarded, etc.) -- that would silently disable H11 for every push.
|
||||
# Require at least one parsed ref line before granting the exemption; with zero lines, fall through
|
||||
# to the existing branch-freshness check below (current behavior preserved).
|
||||
#
|
||||
# `[ -t 0 ] ||` so an interactive run does not hang waiting on a terminal: this script had no stdin
|
||||
# reader before #719, and its own docs call "run by hand" a supported case. A TTY yields no ref
|
||||
# lines, which is exactly the zero-line fall-through.
|
||||
_h11_refs_seen=0
|
||||
_h11_all_tags=1
|
||||
[ -t 0 ] || while IFS=' ' read -r _h11_local_ref _h11_local_sha _h11_remote_ref _h11_remote_sha \
|
||||
|| [ -n "${_h11_local_ref:-}" ]; do # `|| [ -n ... ]` also processes a final line with no trailing newline
|
||||
[ -z "${_h11_local_ref:-}" ] && continue
|
||||
_h11_refs_seen=1
|
||||
case "${_h11_remote_ref:-}" in
|
||||
refs/tags/*) ;;
|
||||
*) _h11_all_tags=0 ;;
|
||||
esac
|
||||
_h11_local_ref=''
|
||||
done
|
||||
if [ "$_h11_refs_seen" = "1" ] && [ "$_h11_all_tags" = "1" ]; then
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# Best-effort fetch of the latest main; offline / no network -> don't block.
|
||||
git fetch origin main --quiet 2>/dev/null || exit 0
|
||||
git rev-parse --verify --quiet origin/main >/dev/null 2>&1 || exit 0
|
||||
|
||||
@@ -40,6 +40,13 @@
|
||||
# Fail-open by design: any parse trouble -> allow (exit 0, no output).
|
||||
set -uo pipefail
|
||||
|
||||
# ersatztv#776 — report that this hook fired. MUST precede any stdin read.
|
||||
# Claude hook: decides by printed JSON, so stdout is captured.
|
||||
ETV_HOOK_FIRE_LIB="${CLAUDE_PROJECT_DIR:-$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." 2>/dev/null && pwd)}/scripts/hook-fire-log.sh" || true
|
||||
[ -r "$ETV_HOOK_FIRE_LIB" ] && . "$ETV_HOOK_FIRE_LIB" || true
|
||||
type etv_hook_fire_begin >/dev/null 2>&1 || etv_hook_fire_begin() { :; }
|
||||
etv_hook_fire_begin pretooluse-agent-model "" capture || true
|
||||
|
||||
input=$(cat)
|
||||
|
||||
tool=$(printf '%s' "$input" | jq -r '.tool_name // ""' 2>/dev/null || true)
|
||||
|
||||
@@ -3,6 +3,13 @@
|
||||
# The historic 8-9-way crash was RAM starvation, not CPU load; gate on FREE RAM.
|
||||
# Fail-open: if memory_pressure is unavailable/unparsable → allow.
|
||||
set -euo pipefail
|
||||
|
||||
# ersatztv#776 — report that this hook fired. MUST precede any stdin read.
|
||||
# Claude hook: decides by printed JSON, so stdout is captured.
|
||||
ETV_HOOK_FIRE_LIB="${CLAUDE_PROJECT_DIR:-$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." 2>/dev/null && pwd)}/scripts/hook-fire-log.sh" || true
|
||||
[ -r "$ETV_HOOK_FIRE_LIB" ] && . "$ETV_HOOK_FIRE_LIB" || true
|
||||
type etv_hook_fire_begin >/dev/null 2>&1 || etv_hook_fire_begin() { :; }
|
||||
etv_hook_fire_begin pretooluse-agent-ram "" capture || true
|
||||
free=$(memory_pressure -Q 2>/dev/null | grep -oE 'free percentage: [0-9]+' | grep -oE '[0-9]+' || true)
|
||||
[ -z "${free:-}" ] && exit 0
|
||||
|
||||
|
||||
@@ -2,6 +2,13 @@
|
||||
# PreToolUse / Bash — deny commands that violate a HARD RULE.
|
||||
# Fail-open: any parse trouble → allow (exit 0 with no output).
|
||||
set -euo pipefail
|
||||
|
||||
# ersatztv#776 — report that this hook fired. MUST precede any stdin read.
|
||||
# Claude hook: decides by printed JSON, so stdout is captured.
|
||||
ETV_HOOK_FIRE_LIB="${CLAUDE_PROJECT_DIR:-$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." 2>/dev/null && pwd)}/scripts/hook-fire-log.sh" || true
|
||||
[ -r "$ETV_HOOK_FIRE_LIB" ] && . "$ETV_HOOK_FIRE_LIB" || true
|
||||
type etv_hook_fire_begin >/dev/null 2>&1 || etv_hook_fire_begin() { :; }
|
||||
etv_hook_fire_begin pretooluse-bash-guard "" capture || true
|
||||
input=$(cat)
|
||||
cmd=$(printf '%s' "$input" | jq -r '.tool_input.command // ""' 2>/dev/null || true)
|
||||
|
||||
|
||||
@@ -18,6 +18,13 @@
|
||||
# the reason a commit can't happen; CI is still the backstop.
|
||||
set -uo pipefail
|
||||
|
||||
# ersatztv#776 — report that this hook fired. MUST precede any stdin read.
|
||||
# Claude hook: decides by printed JSON, so stdout is captured.
|
||||
ETV_HOOK_FIRE_LIB="${CLAUDE_PROJECT_DIR:-$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." 2>/dev/null && pwd)}/scripts/hook-fire-log.sh" || true
|
||||
[ -r "$ETV_HOOK_FIRE_LIB" ] && . "$ETV_HOOK_FIRE_LIB" || true
|
||||
type etv_hook_fire_begin >/dev/null 2>&1 || etv_hook_fire_begin() { :; }
|
||||
etv_hook_fire_begin pretooluse-bom-guard "" capture || true
|
||||
|
||||
input=$(cat)
|
||||
cmd=$(printf '%s' "$input" | jq -r '.tool_input.command // ""' 2>/dev/null || true)
|
||||
[ -n "$cmd" ] || exit 0
|
||||
@@ -66,7 +73,11 @@ while IFS= read -r f; do
|
||||
esac
|
||||
p="$root/$f"
|
||||
[ -f "$p" ] || continue
|
||||
if [ "$(head -c3 "$p" 2>/dev/null | xxd -p 2>/dev/null)" = "efbbbf" ]; then
|
||||
# `od`, NOT `xxd`. `xxd` ships with vim and is absent on plain Linux hosts including this repo's
|
||||
# CI runner, where the command substitution yielded empty, never equalled `efbbbf`, and this guard
|
||||
# therefore passed every BOM in silence. It has been fail-open on any host without vim since it
|
||||
# was written. `od -A n -t x1 -N 3` is POSIX and produces byte-identical output on macOS and Linux.
|
||||
if [ "$(od -A n -t x1 -N 3 < "$p" 2>/dev/null | tr -d ' \n')" = "efbbbf" ]; then
|
||||
bad="${bad} ${f}"$'\n'
|
||||
fi
|
||||
done < /tmp/.bom-guard-files.$$
|
||||
|
||||
@@ -8,8 +8,12 @@
|
||||
# LATEST commit was reviewed, not a stale earlier diff (the ersatztv#242 failure mode:
|
||||
# "re-review the fix commit, not just the initial PR diff").
|
||||
#
|
||||
# EVERY ONE OF THOSE IS A SNAPSHOT, taken when the merge tool is called. That is sound for an
|
||||
# immediate merge and UNSOUND for a scheduled one: with merge_when_checks_succeed, Gitea merges
|
||||
# EVERY ONE OF THOSE IS A SNAPSHOT, taken when the merge tool is called. The window is SMALL for an
|
||||
# immediate merge and UNBOUNDED for a scheduled one. Small is not zero, and this comment used to say
|
||||
# "sound", which is the overclaim ersatztv#778 removed: this hook returns `allow` and a SEPARATE call
|
||||
# performs the merge, so a push can still land in between. The merge API accepts an optional
|
||||
# `head_commit_id` that would make that call a true compare-and-set; a PreToolUse hook cannot add an
|
||||
# argument, only refuse without one. With merge_when_checks_succeed, Gitea merges
|
||||
# later, against whatever head is green then (ersatztv#622). So the sha-bound half of H10 is
|
||||
# enforced by the SERVER, not here — `review-verdict/h10` is a required status check on `main`,
|
||||
# written per-sha by scripts/post-review-verdict.sh, and a new commit cannot inherit it. This hook
|
||||
@@ -40,6 +44,13 @@
|
||||
# Gitea auth from env (never committed): ETV_GITEA_TOKEN (a token) OR ETV_GITEA_BASICAUTH (user:pass).
|
||||
# ETV_GITEA_URL overrides the base (default: the LAN instance; a LAN address, not a secret).
|
||||
set -euo pipefail
|
||||
|
||||
# ersatztv#776 — report that this hook fired. MUST precede any stdin read.
|
||||
# Claude hook: decides by printed JSON, so stdout is captured.
|
||||
ETV_HOOK_FIRE_LIB="${CLAUDE_PROJECT_DIR:-$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." 2>/dev/null && pwd)}/scripts/hook-fire-log.sh" || true
|
||||
[ -r "$ETV_HOOK_FIRE_LIB" ] && . "$ETV_HOOK_FIRE_LIB" || true
|
||||
type etv_hook_fire_begin >/dev/null 2>&1 || etv_hook_fire_begin() { :; }
|
||||
etv_hook_fire_begin pretooluse-merge-consent "" capture || true
|
||||
input=$(cat)
|
||||
|
||||
decide() { # $1=grant|allow|deny|ask $2=reason
|
||||
@@ -168,10 +179,39 @@ fi
|
||||
# after which a later, successful status read could still auto-grant. A transient failure would then
|
||||
# have produced a "merge gate: satisfied" message for a comparison that never happened. Every
|
||||
# unreadable input here therefore falls through to a human (`ask`), never to silence.
|
||||
live_base=$(printf '%s' "$prjson" | jq -r '.base.ref // ""' 2>/dev/null || true)
|
||||
if [ -z "$live_base" ]; then
|
||||
# RE-READ THE BASE HERE, ONCE, FOR EVERY PATH BELOW (ersatztv#778).
|
||||
#
|
||||
# "Below" is literal, and the one consumer ABOVE is disclosed rather than implied: the docs-only
|
||||
# enumeration still runs against the snapshot `$base_ref` and can `decide allow` before reaching
|
||||
# this point. That is bounded and deliberate — a docs-only match is a PASSTHROUGH to the ordinary
|
||||
# human prompt, never an auto-grant, so a stale base there costs a prompt someone was going to see
|
||||
# anyway. Every path that can GRANT passes through the check below.
|
||||
#
|
||||
# `$base_ref` above comes from the PR snapshot taken at the top of this hook, and the docs-only
|
||||
# enumeration between there and here is up to forty round trips. A PERSISTENT retarget in that gap
|
||||
# needs no ABA and no force-push: every base-dependent decision below would be formed against a
|
||||
# branch the PR no longer targets. Checking a stale identifier is not checking — which is the whole
|
||||
# of `process.check-and-use-pins-a-version`, so the guard enforcing that rule must not break it.
|
||||
#
|
||||
# This re-read first landed inside the scheduled-auto-merge branch only, which fixed the branch-
|
||||
# protection lookup and left the #632 retarget DETECTION below still reading the stale snapshot. Cold
|
||||
# review demonstrated the consequence with this repo's own fixture: scheduled+retarget denied, while
|
||||
# immediate+retarget auto-GRANTED. That is the twin-missed shape — a fix applied to the path where it
|
||||
# was noticed — so the re-read is hoisted above every consumer rather than duplicated into each.
|
||||
prjson_now=$(gq "repos/$owner/$repo/pulls/$pr")
|
||||
if [ -z "${prjson_now//[[:space:]]/}" ] || ! printf '%s' "$prjson_now" | jq -e 'type == "object"' >/dev/null 2>&1; then
|
||||
decide ask "H10 merge gate: could not re-read PR #$pr to confirm it still targets '$base_ref' before checking the verdict against it. Confirm the target branch, then merge."
|
||||
fi
|
||||
base_now=$(printf '%s' "$prjson_now" | jq -r '.base.ref // ""' 2>/dev/null || true)
|
||||
if [ -z "$base_now" ]; then
|
||||
decide ask "H10 merge gate: PR #$pr reports no base branch (.base.ref), so the verdict cannot be checked against the branch it was formed for (ersatztv#632). Confirm the PR still targets the branch it was reviewed against before merging."
|
||||
fi
|
||||
if [ -n "$base_ref" ] && [ "$base_now" != "$base_ref" ]; then
|
||||
decide deny "H6/H10 merge gate: BLOCKED — PR #$pr was retargeted from '$base_ref' to '$base_now' while this gate was evaluating. Every check formed against '$base_ref', including the changed-file enumeration and the review verdict, describes a merge that is no longer the one being requested (ersatztv#632). Re-review against '$base_now' and run: scripts/post-review-verdict.sh $pr MERGEABLE"
|
||||
fi
|
||||
# From here on both names are the freshly-confirmed base; they are equal by the check above.
|
||||
base_ref=$base_now
|
||||
live_base=$base_now
|
||||
if [ -n "$sha" ]; then
|
||||
# This is the THIRD read of this endpoint in a worst-case hook run (the ordinary-CI branch and the
|
||||
# scheduled-auto-merge branch each do their own). Sharing one snapshot would close a narrow
|
||||
@@ -301,8 +341,19 @@ else
|
||||
# status decide this. Here the fallthrough happens to land on `vstate=""` -> deny (fail-CLOSED,
|
||||
# so this was never a hole), but it would have surfaced the wrong message — a "BLOCKED, no
|
||||
# verdict" deny instead of the "could not read the status" ask this branch exists to give.
|
||||
if [ -z "${vjson//[[:space:]]/}" ] || ! printf '%s' "$vjson" | jq -e '.statuses | type == "array"' >/dev/null 2>&1; then
|
||||
decide ask "H6/H10 merge gate: could not read the 'review-verdict/h10' status for PR #$pr head ${sha:0:7} (Gitea unreachable or an unexpected response). Confirm the current head is reviewed before scheduling an auto-merge."
|
||||
# Validate the MEMBERS, not just the array. `.statuses | type == "array"` passes for
|
||||
# `{"statuses":[1]}`, and the extraction below then errors with "Cannot index number with string"
|
||||
# and exits 5 — which, under `set -e`, aborts this hook with NO JSON on stdout at all. A consent
|
||||
# hook that emits nothing has violated its own contract: it neither grants, denies nor asks. Same
|
||||
# one-level-down swallow as the #632 base-change guard and the branch-protection shape check
|
||||
# below; the validation domain must match the CONSUMPTION domain (ersatztv#778).
|
||||
if [ -z "${vjson//[[:space:]]/}" ] \
|
||||
|| ! printf '%s' "$vjson" \
|
||||
| jq -e '(.statuses | type == "array")
|
||||
and all(.statuses[]; type == "object"
|
||||
and ((.context | type) == "string")
|
||||
and ((.status | type) == "string"))' >/dev/null 2>&1; then
|
||||
decide ask "H6/H10 merge gate: could not read the 'review-verdict/h10' status for PR #$pr head ${sha:0:7} (Gitea unreachable, or a response whose status rows are not the expected shape). Confirm the current head is reviewed before scheduling an auto-merge."
|
||||
fi
|
||||
vstate=$(printf '%s' "$vjson" | jq -r '[.statuses[] | select(.context == "review-verdict/h10")] | first | .status // ""')
|
||||
case "$vstate" in
|
||||
@@ -311,6 +362,210 @@ else
|
||||
pending) decide deny "H6/H10 merge gate: BLOCKED — 'review-verdict/h10' is still pending on PR #$pr head ${sha:0:7} (no verdict posted for this commit yet). Review the current head and run: scripts/post-review-verdict.sh $pr MERGEABLE" ;;
|
||||
*) decide deny "H6/H10 merge gate: BLOCKED — 'review-verdict/h10' is '$vstate' on PR #$pr head ${sha:0:7}. Resolve the findings, then run: scripts/post-review-verdict.sh $pr MERGEABLE" ;;
|
||||
esac
|
||||
|
||||
# --- The mitigation this path RESTS on, verified instead of asserted (ersatztv#778). -----------
|
||||
# Everything above proves a property of the head that exists NOW. What makes that safe under
|
||||
# merge_when_checks_succeed is stated in the paragraph opening this branch: `review-verdict/h10`
|
||||
# is a REQUIRED status check on the base, a commit status belongs to exactly ONE sha, so a commit
|
||||
# pushed after scheduling cannot inherit it and Gitea's own gate refuses the merge.
|
||||
#
|
||||
# That guarantee is branch-protection CONFIG. It lives outside this repo, no code here owned it,
|
||||
# and until #778 nothing compared the two — so the grant reason handed to a human cited a
|
||||
# protection that could have been switched off with no signal anywhere. The comment above and the
|
||||
# grant string below are claims about the past; a dated claim is not a check.
|
||||
#
|
||||
# This is the hook's OWN defect class (#778 / `process.check-and-use-pins-a-version`): a check
|
||||
# ("a later push clears the status") authorizes an action ("arm an auto-merge that Gitea completes
|
||||
# later") over state that can change in between, with nothing pinning it. The read here does not
|
||||
# pin anything either — branch protection can still be edited after this call — but it converts an
|
||||
# ASSUMPTION that was never observed into a precondition that is, which is the honest ceiling for
|
||||
# a config whose API offers no version, ETag or conditional read.
|
||||
#
|
||||
# Tri-state, matching this file's idiom throughout: unreadable -> ask (a human adjudicates),
|
||||
# present -> proceed, ABSENT -> deny. Absence is not a degraded read; it is #622's hole reopened,
|
||||
# and the whole point of that issue is that the failure is silent from the merge caller's side.
|
||||
# Belt-and-braces: `$base_ref` was proven non-empty and re-confirmed at the hoisted check above,
|
||||
# so this cannot fire today. Kept because it is the precondition this block's URL depends on, and
|
||||
# a future edit that moves either piece should fail loudly here rather than request a URL with an
|
||||
# empty path segment.
|
||||
[ -n "$base_ref" ] || decide ask "H6/H10 merge gate: could not resolve PR #$pr's base branch, so the 'review-verdict/h10' required-check protection that makes a scheduled auto-merge safe (ersatztv#622) can't be confirmed. Verify branch protection on the base, or merge immediately instead of scheduling."
|
||||
# The base was re-read and confirmed unchanged above, for every path — see the hoist comment
|
||||
# there. It is deliberately NOT re-read a second time here: two reads would create a window
|
||||
# between them for no gain, and the hoisted check already covers the enumeration gap that made
|
||||
# this necessary.
|
||||
# A read failure here is NOT evidence about the branch. The deleted by-name endpoint answered 404
|
||||
# for "no rule with this name", which was a finding; the LIST endpoint's 404 means the repo was not
|
||||
# found or is invisible to this credential, which is a read failure. Absence is now established by
|
||||
# the classifier returning `nomatch` over a list that WAS read, never by an HTTP status.
|
||||
# ALWAYS enumerate the rule LIST; never look a rule up by name. The by-name endpoint
|
||||
# (`branch_protections/{name}`) is an exact DB lookup — `GetProtectedBranchRuleByName` — which
|
||||
# performs no matching and knows nothing about precedence, so a 200 from it means only "a rule
|
||||
# with this NAME exists and lists this context", never "this context is required on this branch".
|
||||
#
|
||||
# It was used first, with the list consulted only on a 404, and cold review found what that left
|
||||
# behind: the precedence argument below guarded the 404 path while the 200 path — the one this
|
||||
# repo actually takes — granted without it. Given a rule `main` requiring `review-verdict/h10` and
|
||||
# a rule `m*` with better Priority that does not, Gitea applies `m*`, and the by-name hit on
|
||||
# `main` granted anyway. The hardened path was dead code and the unhardened one was live. Deleting
|
||||
# the twin rather than documenting it is the point: one fetch, one classifier, one argument, and
|
||||
# no second path to keep in step. The ref no longer reaches a URL segment, so it needs no
|
||||
# encoding either.
|
||||
bp_file=$(mktemp) || decide ask "H6/H10 merge gate: could not allocate a temp file to read branch protection for '$base_ref'. Confirm the 'review-verdict/h10' required check manually before scheduling an auto-merge."
|
||||
if [ -n "${ETV_GITEA_TOKEN:-}" ]; then
|
||||
bp_code=$(curl -s -o "$bp_file" -w '%{http_code}' -H "Authorization: token $ETV_GITEA_TOKEN" "$base_url/repos/$owner/$repo/branch_protections" 2>/dev/null || true)
|
||||
else
|
||||
bp_code=$(curl -s -o "$bp_file" -w '%{http_code}' -u "$ETV_GITEA_BASICAUTH" "$base_url/repos/$owner/$repo/branch_protections" 2>/dev/null || true)
|
||||
fi
|
||||
bp_list=$(cat "$bp_file" 2>/dev/null || true)
|
||||
bp=""
|
||||
if [ "$bp_code" = "200" ] && printf '%s' "$bp_list" | jq -e 'type == "array"' >/dev/null 2>&1; then
|
||||
# DO NOT claim parity with Gitea's matcher — this code cannot have it, and asserting it would
|
||||
# be the exact defect this PR records (a mitigation outside the code, asserted rather than
|
||||
# verified). Gitea compiles a rule name with gobwas/glob and a `/` separator, so its `*` does
|
||||
# NOT cross a slash, `?`/`[…]`/`{a,b}` are wildcards, and a plain name is folded case-
|
||||
# insensitively. Reimplementing that here would be a second copy of somebody else's parser.
|
||||
#
|
||||
# So the classification is deliberately THREE-way, and each arm is safe without knowing the
|
||||
# dialect:
|
||||
# exact — no glob rule could apply, AND some rule name has no glob metacharacter and
|
||||
# equals the base case-insensitively. Only then is a single rule decidable.
|
||||
#
|
||||
# UNDECIDABLE IS EVALUATED FIRST, and the order is the point. Gitea picks the
|
||||
# governing rule with `GetFirstMatched` over a list sorted by Priority, THEN
|
||||
# by plain-name-ness — so a glob rule with a better Priority outranks an
|
||||
# exactly-named one. Preferring `exact` would therefore inspect a rule Gitea
|
||||
# might not be applying: if the exact rule requires `review-verdict/h10` and a
|
||||
# higher-priority glob rule does not, the gate auto-grants on a base where the
|
||||
# check is not enforced. Asking whenever ANY glob rule could apply is sound
|
||||
# without knowing the precedence rules at all, which is the only claim this
|
||||
# code is entitled to make about somebody else's resolver.
|
||||
#
|
||||
# Case folding is ASCII-only here, while Gitea's `EqualFold` is
|
||||
# Unicode-aware — so a rule `ünstable` and a base `Ünstable` fold equal there
|
||||
# and not here. ASCII-fold equality implies EqualFold equality, so the gap can
|
||||
# only MISS a match, never invent one; but a miss lands on `none`, which
|
||||
# DENIES with the stated cause that no rule can govern the base. The backslash
|
||||
# paragraph below rejects "nearly unreachable" as a standard for that arm, and
|
||||
# the same standard has to apply here, so a rule name carrying any non-ASCII
|
||||
# byte is `undecidable` rather than fold-compared. Two fold-equal plain names
|
||||
# are undecidable too: this code picks by list order while Gitea picks by
|
||||
# Priority, and guessing which one is enforced is the defect the arm order
|
||||
# above exists to avoid.
|
||||
# undecidable — some glob rule COULD govern this base. Tested with a provable SUPERSET of any
|
||||
# glob dialect: literal prefix before the first metacharacter, `.*`, literal
|
||||
# suffix after the last. If even that does not match, no dialect can, because
|
||||
# every dialect requires the literal head and tail to match literally.
|
||||
#
|
||||
# BACKSLASH counts as a metacharacter for that purpose, and it is the one case that breaks the
|
||||
# superset proof if it does not. gobwas/glob reads `\{` as a LITERAL brace, so a rule `a\{b`
|
||||
# governs the base `a{b` — while a superset that treated `\` as literal would build `a\.*b`,
|
||||
# fail to match, and answer `none`, i.e. deny a base that IS protected. Git ref rules make this
|
||||
# nearly unreachable (a branch name may not contain `*`, `?`, `[` or `\`, though it MAY contain
|
||||
# `{`), but `none` is the arm that authorises a DENY on the stated grounds "nothing can govern
|
||||
# this base", so its premise has to hold unconditionally rather than usually.
|
||||
# none — nothing can possibly govern the base, so it is genuinely unprotected.
|
||||
#
|
||||
# `undecidable` asks rather than granting or denying. Over-matching would auto-grant on a base
|
||||
# whose protection we never established (#622's hole, reached through the block written to
|
||||
# close it); under-matching would deny with a stated cause that is false, which this block's
|
||||
# own comment calls the worse outcome. Asking is the only answer that is honest in both
|
||||
# directions, and it is rare in practice: as of 2026-08-19 this repo's only rule is the plain
|
||||
# name `main`, which the classifier resolves to `exact` on every run. That is a dated
|
||||
# observation about mutable remote config, not a property to rely on.
|
||||
bp_verdict=$(printf '%s' "$bp_list" | jq --arg b "$base_ref" -c '
|
||||
def esc: gsub("(?<c>[.+?^${}()|\\[\\]\\\\])"; "\\" + .c);
|
||||
def offs: [match("[*?\\[\\]{}\\\\]"; "g").offset];
|
||||
def superset: . as $n | (offs) as $o
|
||||
| ($n[0:$o[0]] | esc) + ".*" + ($n[($o[-1]+1):] | esc);
|
||||
def nonascii: explode | any(. > 127);
|
||||
. as $rules | $b as $base |
|
||||
($rules | map(select((.branch_name // .rule_name // "") as $n
|
||||
| (($n|offs|length) == 0)
|
||||
and (($n|ascii_downcase) == ($base|ascii_downcase))))) as $exacts |
|
||||
(($base|nonascii) or ($rules | any((.branch_name // .rule_name // "") as $n
|
||||
| ($n|offs|length) == 0 and ($n|nonascii)))) as $unfoldable |
|
||||
if ($rules | any((.branch_name // .rule_name // "") as $n
|
||||
| (($n|offs|length) > 0)
|
||||
and ($base | test("^" + ($n|superset) + "$")))) then {verdict:"undecidable"}
|
||||
elif $unfoldable then {verdict:"undecidable"}
|
||||
elif ($exacts | length) > 1 then {verdict:"undecidable"}
|
||||
elif ($exacts | length) == 1 then {verdict:"exact", rule:($exacts | first)}
|
||||
else {verdict:"none"} end' 2>/dev/null || true)
|
||||
case $(printf '%s' "$bp_verdict" | jq -r '.verdict // ""' 2>/dev/null || true) in
|
||||
exact) bp=$(printf '%s' "$bp_verdict" | jq -c '.rule' 2>/dev/null || true); bp_code=200 ;;
|
||||
undecidable) rm -f "$bp_file"
|
||||
decide ask "H6/H10 merge gate: no branch-protection rule on this repo governs '$base_ref' decidably — a GLOB rule could govern it, or two rule names fold-equal, or a name is non-ASCII. This hook deliberately does not reimplement Gitea's glob matcher, so whether 'review-verdict/h10' is required on this base cannot be derived here (ersatztv#778). Confirm it in the repo's branch-protection settings, or merge immediately instead of scheduling." ;;
|
||||
none) bp_code=nomatch; bp="" ;;
|
||||
*) bp_code=unreadable-rules; bp="" ;;
|
||||
esac
|
||||
else
|
||||
# A 200 whose body is NOT an array never reaches the classifier — it is diverted by the array
|
||||
# gate above — so it needs the same sentinel, or the generic ask below reports
|
||||
# "HTTP '200' — Gitea unreachable" about a read that plainly succeeded. Same defect as the
|
||||
# throw-inside-the-classifier arm, one branch earlier; fixing only the arm where it was noticed
|
||||
# is the twin-missed shape this PR is largely about.
|
||||
if [ "$bp_code" = "200" ]; then
|
||||
bp_code=unreadable-rules
|
||||
else
|
||||
bp_code=${bp_code:-000} # a real transport/HTTP failure -> the ask arm below
|
||||
fi
|
||||
bp=""
|
||||
fi
|
||||
rm -f "$bp_file"
|
||||
# `nomatch` is the CLASSIFIER's verdict, deliberately not an HTTP code. Reusing 404 for it made
|
||||
# this deny reachable from an HTTP 404 on the list read too — repo not found, or invisible to the
|
||||
# credential, which Gitea also answers 404 — and then the reason claimed "the full rule list was
|
||||
# read and none matches" about a read that never happened. A transport failure must reach the ask
|
||||
# below, not a deny stating a finding.
|
||||
if [ "$bp_code" = "nomatch" ]; then
|
||||
decide deny "H6/H10 merge gate: BLOCKED — no branch-protection rule on this repo can govern '$base_ref' (the full rule list was read and none matches), so 'review-verdict/h10' is not a required check on it. A scheduled auto-merge is safe ONLY because that per-sha required check stops a commit pushed after scheduling from merging unreviewed (ersatztv#622). Restore branch protection on '$base_ref', or merge immediately (without merge_when_checks_succeed) once CI is green."
|
||||
fi
|
||||
# `unreadable-rules` is the CLASSIFIER failing on a 200 it could not parse — a numeric
|
||||
# `branch_name` makes jq throw, and `//` does not catch it because it fires only on null/false.
|
||||
# It gets its own sentinel for the same reason `nomatch` does: reporting "HTTP '000' — Gitea
|
||||
# unreachable" about a successful 200 read states a cause that did not happen, which is the defect
|
||||
# fixed one arm over for the deny.
|
||||
if [ "$bp_code" = "unreadable-rules" ]; then
|
||||
decide ask "H6/H10 merge gate: this repo's branch-protection rules came back in a shape this hook could not parse, so whether 'review-verdict/h10' is required on '$base_ref' is unknown. Check the rules manually, or merge immediately instead of scheduling."
|
||||
fi
|
||||
if [ "$bp_code" != "200" ] || [ -z "${bp//[[:space:]]/}" ] || ! printf '%s' "$bp" | jq -e 'type == "object"' >/dev/null 2>&1; then
|
||||
decide ask "H6/H10 merge gate: could not read this repo's branch-protection rules (HTTP '${bp_code:-none}' — Gitea unreachable, or these credentials lack the repo-admin scope that endpoint needs), so whether 'review-verdict/h10' is required on '$base_ref' is unknown. Scheduling an auto-merge is only safe while 'review-verdict/h10' is a REQUIRED check there (ersatztv#622) — confirm that manually, or merge immediately instead of scheduling."
|
||||
fi
|
||||
# The membership test is `any(.[]; . == …)` over a value FIRST PROVEN to be an array of strings —
|
||||
# never `index()`. `index` on a STRING is substring search, so a `status_check_contexts` that
|
||||
# arrived as the string "prefix-review-verdict/h10-suffix" would answer "yes" and auto-grant a
|
||||
# merge on a base where no such context is required. That is a FALSE-OPEN in the gate, reachable
|
||||
# from any payload shape drift, and it is the direction that matters: a false-closed costs a
|
||||
# prompt, a false-open costs an unreviewed merge.
|
||||
#
|
||||
# Validating `$bp` as an object does not make its MEMBERS well-formed, which is the same
|
||||
# one-level-down swallow that survived the first fix in the #632 base-change guard — the
|
||||
# validation domain has to match the CONSUMPTION domain, not stop at the top-level type. So the
|
||||
# shape is checked explicitly and anything else becomes "unknown" rather than a decision.
|
||||
#
|
||||
# `null` and `[]` are legitimate (an unprotected-in-practice branch) and answer "no", not
|
||||
# "unknown": absent IS the finding here, not a read failure. The word is then matched
|
||||
# exhaustively, because "" is not a third synonym for "no".
|
||||
# `// []` defaults on FALSE as well as on null, because jq's alternative operator fires for both.
|
||||
# So `"status_check_contexts": false` — a malformed shape — became `[]` and answered "no", i.e. a
|
||||
# confident DENY derived from a payload that was never understood. Absent and null are defaulted
|
||||
# explicitly; every other non-array is "unknown".
|
||||
# `enable_status_check` is validated as a BOOLEAN before it is trusted, for the same reason the
|
||||
# contexts list is: `"true"` (the string) is not `true`, and comparing it to `true` yields a
|
||||
# confident "no" -> deny derived from a payload never understood. Every malformed shape on this
|
||||
# endpoint has to reach the same "unknown" -> ask arm, or the tri-state is only two states.
|
||||
guarded=$(printf '%s' "$bp" \
|
||||
| jq -r 'def ctxs: if (has("status_check_contexts") | not) or .status_check_contexts == null
|
||||
then [] else .status_check_contexts end;
|
||||
if (.enable_status_check | type) != "boolean" then "unknown"
|
||||
elif (ctxs | type) != "array" or any(ctxs[]; type != "string") then "unknown"
|
||||
elif (.enable_status_check == true) and any(ctxs[]; . == "review-verdict/h10") then "yes"
|
||||
else "no" end' 2>/dev/null || true)
|
||||
case "$guarded" in
|
||||
yes) : ;;
|
||||
no) decide deny "H6/H10 merge gate: BLOCKED — 'review-verdict/h10' is NOT a required status check on '$base_ref' (branch protection reports enable_status_check/status_check_contexts without it). A scheduled auto-merge is safe ONLY because that per-sha required check stops a commit pushed after scheduling from merging unreviewed (ersatztv#622); without it, arming merge_when_checks_succeed freezes consent at a head Gitea may not be the one to merge. Restore it in branch protection, or merge immediately (without merge_when_checks_succeed) once CI is green." ;;
|
||||
*) decide ask "H6/H10 merge gate: branch protection for '$base_ref' came back in an unexpected shape, so the 'review-verdict/h10' required check that makes a scheduled auto-merge safe (ersatztv#622) could not be confirmed either way. Check it manually, or merge immediately instead of scheduling." ;;
|
||||
esac
|
||||
fi
|
||||
|
||||
# --- (c) Review-verdict freshness (ersatztv#303 H10): a review-verdict comment must reference the
|
||||
@@ -365,7 +620,7 @@ if [ "$class" = "positive" ]; then
|
||||
# path this hook never read the CI status at all (it is delegated to Gitea), so saying "CI green"
|
||||
# there was a plain falsehood in the one message a human reads to decide whether to trust the gate.
|
||||
if [ "$mwcs" = "true" ]; then
|
||||
decide grant "H6/H10 merge gate: satisfied — all Done-when boxes ticked, and both a positive Review-verdict comment and the 'review-verdict/h10' status cover the current head ($short). CI is gated by Gitea (merge_when_checks_succeed), and because the verdict status is bound to this sha, a commit pushed before Gitea merges will clear it and block the merge (ersatztv#622). Auto-granted."
|
||||
decide grant "H6/H10 merge gate: satisfied — all Done-when boxes ticked, and both a positive Review-verdict comment and the 'review-verdict/h10' status cover the current head ($short). CI is gated by Gitea (merge_when_checks_succeed). A commit pushed before Gitea merges clears the sha-bound verdict status and is blocked by the 'review-verdict/h10' required check (ersatztv#622) — which this hook has just CONFIRMED is still required on '$base_ref' — read from the repo's full rule list and matched with Gitea's own plain-vs-glob split, refusing rather than guessing wherever precedence or folding is not derivable. That guarantee holds while that branch protection stands; if it is weakened after this check, nothing here would see it (ersatztv#778). Auto-granted."
|
||||
fi
|
||||
decide grant "H6/H10 merge gate: satisfied — CI green, all Done-when boxes ticked, and a positive Review-verdict references the current head ($short). Auto-granted (no separate confirmation needed)."
|
||||
fi
|
||||
|
||||
@@ -2,6 +2,13 @@
|
||||
# PreToolUse / browser-navigate — deny opening download/stream endpoints in a tab
|
||||
# (they hang the MCP session; curl them instead). Fail-open on parse trouble.
|
||||
set -euo pipefail
|
||||
|
||||
# ersatztv#776 — report that this hook fired. MUST precede any stdin read.
|
||||
# Claude hook: decides by printed JSON, so stdout is captured.
|
||||
ETV_HOOK_FIRE_LIB="${CLAUDE_PROJECT_DIR:-$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." 2>/dev/null && pwd)}/scripts/hook-fire-log.sh" || true
|
||||
[ -r "$ETV_HOOK_FIRE_LIB" ] && . "$ETV_HOOK_FIRE_LIB" || true
|
||||
type etv_hook_fire_begin >/dev/null 2>&1 || etv_hook_fire_begin() { :; }
|
||||
etv_hook_fire_begin pretooluse-nav-guard "" capture || true
|
||||
input=$(cat)
|
||||
url=$(printf '%s' "$input" | jq -r '.tool_input.url // ""' 2>/dev/null || true)
|
||||
|
||||
|
||||
@@ -8,6 +8,13 @@
|
||||
# So the main tree (never marked) and pre-convention worktrees (no marker) are unaffected;
|
||||
# only a commit/merge into another session's marked worktree is blocked.
|
||||
set -euo pipefail
|
||||
|
||||
# ersatztv#776 — report that this hook fired. MUST precede any stdin read.
|
||||
# Claude hook: decides by printed JSON, so stdout is captured.
|
||||
ETV_HOOK_FIRE_LIB="${CLAUDE_PROJECT_DIR:-$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." 2>/dev/null && pwd)}/scripts/hook-fire-log.sh" || true
|
||||
[ -r "$ETV_HOOK_FIRE_LIB" ] && . "$ETV_HOOK_FIRE_LIB" || true
|
||||
type etv_hook_fire_begin >/dev/null 2>&1 || etv_hook_fire_begin() { :; }
|
||||
etv_hook_fire_begin pretooluse-worktree-guard "" capture || true
|
||||
input=$(cat)
|
||||
cmd=$(printf '%s' "$input" | jq -r '.tool_input.command // ""' 2>/dev/null || true)
|
||||
cwd=$(printf '%s' "$input" | jq -r '.cwd // ""' 2>/dev/null || true)
|
||||
|
||||
@@ -363,7 +363,7 @@ docker start ersatztv
|
||||
```
|
||||
- **Dispatcharr caches ErsatzTV's XMLTV.** Repointing its DB rows is not enough — it keeps serving a stale EPG full of dead `ersatztv:8409` artwork URLs (breaks Kodi artwork). Force a refresh (EPG source 9):
|
||||
```bash
|
||||
ssh timothy@192.168.1.99 'docker exec dispatcharr python manage.py shell -c \
|
||||
ssh timothy@192.168.1.29 'docker exec dispatcharr python manage.py shell -c \
|
||||
"from apps.epg.tasks import refresh_epg_data; refresh_epg_data(9)"'
|
||||
```
|
||||
- **`/api/health` returns 401** (needs an API key). The Telegraf probe has no `response_string_match`, so ErsatzTV reads as **unhealthy in Grafana** — a false alarm, and **pre-existing**, not caused by the move. The container healthcheck uses the unauthenticated internal `/health` and is unaffected.
|
||||
|
||||
@@ -41,6 +41,17 @@ name: Build ErsatzTV Image
|
||||
# (verified, throwaway PR #418) and we don't rely on how branch protection treats a skipped
|
||||
# REQUIRED context. See docs/ci-cd.md -> "Docs-only skip".
|
||||
#
|
||||
# RELEASE-PATH DELIMITER GATE (ersatztv#767): the `scan` job runs the PyYAML-based delimiter-ban
|
||||
# test and is a `needs:` of `build`, so a `${{` opener in a banned job's `run:` body means `build`
|
||||
# never runs. It is deliberately NOT gated by either skip below: the gate's coverage must not depend
|
||||
# on a detector the gate is not allowed to trust, and it is cheap enough that gating it buys nothing.
|
||||
# (Do NOT justify that with "the docs-only path still builds an image" — it does not. `Build and
|
||||
# push` carries the docs_only gate too; a tag build is unaffected only because the script forces
|
||||
# docs_only=false there.) Note it installs from PyPI (setup-python + pip), putting a NEW network
|
||||
# dependency between a `v*` tag and its image. Not the only one on this path — `test` runs
|
||||
# `dotnet restore` and `npm ci` behind actions/cache, and a cache miss reaches nuget.org/npm — but
|
||||
# newly added here. Fail-closed and loud, and still a real availability dependency.
|
||||
#
|
||||
# ALREADY-VALIDATED SKIP (ersatztv#420): a second, sibling gate in `test`, `migrations` and
|
||||
# `functional-e2e` only (NOT `build`). On a push-to-main merge commit, `id: revalidate` runs
|
||||
# `scripts/ci-detect-already-validated.sh`, which emits `skip=true` only when the merged tree is
|
||||
@@ -122,14 +133,23 @@ jobs:
|
||||
# ersatztv#416: is this a docs-only change? If so, every heavy step below is skipped and this
|
||||
# REQUIRED job reports success in seconds. It still RUNS (never `if:`-skipped) so the required
|
||||
# context keeps reporting — see the workflow header and docs/ci-cd.md -> "Docs-only skip".
|
||||
# EVERY consequential `run:` step in this job marks itself as its FIRST act (ersatztv#756),
|
||||
# and the trailing `Assert every expected step executed` guard fails the job when one is
|
||||
# missing. This is a REQUIRED context on `main`, and a step the runner drops takes the job
|
||||
# GREEN having done no work — see scripts/ci-step-ran.sh for why that is fail-OPEN here while
|
||||
# the same drop in review-verdict.yml is fail-CLOSED.
|
||||
- name: Detect docs-only changes
|
||||
id: detect
|
||||
run: scripts/ci-detect-docs-only.sh
|
||||
run: |
|
||||
"${GITHUB_WORKSPACE:-.}/scripts/ci-step-ran.sh" mark detect
|
||||
scripts/ci-detect-docs-only.sh
|
||||
- name: Detect already-validated tree (#420)
|
||||
id: revalidate
|
||||
env:
|
||||
ETV_STATUS_AUTH: ${{ secrets.REGISTRY_USER }}:${{ secrets.REGISTRY_PASSWORD }}
|
||||
run: scripts/ci-detect-already-validated.sh
|
||||
run: |
|
||||
"${GITHUB_WORKSPACE:-.}/scripts/ci-step-ran.sh" mark revalidate
|
||||
scripts/ci-detect-already-validated.sh
|
||||
|
||||
- name: Cache NuGet packages
|
||||
if: steps.detect.outputs.docs_only != 'true' && steps.revalidate.outputs.skip != 'true'
|
||||
@@ -141,7 +161,9 @@ jobs:
|
||||
|
||||
- name: Restore
|
||||
if: steps.detect.outputs.docs_only != 'true' && steps.revalidate.outputs.skip != 'true'
|
||||
run: dotnet restore
|
||||
run: |
|
||||
"${GITHUB_WORKSPACE:-.}/scripts/ci-step-ran.sh" mark restore
|
||||
dotnet restore
|
||||
|
||||
# Replaces setup-node's built-in `cache: npm`. The toolchain image supplies node/npm, but
|
||||
# the SPA's package downloads are project deps, so they stay cached per lockfile.
|
||||
@@ -156,36 +178,50 @@ jobs:
|
||||
- name: Install SPA dependencies
|
||||
if: steps.detect.outputs.docs_only != 'true' && steps.revalidate.outputs.skip != 'true'
|
||||
working-directory: web
|
||||
run: npm ci
|
||||
run: |
|
||||
"${GITHUB_WORKSPACE:-.}/scripts/ci-step-ran.sh" mark npm-ci
|
||||
npm ci
|
||||
|
||||
- name: Check generated SPA API client
|
||||
if: steps.detect.outputs.docs_only != 'true' && steps.revalidate.outputs.skip != 'true'
|
||||
working-directory: web
|
||||
run: npm run check:api
|
||||
run: |
|
||||
"${GITHUB_WORKSPACE:-.}/scripts/ci-step-ran.sh" mark check-api
|
||||
npm run check:api
|
||||
|
||||
- name: Lint SPA
|
||||
if: steps.detect.outputs.docs_only != 'true' && steps.revalidate.outputs.skip != 'true'
|
||||
working-directory: web
|
||||
run: npm run lint
|
||||
run: |
|
||||
"${GITHUB_WORKSPACE:-.}/scripts/ci-step-ran.sh" mark lint
|
||||
npm run lint
|
||||
|
||||
- name: Typecheck SPA
|
||||
if: steps.detect.outputs.docs_only != 'true' && steps.revalidate.outputs.skip != 'true'
|
||||
working-directory: web
|
||||
run: npm run typecheck
|
||||
run: |
|
||||
"${GITHUB_WORKSPACE:-.}/scripts/ci-step-ran.sh" mark typecheck
|
||||
npm run typecheck
|
||||
|
||||
- name: Test SPA
|
||||
if: steps.detect.outputs.docs_only != 'true' && steps.revalidate.outputs.skip != 'true'
|
||||
working-directory: web
|
||||
run: npm test -- --run
|
||||
run: |
|
||||
"${GITHUB_WORKSPACE:-.}/scripts/ci-step-ran.sh" mark web-test
|
||||
npm test -- --run
|
||||
|
||||
- name: Build SPA
|
||||
if: steps.detect.outputs.docs_only != 'true' && steps.revalidate.outputs.skip != 'true'
|
||||
working-directory: web
|
||||
run: npm run build
|
||||
run: |
|
||||
"${GITHUB_WORKSPACE:-.}/scripts/ci-step-ran.sh" mark web-build
|
||||
npm run build
|
||||
|
||||
- name: Strip Scanner project ref (matches Docker build)
|
||||
if: steps.detect.outputs.docs_only != 'true' && steps.revalidate.outputs.skip != 'true'
|
||||
run: sed -i '/Scanner/d' ErsatzTV/ErsatzTV.csproj
|
||||
run: |
|
||||
"${GITHUB_WORKSPACE:-.}/scripts/ci-step-ran.sh" mark strip-scanner
|
||||
sed -i '/Scanner/d' ErsatzTV/ErsatzTV.csproj
|
||||
|
||||
# Start the true peak-anon sampler just before the memory-heavy dotnet Build/Test/Coverage so
|
||||
# its high-water mark spans them (SPA build/test above are comparatively light). Paired with the
|
||||
@@ -199,13 +235,16 @@ jobs:
|
||||
|
||||
- name: Build
|
||||
if: steps.detect.outputs.docs_only != 'true' && steps.revalidate.outputs.skip != 'true'
|
||||
run: dotnet build --configuration Release --no-restore
|
||||
run: |
|
||||
"${GITHUB_WORKSPACE:-.}/scripts/ci-step-ran.sh" mark build
|
||||
dotnet build --configuration Release --no-restore
|
||||
|
||||
- name: Test
|
||||
if: steps.detect.outputs.docs_only != 'true' && steps.revalidate.outputs.skip != 'true'
|
||||
run: >-
|
||||
dotnet test --configuration Release --no-build --blame-hang-timeout "2m" --verbosity normal
|
||||
--collect:"XPlat Code Coverage" --settings coverlet.runsettings --results-directory ./coverage
|
||||
run: |
|
||||
"${GITHUB_WORKSPACE:-.}/scripts/ci-step-ran.sh" mark dotnet-test
|
||||
dotnet test --configuration Release --no-build --blame-hang-timeout "2m" --verbosity normal \
|
||||
--collect:"XPlat Code Coverage" --settings coverlet.runsettings --results-directory ./coverage
|
||||
|
||||
# Coverage reporting (ersatztv#15 scope item 4): coverlet.collector emits a Cobertura report
|
||||
# per test project (via --collect above); ReportGenerator merges them into a human-readable
|
||||
@@ -258,6 +297,43 @@ jobs:
|
||||
continue-on-error: true
|
||||
run: scripts/ci-peak-anon.sh report
|
||||
|
||||
# THE DROPPED-STEP GUARD (ersatztv#756). Every `run:` step above records that it began; this
|
||||
# asserts the whole expected SET was recorded. A step the runner declines to interpolate is
|
||||
# DROPPED and still concludes `success` (ersatztv#751), so without this a REQUIRED context
|
||||
# reports green having done no work — fail-OPEN, and strictly worse than the fail-CLOSED
|
||||
# version of the same bug that #751 fixed in review-verdict.yml.
|
||||
#
|
||||
# NO `if:` HERE, WHICH IS A DELIBERATE DEPARTURE FROM THE #751 GUARD and the one decision in
|
||||
# this block that is easy to "fix" wrongly. That guard uses `if: always()` because its job has
|
||||
# exactly one real step, so there is no ordinary red for it to talk over. Here there are
|
||||
# twelve, and a genuine failure in an early one (a lint error, a failing test) SKIPS every
|
||||
# later step — an `always()` guard would then announce "these steps never executed: typecheck
|
||||
# web-test build dotnet-test" on top of every normal red build. That is not a dropped step, it
|
||||
# is the runner doing what it is told, and a guard that cries wolf on every red build is a
|
||||
# guard that gets deleted.
|
||||
#
|
||||
# The default `if:` is `success()`, which is exactly the condition wanted, and the invariant it
|
||||
# rests on is worth stating because it is what makes the omission safe rather than lucky: this
|
||||
# step is skipped ONLY when an earlier step failed, and an earlier step failing already fails
|
||||
# the job. So `guard skipped => job red`, and the only path to a green job runs the guard. A
|
||||
# dropped step is invisible precisely because it concludes `success`, which keeps the job green
|
||||
# and therefore reaches here.
|
||||
#
|
||||
# ITS OWN BODY CANNOT BE DROPPED BY THE MECHANISM IT GUARDS AGAINST: it is a single command
|
||||
# with no expression delimiter anywhere in the scalar, so the runner has nothing to rewrite.
|
||||
# The two gate values come in through `env:`, which is interpolated PER VALUE — a bad payload
|
||||
# there cannot take the body with it (`ci.workflow-run-body-no-expressions`), and both paths
|
||||
# are held to naming a real context by
|
||||
# test_every_workflow_expression_names_a_REAL_context_or_function.
|
||||
- name: Assert every expected step executed (ersatztv#756)
|
||||
env:
|
||||
ETV_DOCS_ONLY: ${{ steps.detect.outputs.docs_only }}
|
||||
ETV_REVALIDATE_SKIP: ${{ steps.revalidate.outputs.skip }}
|
||||
run: >-
|
||||
scripts/ci-step-ran.sh assert
|
||||
--always detect revalidate
|
||||
--gated restore npm-ci check-api lint typecheck web-test web-build strip-scanner build dotnet-test
|
||||
|
||||
migrations:
|
||||
name: EF migration integrity (SQLite + MySql)
|
||||
runs-on: ubuntu-latest
|
||||
@@ -328,14 +404,21 @@ jobs:
|
||||
|
||||
# ersatztv#416: docs-only? Skip the build + migration replay; the job still reports success in
|
||||
# seconds. REQUIRED context, so it always RUNS (never `if:`-skipped). See the workflow header.
|
||||
# Same per-step marker contract as the `test` job above (ersatztv#756) — this is the other
|
||||
# REQUIRED context, so a dropped migration-replay step would report EF integrity green having
|
||||
# replayed nothing.
|
||||
- name: Detect docs-only changes
|
||||
id: detect
|
||||
run: scripts/ci-detect-docs-only.sh
|
||||
run: |
|
||||
"${GITHUB_WORKSPACE:-.}/scripts/ci-step-ran.sh" mark detect
|
||||
scripts/ci-detect-docs-only.sh
|
||||
- name: Detect already-validated tree (#420)
|
||||
id: revalidate
|
||||
env:
|
||||
ETV_STATUS_AUTH: ${{ secrets.REGISTRY_USER }}:${{ secrets.REGISTRY_PASSWORD }}
|
||||
run: scripts/ci-detect-already-validated.sh
|
||||
run: |
|
||||
"${GITHUB_WORKSPACE:-.}/scripts/ci-step-ran.sh" mark revalidate
|
||||
scripts/ci-detect-already-validated.sh
|
||||
|
||||
- name: Cache NuGet packages
|
||||
if: steps.detect.outputs.docs_only != 'true' && steps.revalidate.outputs.skip != 'true'
|
||||
@@ -347,11 +430,15 @@ jobs:
|
||||
|
||||
- name: Restore
|
||||
if: steps.detect.outputs.docs_only != 'true' && steps.revalidate.outputs.skip != 'true'
|
||||
run: dotnet restore
|
||||
run: |
|
||||
"${GITHUB_WORKSPACE:-.}/scripts/ci-step-ran.sh" mark restore
|
||||
dotnet restore
|
||||
|
||||
- name: Build
|
||||
if: steps.detect.outputs.docs_only != 'true' && steps.revalidate.outputs.skip != 'true'
|
||||
run: dotnet build --configuration Release --no-restore
|
||||
run: |
|
||||
"${GITHUB_WORKSPACE:-.}/scripts/ci-step-ran.sh" mark build
|
||||
dotnet build --configuration Release --no-restore
|
||||
|
||||
# dotnet-ef is baked into the CI toolchain image (docker/ci/Dockerfile) and already on PATH
|
||||
# — no per-run `dotnet tool install`. Bump its version there (ersatztv#390).
|
||||
@@ -361,6 +448,7 @@ jobs:
|
||||
if: steps.detect.outputs.docs_only != 'true' && steps.revalidate.outputs.skip != 'true'
|
||||
run: |
|
||||
set -euo pipefail
|
||||
"${GITHUB_WORKSPACE:-.}/scripts/ci-step-ran.sh" mark sqlite
|
||||
echo "::group::SQLite model drift (has-pending-model-changes)"
|
||||
dotnet ef migrations has-pending-model-changes --no-build --configuration Release \
|
||||
--context TvContext --startup-project ErsatzTV --project ErsatzTV.Infrastructure.Sqlite -- --provider Sqlite
|
||||
@@ -384,6 +472,7 @@ jobs:
|
||||
MySql__ConnectionString: "Server=mysql;Port=3306;Database=ersatztv_migrations;Uid=root;Pwd=ersatztv;DefaultCommandTimeout=300;"
|
||||
run: |
|
||||
set -euo pipefail
|
||||
"${GITHUB_WORKSPACE:-.}/scripts/ci-step-ran.sh" mark mysql
|
||||
echo "::group::MySql model drift (has-pending-model-changes)"
|
||||
dotnet ef migrations has-pending-model-changes --no-build --configuration Release \
|
||||
--context TvContext --startup-project ErsatzTV --project ErsatzTV.Infrastructure.MySql -- --provider MySql
|
||||
@@ -418,6 +507,42 @@ jobs:
|
||||
# how the original defects escaped. The fixture itself is retained and is opt-in via
|
||||
# ETV_TEST_MYSQL_CONNECTION (skipped, visibly, without it). Re-arming it here is tracked by #627.
|
||||
|
||||
# THE DROPPED-STEP GUARD (ersatztv#756). Every `run:` step above records that it began; this
|
||||
# asserts the whole expected SET was recorded. A step the runner declines to interpolate is
|
||||
# DROPPED and still concludes `success` (ersatztv#751), so without this a REQUIRED context
|
||||
# reports green having done no work — fail-OPEN, and strictly worse than the fail-CLOSED
|
||||
# version of the same bug that #751 fixed in review-verdict.yml.
|
||||
#
|
||||
# NO `if:` HERE, WHICH IS A DELIBERATE DEPARTURE FROM THE #751 GUARD and the one decision in
|
||||
# this block that is easy to "fix" wrongly. That guard uses `if: always()` because its job has
|
||||
# exactly one real step, so there is no ordinary red for it to talk over. Here a genuine
|
||||
# failure in an early step (a failing `dotnet build`, a MySql replay error) SKIPS every later
|
||||
# step — an `always()` guard would then announce "these steps never executed: sqlite mysql" on
|
||||
# top of every normal red build. That is not a dropped step, it is the runner doing what it is
|
||||
# told, and a guard that cries wolf on every red build is a guard that gets deleted.
|
||||
#
|
||||
# The default `if:` is `success()`, which is exactly the condition wanted, and the invariant it
|
||||
# rests on is worth stating because it is what makes the omission safe rather than lucky: this
|
||||
# step is skipped ONLY when an earlier step failed, and an earlier step failing already fails
|
||||
# the job. So `guard skipped => job red`, and the only path to a green job runs the guard. A
|
||||
# dropped step is invisible precisely because it concludes `success`, which keeps the job green
|
||||
# and therefore reaches here.
|
||||
#
|
||||
# ITS OWN BODY CANNOT BE DROPPED BY THE MECHANISM IT GUARDS AGAINST: it is a single command
|
||||
# with no expression delimiter anywhere in the scalar, so the runner has nothing to rewrite.
|
||||
# The two gate values come in through `env:`, which is interpolated PER VALUE — a bad payload
|
||||
# there cannot take the body with it (`ci.workflow-run-body-no-expressions`), and both paths
|
||||
# are held to naming a real context by
|
||||
# test_every_workflow_expression_names_a_REAL_context_or_function.
|
||||
- name: Assert every expected step executed (ersatztv#756)
|
||||
env:
|
||||
ETV_DOCS_ONLY: ${{ steps.detect.outputs.docs_only }}
|
||||
ETV_REVALIDATE_SKIP: ${{ steps.revalidate.outputs.skip }}
|
||||
run: >-
|
||||
scripts/ci-step-ran.sh assert
|
||||
--always detect revalidate
|
||||
--gated restore build sqlite mysql
|
||||
|
||||
functional-e2e:
|
||||
name: Functional E2E (curl + UI contracts)
|
||||
runs-on: ubuntu-latest
|
||||
@@ -529,6 +654,106 @@ jobs:
|
||||
# server. Its exit status is Playwright's.
|
||||
scripts/e2e-ui.sh
|
||||
|
||||
# THE DELIMITER BAN, RE-CHECKED ON THE RELEASE PATH ITSELF (ersatztv#767).
|
||||
#
|
||||
# The ban that keeps `build`'s `Smoke + IPTV E2E` from being silently dropped was enforced only by
|
||||
# `test_the_delimiter_banned_jobs_have_NO_expression_delimiter_in_any_run_body` in the
|
||||
# `script-tests` job of pr-checks.yml — `on: pull_request`, and NOT a required context. So the ban
|
||||
# was REVIEW-TIME only: nothing re-checked it on a `v*` tag push, which is precisely when the
|
||||
# candidate image is published and `DeployStack jazz-media` promotes it.
|
||||
#
|
||||
# WHY A JOB AND NOT A STEP INSIDE `build`. A step cannot protect the thing it shares a job with:
|
||||
# `build` is what publishes, so a guard step there fails OPEN if the runner drops it, and "my body
|
||||
# has no opener so I cannot be dropped" is circular when the only thing enforcing that property is
|
||||
# the same PR-only test being backstopped. As a `needs:` of `build`, a red here means `build` never
|
||||
# runs at all — the image is not built, let alone pushed. Fail-closed by dependency, not by
|
||||
# assertion.
|
||||
#
|
||||
# WHY IT RUNS THE REAL PYTEST rather than a bespoke scanner. The first cut of #767 hand-parsed the
|
||||
# workflow YAML in stdlib Python, to avoid provisioning PyYAML on `build`'s bare runner. Two
|
||||
# independent reviews found ~10 false NEGATIVES in that parser within one round (flow mappings
|
||||
# `{run: …}`, a quoted `"run":` key, aliases, multiline quoted scalars) — i.e. it was strictly
|
||||
# WEAKER than the check it was meant to backstop, in the one direction that matters for a security
|
||||
# gate. Running the existing PyYAML-based test needs no second implementation of "what is a `run:`
|
||||
# body" and therefore has no drift surface. `small` is git-only, so Python is provisioned here the
|
||||
# same way `script-tests` does it.
|
||||
#
|
||||
# This job's OWN steps carry #756 markers and a trailing assert, so a drop inside THIS job is
|
||||
# caught too. That terminates the regress at the same axiom the sibling guards already rest on —
|
||||
# to fail open you must now drop the pytest step AND the assert step, rather than either one.
|
||||
#
|
||||
# THIS PUTS A `small`-LANE JOB BACK ON THE TAG PATH, which ersatztv#535 deliberately moved away
|
||||
# from — say so rather than letting it look accidental. #535 split the git-only gates into
|
||||
# pr-checks.yml because on the v26.12.0 tag they wedged in act's setup phase, were killed, and
|
||||
# reported `failure` with no logs. The blast radius here is WORSE than it was then: as a `needs:`
|
||||
# of `build`, that flake would not merely redden a status, it would skip the build and produce no
|
||||
# release image at all.
|
||||
#
|
||||
# It is acceptable now for a stated reason rather than an assumed one, and the evidence is weaker
|
||||
# than it first looks — so read the limits. Per `ci.small-lane-git-only`, the lane's per-job cap was
|
||||
# forced to 10g by its two HEAVIEST members (this file's `build` AND ci-image.yml's toolchain
|
||||
# buildx), not by `build` alone, and that cap is what pinned the lane to one slot on a 25 GiB host;
|
||||
# both were moved off in server-management#639, after which the lane is git-only and runs wide and
|
||||
# tiny. What has NOT been demonstrated is this lane on a TAG PUSH: `script-tests` runs there happily
|
||||
# but lives in pr-checks.yml (`on: pull_request`), so it has never exercised the condition #535
|
||||
# measured, and #767's own runs (1928/1929) were `workflow_dispatch` on a scratch branch. The
|
||||
# lane-width argument is what carries this, not a like-for-like observation. If the wedging returns,
|
||||
# move this job to `ubuntu-latest` rather than weakening the `needs:` edge — a slower gate is fine,
|
||||
# an optional one is not.
|
||||
scan:
|
||||
name: Delimiter ban (release path)
|
||||
runs-on: small
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: '3.x'
|
||||
- name: Install test dependencies
|
||||
run: |
|
||||
"${GITHUB_WORKSPACE:-.}/scripts/ci-step-ran.sh" mark deps
|
||||
python3 -m pip install --disable-pip-version-check --quiet pytest pyyaml
|
||||
# The ban test plus the structural tests that hold this job's own shape. NOT the whole
|
||||
# scripts/tests suite: that is `script-tests`'s job, it needs jq/git preflights, and an
|
||||
# unrelated pytest regression must not be able to block a release.
|
||||
- name: Run the delimiter-ban tests
|
||||
run: |
|
||||
"${GITHUB_WORKSPACE:-.}/scripts/ci-step-ran.sh" mark ban
|
||||
PYTHONPATH=. python3 -m pytest scripts/tests/test_ci_dropped_step_guard.py scripts/tests/test_ci_release_path_scan_job.py -q
|
||||
# THE POSITIVE CONTROL, and it is deliberately NOT a test (ersatztv#767). The step above proves
|
||||
# the ban HOLDS; it cannot prove the ban would NOTICE. Review disarmed the entire gate with one
|
||||
# repo-root `pytest.ini` (`addopts = -k "not delimiter_banned"`) or `conftest.py`
|
||||
# (`pytest_collection_modifyitems`), which deselects the ban test and every test guarding it,
|
||||
# leaving all jobs green with a delimiter sitting in `Smoke`. Nothing inside pytest can be
|
||||
# trusted to catch that, because pytest's own configuration outranks it.
|
||||
#
|
||||
# So this poisons the checked-out workflow, re-runs the SAME command, and fails the job if it
|
||||
# PASSES. It runs in the real checkout — an isolated copy does not inherit the repo-root config
|
||||
# a disarm would live in, which made the first version of this script report healthy while the
|
||||
# job's real invocation was deselected. The workflow file is restored by an EXIT trap.
|
||||
- name: Prove the ban would DETECT a delimiter (ersatztv#767)
|
||||
run: |
|
||||
"${GITHUB_WORKSPACE:-.}/scripts/ci-step-ran.sh" mark selfcheck
|
||||
scripts/ci-prove-ban-detects.sh
|
||||
# No `if:` — see the sibling guards in `test`/`migrations` for why the default `success()` is
|
||||
# the wanted condition. Both keys are `--always`: every step in this job is unconditional.
|
||||
#
|
||||
# THE MARKER-PATH RATIONALE DOES NOT TRANSFER HERE, and assuming it did would be the mistake
|
||||
# `ci.required-job-step-execution-markers` itself warns about. That record says the run-id and
|
||||
# attempt keying is "defence in depth" because "these jobs get a fresh container, which is the
|
||||
# primary protection". This job has NO `container:` — it is on `small`, where RUNNER_TEMP is
|
||||
# the shared host /tmp. So here the keying is the ONLY protection, and the residual is a
|
||||
# single-job re-run that does not increment GITHUB_RUN_ATTEMPT: it would find the previous
|
||||
# attempt's marker file and the assert would pass even had the pytest step been dropped.
|
||||
# Identity was read off a real run rather than assumed — run 1929 printed
|
||||
# `Marker identity: job=scan run=1929 attempt=1 (from the runner)`, so all three variables are
|
||||
# populated on this lane.
|
||||
- name: Assert every expected step executed (ersatztv#756)
|
||||
run: >-
|
||||
scripts/ci-step-ran.sh assert
|
||||
--always deps ban selfcheck
|
||||
|
||||
build:
|
||||
name: Build & push image (amd64)
|
||||
# Moved back off `small` (server-management#639). This is the one HEAVY job that
|
||||
@@ -540,12 +765,15 @@ jobs:
|
||||
#
|
||||
# The `ubuntu-latest` queueing that sent it to `small` in the first place
|
||||
# (server-management#574: a PR-run skip stuck 31 min behind long builds) does not
|
||||
# come back, because `needs: [test, migrations]` means this job cannot be
|
||||
# dispatched until those two have already finished — by which point the lane it
|
||||
# come back, because `needs: [test, migrations, scan]` means this job cannot be
|
||||
# dispatched until those three have already finished — by which point the lane it
|
||||
# was queueing behind has drained. Real builds (main/tags) get the full
|
||||
# ubuntu-latest allotment: 4 CPUs / 10g on ci-runner (.127).
|
||||
runs-on: ubuntu-latest
|
||||
needs: [test, migrations]
|
||||
# `scan` (ersatztv#767) re-checks the delimiter ban on the release path. As a `needs:` its red
|
||||
# SKIPS this job outright, so a delimiter in `Smoke + IPTV E2E` can no longer reach the point
|
||||
# where an image is published and never booted.
|
||||
needs: [test, migrations, scan]
|
||||
if: github.event_name != 'pull_request'
|
||||
steps:
|
||||
- name: Checkout
|
||||
@@ -616,11 +844,33 @@ jobs:
|
||||
cache-from: type=registry,ref=192.168.1.95:3000/timothy/ersatztv:buildcache
|
||||
cache-to: type=registry,ref=192.168.1.95:3000/timothy/ersatztv:buildcache,mode=max,ignore-error=true
|
||||
|
||||
# THE TWO VALUES COME IN THROUGH `env:`, NOT INLINE (ersatztv#756). This step runs AFTER
|
||||
# `Build and push`, so on a `v*` tag the image is already in the registry as the release
|
||||
# candidate — and it is this smoke run that decides whether the candidate was ever booted at
|
||||
# all. A stray expression delimiter anywhere in this body (a comment is not inert — #751) would
|
||||
# DROP the step and conclude the job `success`: a candidate published, never smoke-tested, and
|
||||
# `DeployStack jazz-media` promotes exactly that image. `env:` is interpolated PER VALUE, so a
|
||||
# bad payload there fails that value instead of taking the whole body with it, and with the
|
||||
# body delimiter-free the class is unreachable here — held by
|
||||
# test_the_delimiter_banned_jobs_have_NO_expression_delimiter_in_any_run_body.
|
||||
#
|
||||
# The ban IS re-checked on the release path now (ersatztv#767): the `scan` job above runs the
|
||||
# PyYAML-based ban test and is a `needs:` of this job, so a delimiter here means `build` never
|
||||
# runs and no image is published. Do not re-add the note that once stood here saying the ban is
|
||||
# "review-time only, tracked as #767" — that was true before the `scan` job existed.
|
||||
#
|
||||
# This step still carries no per-step markers, and that is a genuine (smaller) residual rather
|
||||
# than a dismissal: markers would additionally catch a drop caused by something OTHER than a
|
||||
# delimiter. Adding them needs a bucket modelling this step's publish-ref `if:`, which the
|
||||
# guard's always/gated buckets do not express. The delimiter class itself is covered.
|
||||
- name: Smoke + IPTV E2E (assert key endpoints)
|
||||
if: ${{ (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/v')) && steps.detect.outputs.docs_only != 'true' }}
|
||||
env:
|
||||
SMOKE_SHORT_SHA: ${{ steps.meta.outputs.short }}
|
||||
SMOKE_RUN_ID: ${{ github.run_id }}
|
||||
run: |
|
||||
IMG="${IMAGE}:${{ steps.meta.outputs.short }}"
|
||||
NAME="etv-smoke-${{ github.run_id }}"
|
||||
IMG="${IMAGE}:${SMOKE_SHORT_SHA}"
|
||||
NAME="etv-smoke-${SMOKE_RUN_ID}"
|
||||
trap 'docker rm -f "$NAME" >/dev/null 2>&1 || true' EXIT
|
||||
echo "Pulling ${IMG}"
|
||||
docker pull "$IMG"
|
||||
|
||||
+232
-15
@@ -159,6 +159,36 @@ jobs:
|
||||
echo "Parity-doc reminder: nothing to flag."
|
||||
fi
|
||||
|
||||
# ersatztv#784 — ADVISORY nudge for `docs.no-session-narrative`. Deliberately NON-BLOCKING and
|
||||
# deliberately in this job rather than a gate of its own: it is a string predicate over prose,
|
||||
# and `docs/defect-shapes-773.md` §4 argues that class must not be load-bearing. The script
|
||||
# exits 0 on every path (asserted per argument shape in scripts/tests/test_check_doc_narrative.py,
|
||||
# not only in prose), so this step cannot redden the run even on a hit; if you find yourself
|
||||
# wanting it to fail, read the decision record first — it says no in as many words.
|
||||
# `python3` is not guaranteed on the bare `small` lane (docs/ci-cd.md), and every other
|
||||
# python-using job on it declares this. Without it a missing interpreter is exit 127 — a RED
|
||||
# advisory job joining the combined status, which is the one thing this step must never be.
|
||||
#
|
||||
# Both steps carry `continue-on-error` because the SCRIPT exiting 0 is not the whole invariant:
|
||||
# a setup-python download failure reddens the job just as effectively as a hit would, and an
|
||||
# advisory red still joins the combined status the merge gate reads (ersatztv#598). Scope,
|
||||
# stated rather than implied: this covers the two steps that exist to run the check. A failed
|
||||
# `Checkout` is NOT covered and deliberately so — with no tree there is nothing to check, and
|
||||
# a job that cannot run is a different failure from an advisory one that ran and disagreed.
|
||||
# Measured on this runner (PR#811, run 2179): the job reports `success` and the commit status
|
||||
# context is `success` with both steps green under `continue-on-error`.
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v5
|
||||
continue-on-error: true
|
||||
with:
|
||||
python-version: '3.x'
|
||||
- name: Warn when a doc narrates its own revision history
|
||||
continue-on-error: true
|
||||
run: |
|
||||
base_ref="${{ github.base_ref }}"
|
||||
git fetch --no-tags --depth=100 origin "$base_ref" || true
|
||||
python3 scripts/check-doc-narrative.py --diff "origin/${base_ref}"
|
||||
|
||||
# BLOCKING (ersatztv#521, supersedes the ersatztv#303 H9 append-only mechanic): validates decision-
|
||||
# record lifecycle invariants (metadata schema, one active record per key, reciprocal
|
||||
# supersedes/superseded-by links, no rationale-prose rewrite without a Decisions-Edit: yes git
|
||||
@@ -211,8 +241,125 @@ jobs:
|
||||
# `.claude/hooks/pretooluse-merge-consent.sh`, so its true input set spans at least two top-level
|
||||
# directories. A `scripts/**` filter would silently miss a `.claude/hooks/**` edit — and at ~10s a
|
||||
# filter buys nothing but drift.
|
||||
prove-fix:
|
||||
name: "Fix proofs (Proves trailers)"
|
||||
runs-on: small
|
||||
if: github.event_name == 'pull_request'
|
||||
steps:
|
||||
- name: Checkout
|
||||
# Full history: prove-fix.sh reverts each commit against its PARENT, so a shallow
|
||||
# clone would leave it unable to resolve `<sha>^` and it would refuse every commit.
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: '3.x'
|
||||
- name: Install test dependencies
|
||||
run: python3 -m pip install --disable-pip-version-check --quiet pytest pyyaml
|
||||
# OPT-IN BY TRAILER, deliberately. Requiring `Proves:` on every commit would block
|
||||
# docs, CI and refactor commits that have no code side to revert, and a gate that
|
||||
# blocks ordinary work gets disabled — which is how a check ends up running nowhere
|
||||
# (#631). So the trailer is the AUTHOR'S CLAIM, and this job checks claims: write
|
||||
# one and it must hold. Coverage is therefore honest rather than assumed, and
|
||||
# `docs/decisions/records/testing/fix-ships-a-witnessed-red-test.md` says so.
|
||||
- name: Prove every commit that claims a proof
|
||||
run: |
|
||||
set -uo pipefail
|
||||
base="${{ github.event.pull_request.base.sha }}"
|
||||
head="${{ github.event.pull_request.head.sha }}"
|
||||
echo "range: $base..$head"
|
||||
|
||||
# Capture and VALIDATE the enumeration before looping. `for sha in $(git ...)`
|
||||
# swallows a git failure: the command substitution yields nothing, the loop body
|
||||
# never runs, and the job reports "0 claims" green. Fail-open enumeration in the
|
||||
# thing that decides what gets checked is the defect this job exists to catch.
|
||||
if ! shas="$(git rev-list "$base".."$head")"; then
|
||||
echo "::error::git rev-list failed for $base..$head — cannot enumerate commits," \
|
||||
"so this job cannot assert anything. Refusing to pass."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
claimed=0; proven=0; failed=0
|
||||
while IFS= read -r sha; do
|
||||
[ -n "$sha" ] || continue
|
||||
# Trim whitespace only — NOT `xargs`, which applies quote parsing and turns a
|
||||
# legitimate parametrised node id like test_x[can't] into an empty selector,
|
||||
# silently dropping a real claim.
|
||||
# Extract with a CHECKED status. `sel="$(git show ... )"` under `set -uo
|
||||
# pipefail` but no `-e` yields an empty selector when git fails, the commit is
|
||||
# skipped, and the job exits 0 having been unable to inspect a possible claim —
|
||||
# fail-open in the step that decides what gets checked.
|
||||
if ! raw="$(git show -s --format='%(trailers:key=Proves,valueonly)' "$sha")"; then
|
||||
echo "::error::git show failed for $sha — cannot read its trailers, so this" \
|
||||
"job cannot assert anything about it. Refusing to pass."
|
||||
exit 1
|
||||
fi
|
||||
# Refuse MORE THAN ONE `Proves:` here too. prove-fix.sh has this guard, but it
|
||||
# only fires when it reads the trailer itself — and this job passes the selector
|
||||
# explicitly, so the guard was bypassed on the one path that actually enforces.
|
||||
# Measured: a commit with two trailers reported PROVEN while the second was never
|
||||
# run. Fixing the script and not its twin is how a guard reads as coverage.
|
||||
# Count trailer PRESENCE, not non-empty values: `%(...valueonly)` renders a bare
|
||||
# `Proves:` as an empty line, so counting non-empty lines misses a commit whose
|
||||
# FIRST trailer is empty — `sel` then comes out empty and the commit is skipped
|
||||
# in silence, with a real second selector never checked. Fail-open in CI while
|
||||
# the script is fail-closed is the same asymmetry this guard exists to remove.
|
||||
present="$(git show -s --format='%(trailers:key=Proves)' "$sha")"
|
||||
if [ "$(printf '%s\n' "$present" | grep -c .)" -gt 1 ]; then
|
||||
claimed=$((claimed + 1)); failed=$((failed + 1))
|
||||
echo "::error::commit $sha carries more than one 'Proves:' trailer; only the" \
|
||||
"first would be checked, so the rest would read as proven without ever" \
|
||||
"running. Use a single selector."
|
||||
continue
|
||||
fi
|
||||
sel="$(printf '%s\n' "$raw" | head -1 | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//')"
|
||||
# A trailer that is PRESENT but empty is a claim with no selector. Refuse it
|
||||
# loudly; skipping it silently would let the job report "no claims" for a PR that
|
||||
# made one.
|
||||
if [ -n "$present" ] && [ -z "$sel" ]; then
|
||||
claimed=$((claimed + 1)); failed=$((failed + 1))
|
||||
echo "::error::commit $sha carries a 'Proves:' trailer with no selector."
|
||||
continue
|
||||
fi
|
||||
[ -n "$sel" ] || continue
|
||||
claimed=$((claimed + 1))
|
||||
|
||||
# A merge commit has several parents, so "before this change" is ambiguous.
|
||||
# prove-fix.sh refuses them; catch it here with a clearer message rather than
|
||||
# letting the trailer be silently skipped (which --no-merges used to do).
|
||||
if [ "$(git rev-list --parents -n 1 "$sha" | wc -w)" -gt 2 ]; then
|
||||
failed=$((failed + 1))
|
||||
echo "::error::commit $sha is a MERGE carrying 'Proves: $sel'. Put the trailer" \
|
||||
"on the commit that carries the fix — a merge has no single 'before'."
|
||||
continue
|
||||
fi
|
||||
|
||||
echo "::group::prove $sha -> $sel"
|
||||
if bash ./scripts/prove-fix.sh "$sha" "$sel"; then
|
||||
proven=$((proven + 1)); echo "PROVEN $sha"
|
||||
else
|
||||
rc=$?
|
||||
failed=$((failed + 1))
|
||||
echo "::error::commit $sha claims 'Proves: $sel' but prove-fix.sh exited $rc." \
|
||||
"A claimed proof that does not hold is worse than none — it reads as" \
|
||||
"coverage. Strengthen the test until reverting the fix reddens it, or" \
|
||||
"drop the trailer."
|
||||
fi
|
||||
echo "::endgroup::"
|
||||
done <<< "$shas"
|
||||
|
||||
echo "commits claiming a proof: $claimed (proven $proven, failed $failed)"
|
||||
if [ "$claimed" -eq 0 ]; then
|
||||
echo "::notice::No commit in this PR carries a 'Proves:' trailer, so nothing was" \
|
||||
"verified here. That is allowed — the trailer is opt-in — but it means this" \
|
||||
"job asserts NOTHING about this PR. Do not read its green as fix coverage."
|
||||
fi
|
||||
[ "$failed" -eq 0 ]
|
||||
|
||||
script-tests:
|
||||
name: Script tests (pytest)
|
||||
name: Script lint and tests (ruff + pytest)
|
||||
runs-on: small
|
||||
if: github.event_name == 'pull_request'
|
||||
steps:
|
||||
@@ -222,6 +369,90 @@ jobs:
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: '3.x'
|
||||
# Preflight, not an install (ersatztv#390 removed run-time `apt-get` from CI on purpose).
|
||||
# Two consumers need `git`: the lint steps below derive their population from `git ls-files`,
|
||||
# and test_post_review_verdict.py / test_merge_consent_exemption.py exec the REAL
|
||||
# post-review-verdict.sh / pretooluse-merge-consent.sh. `curl` those tests shim on PATH; `jq`
|
||||
# and `git` they do NOT. It stays AHEAD of the lint steps, not merely ahead of pytest: without
|
||||
# it, a missing git reaches the lint steps as an empty population, which they report as a
|
||||
# population problem. One actionable line beats a misdirected one, and beats the wall of
|
||||
# unattributable assertion failures the suite produces without git.
|
||||
- name: Preflight external tools
|
||||
run: |
|
||||
if ! command -v git >/dev/null 2>&1; then
|
||||
echo "::error::script-tests needs git on PATH but it is absent. The lint steps derive" \
|
||||
"their population from it and the suite execs real shell scripts that use it." \
|
||||
"Bake it into the runner image rather than apt-get installing here (ersatztv#390)."
|
||||
exit 1
|
||||
fi
|
||||
echo "Preflight OK: $(git --version)"
|
||||
# ersatztv#780. Lint runs EARLY — after the git preflight it depends on, but before the test
|
||||
# dependencies, the jq preflight and the ~4-minute pytest run. A style red therefore arrives in
|
||||
# seconds, and, more importantly, the lint does not sit behind `Preflight jq version`: that is
|
||||
# an `--expect` tripwire, so a runner jq bump would take the lint dark for as long as the jq
|
||||
# contract is broken, under a red that says "jq".
|
||||
#
|
||||
# The version is PINNED: an unpinned ruff makes the verdict a function of whenever the job ran
|
||||
# — the same environment-divergence the committed ruff.toml exists to close. Bumping it is a
|
||||
# deliberate PR (new rules may fire), exactly like the jq pin below. `pytest`/`pyyaml` are
|
||||
# deliberately NOT pinned: a pytest release does not add assertions to your suite, a ruff
|
||||
# release adds rules to your lint.
|
||||
- name: Install ruff
|
||||
run: python3 -m pip install --disable-pip-version-check --quiet 'ruff==0.12.11'
|
||||
# POPULATION. Both steps lint an EXPLICIT list from `git ls-files`, never `ruff check .`, and
|
||||
# pass `--no-force-exclude`. Measured with ruff 0.12.11 and `exclude = ["scripts/**"]` — a
|
||||
# per-FILE pattern, because `exclude` matches per file: a bare `["scripts"]` still works at the
|
||||
# top level but matches nothing under `[lint]`/`[format]`. The subject is a planted tracked file
|
||||
# holding an unused import, a hardcoded credential and a formatting error. GREEN means the gate
|
||||
# was silently off:
|
||||
#
|
||||
# DISCOVERY FORM EXPLICIT FORM (what ships)
|
||||
# exclude scope check . format --check . check format --check
|
||||
# top-level GREEN GREEN red red
|
||||
# [lint] GREEN red red red
|
||||
# [format] red GREEN red red
|
||||
# top + force-exclude GREEN GREEN red red <- with the flag
|
||||
# GREEN GREEN <- without it
|
||||
#
|
||||
# Only the top-level scope empties BOTH discovery commands; `[lint]` empties `check` and
|
||||
# `[format]` empties `format --check`, so in those two the job would still redden on the other
|
||||
# step. `[format]` is where a line appended to ruff.toml lands, by TOML rules. `include = []`,
|
||||
# `extend-exclude` and a nested `scripts/ruff.toml` behave the same way and are equally inert
|
||||
# against the explicit form. The last row is the whole reason for `--no-force-exclude`:
|
||||
# `force-exclude = true` re-applies excludes to explicitly-passed paths, and is the one setting
|
||||
# that reaches explicitly-passed paths at all.
|
||||
#
|
||||
# `ruff check .` over an empty tree exits **0** with only a stderr warning, so every GREEN above
|
||||
# is a gate that was switched off without a red.
|
||||
#
|
||||
# This also derives the population from source rather than from the filesystem
|
||||
# (docs/decisions/records/testing/guard-derives-population-from-source.md) and covers
|
||||
# tracked-but-gitignored files, which `ruff check .` skips. The empty-population arm is the
|
||||
# anti-vacuity check: a completeness check whose population is empty reports that it proved
|
||||
# everything. What it does NOT cover: an emptied RULE set. `select = []` silences every selected
|
||||
# rule, so the `ruff check` step goes green over any lint violation (a syntax error still reds)
|
||||
# while printing a reassuring file count.
|
||||
# `ruff format --check` is unaffected, because formatting is not rule-selected. So half the
|
||||
# gate is killable by a config edit, and only a human reading that edit catches it.
|
||||
- name: Lint scripts (ruff check)
|
||||
run: |
|
||||
mapfile -d '' -t PYFILES < <(git ls-files -z '*.py' '*.pyi' '*.ipynb')
|
||||
if [ "${#PYFILES[@]}" -eq 0 ]; then
|
||||
echo "::error::the lint population is EMPTY — git tracks no Python files. Either the" \
|
||||
"checkout is wrong or the glob is. A lint over nothing passes; see ersatztv#780."
|
||||
exit 1
|
||||
fi
|
||||
echo "Linting ${#PYFILES[@]} tracked Python files"
|
||||
python3 -m ruff check --no-force-exclude -- "${PYFILES[@]}"
|
||||
- name: Lint scripts (ruff format --check)
|
||||
run: |
|
||||
mapfile -d '' -t PYFILES < <(git ls-files -z '*.py' '*.pyi' '*.ipynb')
|
||||
if [ "${#PYFILES[@]}" -eq 0 ]; then
|
||||
echo "::error::the format population is EMPTY — git tracks no Python files. See ersatztv#780."
|
||||
exit 1
|
||||
fi
|
||||
echo "Format-checking ${#PYFILES[@]} tracked Python files"
|
||||
python3 -m ruff format --check --no-force-exclude -- "${PYFILES[@]}"
|
||||
# pytest + PyYAML. PyYAML is NOT a contradiction of the dependency-free decisions READ path:
|
||||
# `decisions_lib._read_frontmatter` is hand-written precisely so validation runs where nothing
|
||||
# is installed, but the one-shot WRITE path `migrate_decisions_split.py` uses PyYAML by
|
||||
@@ -232,20 +463,6 @@ jobs:
|
||||
# went red in CI on a collection error.
|
||||
- name: Install test dependencies
|
||||
run: python3 -m pip install --disable-pip-version-check --quiet pytest pyyaml
|
||||
# Preflight, not an install (ersatztv#390 removed run-time `apt-get` from CI on purpose).
|
||||
# test_post_review_verdict.py and test_merge_consent_exemption.py exec the REAL
|
||||
# post-review-verdict.sh / pretooluse-merge-consent.sh, which shell out to `jq` ~26 times.
|
||||
# `curl` those tests shim on PATH; `jq` they do NOT. If it were missing, the suite would fail
|
||||
# as ~20 opaque assertion errors — this turns that into one actionable line.
|
||||
- name: Preflight external tools
|
||||
run: |
|
||||
if ! command -v git >/dev/null 2>&1; then
|
||||
echo "::error::script-tests needs git on PATH but it is absent. The suite execs real" \
|
||||
"shell scripts that use it. Bake it into the runner image rather than apt-get" \
|
||||
"installing here (see ersatztv#390)."
|
||||
exit 1
|
||||
fi
|
||||
echo "Preflight OK: $(git --version)"
|
||||
# jq gets its OWN step because its VERSION, not merely its presence, is load-bearing
|
||||
# (ersatztv#648). `--expect` makes this a TRIPWIRE: scripts/tests exercises the jq 1.6 code path
|
||||
# only because this runner ships 1.6, so an upgrade would silently delete that coverage — and
|
||||
|
||||
@@ -182,6 +182,47 @@ jobs:
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
# NEVER WRITE AN EXPRESSION DELIMITER ANYWHERE IN THIS BODY, INCLUDING IN A COMMENT
|
||||
# (ersatztv#751). A shell comment is not inert here. The runner scans this whole `run:`
|
||||
# scalar for the expression opener before bash ever sees it, and one occurrence makes it
|
||||
# rewrite the ENTIRE body into a single `format(...)` call so the result can be spliced
|
||||
# back in. That rewrite is all-or-nothing: if the payload does not parse as an expression
|
||||
# the interpolation of the whole scalar fails, and — the part that made this expensive —
|
||||
# the runner DROPS THE STEP AND REPORTS THE JOB GREEN.
|
||||
#
|
||||
# That is exactly how this gate spent 2026-08-03 to 2026-08-06 dead: the #706 note above,
|
||||
# explaining why a concurrency group does not work, quoted a `concurrency:` snippet
|
||||
# containing a PR-number expression as ILLUSTRATION. `pr number` is not a valid
|
||||
# expression, so this step never ran, `review-verdict/h10` was never posted by anything
|
||||
# but a human hand, and both exemption classes silently stopped working while every run
|
||||
# reported success. The prose documenting a fix disabled the fix.
|
||||
#
|
||||
# Two things now stop a recurrence, and they are deliberately different in kind: the
|
||||
# start-marker below turns a dropped step into a RED job instead of a green one, and
|
||||
# `test_the_verdict_workflow_has_NO_expression_delimiter_in_any_run_body` in
|
||||
# scripts/tests/test_pr_changed_files.py rejects the delimiter at review time. Note that
|
||||
# the older workflow-shape tests all scan comment-STRIPPED text (`_code_lines`) precisely
|
||||
# so prose cannot redden them — which makes every one of them structurally blind to this
|
||||
# class. The new test reads the raw scalar for that reason. To describe an expression in
|
||||
# prose here, name it (`a github.event.pull_request.number expression`); do not quote the
|
||||
# delimiters.
|
||||
#
|
||||
# THE START MARKER (ersatztv#751). Written before anything else can fail, and read by the
|
||||
# `Assert the classifier actually executed` step below. Its absence means this step did
|
||||
# not begin — the silent-green failure mode — and that is now a job failure. It records
|
||||
# only that execution STARTED: every `exit 0` abstention path below is a legitimate
|
||||
# outcome, so completion is deliberately not what is asserted.
|
||||
# KEYED ON THE RUN, not a fixed name. Measured on this instance, `RUNNER_TEMP` is `/tmp` —
|
||||
# not a private per-job directory — so a fixed name lives at a path other jobs can also
|
||||
# write. The `small` lane starts a container per job today, which makes the file fresh in
|
||||
# practice, but that is a property of the lane rather than a guarantee, and a STALE marker
|
||||
# would satisfy the guard below on a run whose step was dropped: a silent PASS, the exact
|
||||
# failure mode this guard exists to remove. Including the run id and attempt means a marker
|
||||
# from any other run cannot answer for this one. If the runner does not export them the path
|
||||
# degrades to a fixed name, which is no worse than having no key at all.
|
||||
RAN_MARKER="${RUNNER_TEMP:-${GITHUB_WORKSPACE:-/tmp}}/h10-classifier-started-${GITHUB_RUN_ID:-norunid}-${GITHUB_RUN_ATTEMPT:-1}"
|
||||
: > "$RAN_MARKER"
|
||||
|
||||
CONTEXT="review-verdict/h10"
|
||||
# The description this job writes when it repairs its own raced exemption (#706 race 2).
|
||||
# It is a SENTINEL, not just a message: `read_existing_verdict` recognises it, and the
|
||||
@@ -242,7 +283,19 @@ jobs:
|
||||
# containing executable hooks"). A derived rule has to be evaluated against the PR's own
|
||||
# file list, which is the very thing being classified — more moving parts inside a security
|
||||
# predicate, to remove a maintenance burden that is one line per new tooling directory.
|
||||
PROTECTED='^(\.claude/|\.codex/|\.gitea/|\.husky/|scripts/|docker/ci/)'
|
||||
# `CLAUDE.md` and `AGENTS.md` are listed even though they are `.md` files, because they are
|
||||
# not prose about the project — they are the documents that DEFINE the completion protocol,
|
||||
# the merge-consent convention and the H10 rule itself. `.claude/` being protected while the
|
||||
# file that specifies what `.claude/` enforces was docs-only-exempt is the same
|
||||
# self-exemption the header rules out, one directory over. Found by cold review
|
||||
# (ersatztv#751): driving the real classify body with a lone `CLAUDE.md` change produced
|
||||
# `review-verdict/h10=success`, "Exempt: docs-only change (no code, no protected path)".
|
||||
#
|
||||
# It is fixed HERE rather than deferred because this change is what makes it reachable
|
||||
# again: no exemption `success` was writable at all while the classify step was dropped, so
|
||||
# restoring the exemptions restores this hole with them. `README.md` is deliberately NOT
|
||||
# listed — it is ordinary prose and carries no enforcement.
|
||||
PROTECTED='^(\.claude/|\.codex/|\.gitea/|\.husky/|scripts/|docker/ci/|CLAUDE\.md$|AGENTS\.md$)'
|
||||
# Docs-only: prose and decision records. Deliberately narrower than the hook's pattern,
|
||||
# which also lets .claude/.gitea/.husky through — that carve-out is safe there only
|
||||
# because it falls through to a HUMAN PROMPT, whereas here it would post a green status
|
||||
@@ -328,11 +381,85 @@ jobs:
|
||||
read_existing_verdict() {
|
||||
local json row
|
||||
json=$(gh "$BASE_URL/repos/$REPO/commits/$SHA/status?limit=100") || json=""
|
||||
if [ -z "${json//[[:space:]]/}" ] || ! printf '%s' "$json" | jq -e '.statuses | type == "array"' >/dev/null 2>&1; then
|
||||
echo "::error::Could not read existing commit statuses for ${SHA:0:7}. Refusing to post anything rather than risk overwriting an existing verdict."
|
||||
# `.statuses` IS `null`, NOT `[]`, ON A HEAD WITH NO STATUSES YET — the same nil-slice
|
||||
# serialization as the timeline terminator, found by cold review of the fix for that one
|
||||
# (ersatztv#751). Measured on this instance: PR #739's head 5fa672e2 returns
|
||||
# `{"state":"pending","total_count":0,"statuses":null}`. An `array`-only gate read that as
|
||||
# unreadable and took the `exit 1` below, so the job posted NOTHING — fail-closed, but the
|
||||
# user-visible outcome is exactly the one this issue is about: an exempt PR left with no
|
||||
# status and, since ersatztv#743, no bypass. Fixing only the timeline site would have left
|
||||
# the identical bug one function away.
|
||||
#
|
||||
# `total_count` is required to agree, so `null` is accepted only as the empty case and not
|
||||
# as a body that merely lost its array. Anything else is still "cannot tell" and still
|
||||
# refuses to post, which is what protects an existing verdict from a transient API error.
|
||||
# WHETHER PAGE 1 IS THE WHOLE LIST CANNOT BE DECIDED FROM PAGE 1 (ersatztv#751). Two
|
||||
# plausible guards were tried and both are no-ops HERE, which is why this ended up as an
|
||||
# extra round-trip instead of an arithmetic test:
|
||||
#
|
||||
# * `.statuses | length` vs `.total_count` — `total_count` is the count for the PAGE
|
||||
# RETURNED, not for the commit. Measured at 1.27.1 on 3aed43c6 (6 contexts):
|
||||
# `?limit=1` gives `len=1, total_count=1`, `?limit=3` gives `len=3, total_count=3`.
|
||||
# Equal by construction, so the check proves nothing.
|
||||
# * "refuse when the page came back FULL at the requested limit of 100" — this instance
|
||||
# caps `limit` at the server-wide `MAX_RESPONSE_ITEMS`, measured at 50
|
||||
# (`/issues?limit=100` returns 50). A response can therefore never contain 100 rows and
|
||||
# the comparison was DEAD CODE. The repo already knew this and said so in
|
||||
# `scripts/pr-changed-files.sh`, two test files and `ci.script-tests-job`; the guard was
|
||||
# written anyway. Hardcoding the cap instead would re-break the day it is reconfigured.
|
||||
#
|
||||
# So ask the server, and only when it matters. Completeness is needed ONLY to justify the
|
||||
# conclusion "no verdict exists on this head" — if the row IS on page 1 there is nothing
|
||||
# further to learn, since this endpoint returns the latest status per CONTEXT and the same
|
||||
# context cannot recur on a later page. When the row is absent, page 2 is read: any rows
|
||||
# there mean the list is longer than one page and the verdict could be sitting beyond it,
|
||||
# so this refuses rather than concluding absence. Cap-independent by construction.
|
||||
st_kind=$(printf '%s' "$json" | jq -r '.statuses | type' 2>/dev/null) || st_kind=""
|
||||
# NUMBER, not `jq -r` text: `jq -r` renders the JSON number 0 and the JSON string "0"
|
||||
# identically, so a schema-corrupted `"total_count": "0"` would satisfy a string compare
|
||||
# (cold review reproduced this). Requiring the type as well pins the accept path to a real
|
||||
# numeric zero.
|
||||
st_total=$(printf '%s' "$json" | jq -r 'if (.total_count | type) == "number" then (.total_count | tostring) else "x" end' 2>/dev/null) || st_total="x"
|
||||
st_ok=no
|
||||
case "$st_kind" in
|
||||
array) st_ok=yes ;;
|
||||
null) if [ "$st_total" = "0" ]; then st_ok=yes; fi ;;
|
||||
esac
|
||||
if [ -z "${json//[[:space:]]/}" ] || [ "$st_ok" != yes ]; then
|
||||
echo "::error::Could not read existing commit statuses for ${SHA:0:7} (.statuses was '${st_kind:-unparseable}', total_count '${st_total}'). Refusing to post anything rather than risk overwriting an existing verdict."
|
||||
exit 1
|
||||
fi
|
||||
row=$(printf '%s' "$json" | jq -r --arg c "$CONTEXT" '[.statuses[] | select(.context == $c)] | first // {}')
|
||||
# `// []` so the null case cannot hard-error here under `set -e` once it is accepted above.
|
||||
row=$(printf '%s' "$json" | jq -r --arg c "$CONTEXT" '[(.statuses // [])[] | select(.context == $c)] | first // {}')
|
||||
# THE COMPLETENESS PROBE, run only when page 1 shows no verdict — see the note above. An
|
||||
# unreadable or unexpected page 2 is treated as "cannot tell" and refuses, the same
|
||||
# direction as every other unreadable case here: concluding "no verdict exists" is what
|
||||
# licenses posting an exemption over one, so it is the conclusion that must be earned.
|
||||
if [ "$(printf '%s' "$row" | jq -r '.context // ""')" = "" ]; then
|
||||
more=$(gh "$BASE_URL/repos/$REPO/commits/$SHA/status?limit=100&page=2") || more=""
|
||||
if [ -z "${more//[[:space:]]/}" ]; then
|
||||
echo "::error::Could not read page 2 of the commit statuses for ${SHA:0:7}, so 'no verdict exists' cannot be established. Refusing to post anything."
|
||||
exit 1
|
||||
fi
|
||||
more_kind=$(printf '%s' "$more" | jq -r '.statuses | type' 2>/dev/null) || more_kind=""
|
||||
more_len=$(printf '%s' "$more" | jq -r '(.statuses // []) | length' 2>/dev/null) || more_len=""
|
||||
case "$more_kind" in
|
||||
null) ;;
|
||||
array)
|
||||
case "$more_len" in
|
||||
''|*[!0-9]*)
|
||||
echo "::error::Page 2 of the commit statuses for ${SHA:0:7} had a non-numeric length; refusing to conclude that no verdict exists."
|
||||
exit 1 ;;
|
||||
0) ;;
|
||||
*)
|
||||
echo "::error::${CONTEXT} was not on page 1 of the statuses for ${SHA:0:7}, but page 2 carries ${more_len} more row(s) — the list is longer than one page and an existing verdict may be beyond it. Refusing to post anything rather than overwrite a verdict this job cannot see. A human verdict clears this: scripts/post-review-verdict.sh ${PR} MERGEABLE."
|
||||
exit 1 ;;
|
||||
esac ;;
|
||||
*)
|
||||
echo "::error::Page 2 of the commit statuses for ${SHA:0:7} was '${more_kind:-unparseable}'; refusing to conclude that no verdict exists."
|
||||
exit 1 ;;
|
||||
esac
|
||||
fi
|
||||
ex_state=$(printf '%s' "$row" | jq -r '.status // ""')
|
||||
ex_creator=$(printf '%s' "$row" | jq -r '.creator.login // ""')
|
||||
ex_desc=$(printf '%s' "$row" | jq -r '.description // ""')
|
||||
@@ -410,8 +537,8 @@ jobs:
|
||||
# run 7521 (`synchronize`) had started.
|
||||
#
|
||||
# WHY NOT A CONCURRENCY GROUP, which is the obvious answer and what #706 proposed. It does
|
||||
# not work here, also measured: with `concurrency: {group: …-${{ pr number }},
|
||||
# cancel-in-progress: false}` active on an identical probe, runs 7528 and 7529 still ran
|
||||
# not work here, also measured: with a `concurrency:` key grouping by PR number and
|
||||
# `cancel-in-progress: false` active on an identical probe, runs 7528 and 7529 still ran
|
||||
# CONCURRENTLY and 7528 ended 36s after 7529 began. Gitea 1.25.4 does auto-cancel superseded
|
||||
# `push` runs on a branch — a negative control with no `concurrency:` key at all showed that —
|
||||
# but that behaviour does NOT extend to `pull_request_target`. `cancel-in-progress: true` is
|
||||
@@ -438,19 +565,63 @@ jobs:
|
||||
# coming — a real stall. The retarget count moves only for the mutation that actually
|
||||
# invalidates a classification, and that mutation always brings its own re-run.
|
||||
#
|
||||
# Completeness is a guard, not an assumption (`ci.paged-endpoint-completeness`): the count is
|
||||
# Completeness is a guard, not an assumption (`ci.verdict-write-retarget-fence`; this
|
||||
# cited `ci.paged-endpoint-completeness` until 2026-08-06, a key that has never existed as
|
||||
# a record — resolve decisions through the catalog, never through a key or path quoted in a
|
||||
# comment). The count is
|
||||
# trusted ONLY when paging reached a validated EMPTY page. A short page, a non-array body, a
|
||||
# non-numeric length or the page cap all leave `rt_ok=no`, and an untrusted count is treated
|
||||
# below as "cannot tell" rather than as zero.
|
||||
count_retargets() {
|
||||
rt_count=0
|
||||
rt_ok=no
|
||||
local page=1 raw n m total=0
|
||||
local page=1 raw n m kind total=0
|
||||
while [ "$page" -le 20 ]; do
|
||||
raw=$(gh "$BASE_URL/repos/$REPO/issues/$PR/timeline?limit=50&page=${page}") || return 0
|
||||
if [ -z "${raw//[[:space:]]/}" ] || ! printf '%s' "$raw" | jq -e 'type == "array"' >/dev/null 2>&1; then
|
||||
return 0
|
||||
fi
|
||||
if [ -z "${raw//[[:space:]]/}" ]; then return 0; fi
|
||||
# A PAGE PAST THE END IS JSON `null` HERE, NOT `[]` (ersatztv#751). Measured on this
|
||||
# instance at Gitea 1.27.1 (page 1 non-empty, page 2 exhausted): page 2 of PR #752's timeline is the
|
||||
# four bytes `null`, while `/issues/{n}/comments` with no comments returns `[]`. The
|
||||
# instance is NOT consistent between endpoints, so the terminator has to be read from
|
||||
# what this endpoint actually does rather than from the shape a list endpoint "should"
|
||||
# return.
|
||||
#
|
||||
# This mattered far more than it looks. An `array`-only type gate treated `null` as
|
||||
# unreadable, so the walk NEVER reached a validated empty page, `rt_ok` was never `yes`
|
||||
# for ANY pull request, and the fence below therefore withheld every exemption
|
||||
# `success` — permanently. Renovate and docs-only PRs got no status at all rather than
|
||||
# an auto-pass, which is the same user-visible outcome as ersatztv#751 arriving by a
|
||||
# completely different route. It was invisible because it shipped in the SAME commit
|
||||
# (8f6d4f443) that stopped this step from executing at all, so the fence had never once
|
||||
# run in production, and because the test double asserted the wrong shape — it printed
|
||||
# `[]` for a page past the end while claiming to mirror measured reality.
|
||||
#
|
||||
# Read the type as a VALUE rather than through `jq -e`. `jq -e` reports the truthiness
|
||||
# of the last output, so distinguishing "the body is null" from "the predicate is
|
||||
# false" through it means relying on the same exit-status subtlety that already bit this
|
||||
# workflow once at jq 1.6 (ersatztv#647). A `case` over `jq -r 'type'` cannot be read
|
||||
# two ways. Anything that is neither `null` nor `array` is still "cannot tell".
|
||||
kind=$(printf '%s' "$raw" | jq -r 'type' 2>/dev/null) || kind=""
|
||||
# `null` counts as exhaustion only from page 2 ON. THE INVARIANT, not a figure that
|
||||
# rots: a real PR's timeline always carries at least one event on page 1 (it is created
|
||||
# by a push, which is itself an event), so a `null` FIRST page is anomalous rather than
|
||||
# empty. Spot-checked across #752/#753/#749/#739/#717, all non-empty; the counts
|
||||
# themselves are deliberately not recorded here because timelines grow and an earlier
|
||||
# version of this comment cited five numbers of which three were stale within days.
|
||||
# Trusting a zero count from an anomalous first page would
|
||||
# mean trusting that no retarget happened on the strength of a response we cannot
|
||||
# explain. Requiring one real page keeps the property the walk is for: something was
|
||||
# actually read. A PR that somehow has an empty first page falls through to `rt_ok=no`,
|
||||
# which withholds the exemption and asks for a human verdict — the safe direction.
|
||||
# This narrows rather than closes the general concern: a wrong `null` on page 3 is still
|
||||
# read as exhaustion, and no bounded number of round-trips can rule that out.
|
||||
case "$kind" in
|
||||
null)
|
||||
if [ "$page" -gt 1 ]; then rt_ok=yes; rt_count=$total; fi
|
||||
return 0 ;;
|
||||
array) ;;
|
||||
*) return 0 ;;
|
||||
esac
|
||||
n=$(printf '%s' "$raw" | jq -r 'length')
|
||||
case "$n" in ''|*[!0-9]*) return 0 ;; esac
|
||||
if [ "$n" -eq 0 ]; then rt_ok=yes; rt_count=$total; return 0; fi
|
||||
@@ -471,10 +642,12 @@ jobs:
|
||||
# --- Whose verdict is it? (ersatztv#698 route 3) -------------------------------------
|
||||
# This short-circuit used to exit on ANY existing `success`, which made an exemption this job
|
||||
# wrote indistinguishable from a verdict a human wrote. That is what let a forged exemption
|
||||
# survive: obtained once — via the route-1 retarget race, a sibling workflow holding
|
||||
# status-write credentials (#697), a direct API call, or inheritance across PRs by sha (#663)
|
||||
# — it was thereafter accepted unchanged on every run, because the guard exited before it
|
||||
# looked at the PR, the base, the author or the files.
|
||||
# survive: obtained once — via the route-1 retarget race, a collaborator's own token, the
|
||||
# `GITEA_TOKEN` injected into every job, `RENOVATE_TOKEN`, a direct API call, or inheritance
|
||||
# across PRs by sha (#663) — it was thereafter accepted unchanged on every run, because the
|
||||
# guard exited before it looked at the PR, the base, the author or the files. (`#697`'s
|
||||
# registry credential can no longer post a status at all — see `ci.actions-credential-scoping`
|
||||
# — but that closes only one route; the others above still work.)
|
||||
#
|
||||
# The guard still exists for its original reason: re-posting `pending` over a real human
|
||||
# verdict would un-approve a reviewed head and stall the PR. So it discriminates by PROVENANCE.
|
||||
@@ -491,9 +664,11 @@ jobs:
|
||||
# machine-written") an unrecognised shape would be trusted — the fail-open this issue is about.
|
||||
#
|
||||
# What this does NOT claim: the test asks "was this POSTed by a user credential", NOT "by a
|
||||
# reviewer". `ETV_STATUS_AUTH` is basic auth, so head-controlled code can POST a success with a
|
||||
# non-null creator AND an attacker-chosen `Review-verdict:` description, which this guard then
|
||||
# preserves. That is #697 — provenance, not authentication.
|
||||
# reviewer". `ETV_STATUS_AUTH`'s registry credential can no longer POST a status at all (#697,
|
||||
# fixed by scoping it off `write:repository` — see `ci.actions-credential-scoping`), but any
|
||||
# OTHER user credential — a collaborator's own token, `RENOVATE_TOKEN` — can still POST a
|
||||
# success with a non-null creator AND an attacker-chosen `Review-verdict:` description, which
|
||||
# this guard then preserves. That is provenance, not authentication.
|
||||
read_existing_verdict
|
||||
if [ "$ex_human" = yes ] && { [ "$ex_state" = "success" ] || [ "$ex_state" = "failure" ]; }; then
|
||||
echo "${CONTEXT} is already '${ex_state}' on ${SHA:0:7}, written by '${ex_creator}' as a human verdict — leaving it alone."
|
||||
@@ -771,9 +946,21 @@ jobs:
|
||||
# live head), `/statuses/{sha}` a BARE ARRAY (24 rows on the same head) — hence the different
|
||||
# `type == "array"` guard here.
|
||||
#
|
||||
# No claim is made about the order rows come back in, because the check does not depend on
|
||||
# it: it selects by id against the high-water mark rather than inspecting the top of the
|
||||
# list. A verdict older than the mark is invisible to it no matter where it sits.
|
||||
# ORDER IS NOT RELIED ON *WITHIN* A PAGE — the check selects by id against the high-water
|
||||
# mark rather than inspecting the top of the list, so a verdict older than the mark is
|
||||
# invisible to it no matter where it sits. But this read is ONE PAGE, and `limit=100` clamps
|
||||
# to the server-wide `MAX_RESPONSE_ITEMS`, measured at 50 on this instance (ersatztv#751).
|
||||
# An earlier version of this comment claimed order-independence flatly; that is false the
|
||||
# moment a head carries more rows than the clamp, because a raced row can then sit on a page
|
||||
# this never reads. Measured: a probe head reached 33 rows after ~5 runs, and the ordering is
|
||||
# only coarsely newest-first (ids came back `33,32,31,30,28,29,27,…`), so >50 is ordinarily
|
||||
# reachable on a PR with a few CI reruns.
|
||||
#
|
||||
# That matters more here than anywhere else in this job, because this is the ONE path whose
|
||||
# failure direction is toward SUCCESS: missing a raced human `failure` leaves a forged green
|
||||
# standing over a rejection. So the page-2 probe below treats "there are rows I did not read"
|
||||
# as "assume raced" — the conservative direction — rather than as "no race found".
|
||||
# Full paging of this endpoint, including for the high-water mark above, is ersatztv#763.
|
||||
#
|
||||
# The repair is `pending`, NEVER a copy of the human's state. Re-posting their `failure`
|
||||
# would attribute a human verdict to this job — the exact provenance confusion the
|
||||
@@ -818,6 +1005,39 @@ jobs:
|
||||
and (((.description // "") | startswith("Review-verdict:"))))
|
||||
or ((.creator == null) and ((.description // "") == $rd))
|
||||
)] | length')
|
||||
# "NO RACE FOUND ON PAGE 1" IS NOT "NO RACE" (ersatztv#751). If page 1 was not the whole
|
||||
# list, a raced row can be beyond it, so an unread page is treated as a race rather than
|
||||
# as absence. Only checked when page 1 looked clean — a race already found needs no
|
||||
# further evidence. Unreadable page 2 also counts as raced: this is the fail-toward-
|
||||
# SUCCESS path, so uncertainty must resolve to `pending`, never to leaving green.
|
||||
#
|
||||
# BE HONEST ABOUT THE COST: a repair here is STICKY, not a one-run stall. It writes
|
||||
# `$REPAIR_DESC`, and the classification above refuses to grant an exemption over that
|
||||
# sentinel and re-writes it as a fixed point on every later run — so a SPURIOUS repair
|
||||
# (a head with more status rows than the page cap and no actual race) removes that head's
|
||||
# exemption permanently, and only a human `post-review-verdict.sh` clears it. That is
|
||||
# still the right direction, because the alternative is a forged green over a human
|
||||
# rejection. But it is a per-sha loss of the exemption, not an inconvenience, which is
|
||||
# the argument for replacing this with real paging (#763) rather than living with it.
|
||||
if [ "$raced" = "0" ]; then
|
||||
more_hist=$(gh "$BASE_URL/repos/$REPO/statuses/$SHA?limit=100&page=2") || more_hist="UNREADABLE"
|
||||
if [ "$more_hist" = "UNREADABLE" ]; then
|
||||
echo "::warning::Could not read page 2 of the status history for ${SHA:0:7}; treating this as a possible raced verdict and repairing to pending rather than leaving an exemption green on an unverified head."
|
||||
raced=1
|
||||
else
|
||||
mh_kind=$(printf '%s' "$more_hist" | jq -r 'type' 2>/dev/null) || mh_kind=""
|
||||
mh_len=$(printf '%s' "$more_hist" | jq -r 'if type == "array" then length else 0 end' 2>/dev/null) || mh_len=""
|
||||
case "${mh_kind}:${mh_len}" in
|
||||
null:*|array:0) ;;
|
||||
array:*)
|
||||
echo "::warning::The status history for ${SHA:0:7} runs past page 1 (${mh_len} more row(s)), so a raced human verdict could be on a page this job did not read. Repairing to pending rather than leaving the exemption green."
|
||||
raced=1 ;;
|
||||
*)
|
||||
echo "::warning::Page 2 of the status history for ${SHA:0:7} was '${mh_kind:-unparseable}'; treating as a possible raced verdict and repairing to pending."
|
||||
raced=1 ;;
|
||||
esac
|
||||
fi
|
||||
fi
|
||||
case "$raced" in
|
||||
''|*[!0-9]*)
|
||||
echo "::warning::Post-write verification for ${SHA:0:7} returned '${raced}' instead of a count; not acting on it."
|
||||
@@ -851,3 +1071,48 @@ jobs:
|
||||
if [ "$state" = "pending" ]; then
|
||||
echo "::notice::This PR needs an H10 review verdict for head ${SHA:0:7} before it can merge. After reviewing, run: scripts/post-review-verdict.sh ${PR} MERGEABLE"
|
||||
fi
|
||||
|
||||
# THE SILENT-GREEN HALF OF ersatztv#751, WHICH IS THE ACTUAL DEFECT. The stray expression
|
||||
# delimiter was one bug in one comment; this step exists because of how the runner REPORTED it.
|
||||
# A step it declines to interpolate is dropped and the job concludes `success`, so the gate was
|
||||
# dead for three days behind a green tick. Nothing surfaced it: the workflow's own status
|
||||
# context said success, and `review-verdict/h10` was merely ABSENT — which reads as "not
|
||||
# reviewed yet" on a normal PR and is indistinguishable from the correct pending state.
|
||||
#
|
||||
# This turns that class of failure — dropped for ANY reason, not only an interpolation failure
|
||||
# — into a red job. It asserts execution STARTED, never that it completed: the classifier has
|
||||
# several legitimate `exit 0` abstention paths (a human verdict landed mid-run, a mid-run
|
||||
# retarget, a sentinel appearing) and treating those as failures would redden ordinary PRs.
|
||||
#
|
||||
# `if: always()` so it also runs when the classifier failed on purpose — its fail-closed paths
|
||||
# exit non-zero, the job is already red, and this step then finds the marker and stays quiet
|
||||
# rather than piling a misleading second error on top.
|
||||
#
|
||||
# ITS OWN BODY MUST STAY EXPRESSION-FREE. A guard that can be dropped by the very mechanism it
|
||||
# guards against is worse than none, because its absence is also silent. No delimiters here, no
|
||||
# illustrative snippets, and `always()` is written bare as `if:` requires — the static test in
|
||||
# scripts/tests/test_pr_changed_files.py holds this to it.
|
||||
#
|
||||
# THAT A LATER STEP STILL RUNS AFTER AN EARLIER ONE IS DROPPED IS MEASURED, not assumed — it is
|
||||
# the premise this guard stands on, and the #751 report could not settle it because the
|
||||
# classifier was the job's last step, leaving nothing subsequent to observe. Established on this
|
||||
# instance by a scratch-base probe with a negative control (Gitea 1.27.1, 2026-08-06): run 1863
|
||||
# dropped the classifier on a reintroduced bad payload, logged `evaluating expression 'always()'
|
||||
# -> true`, ran THIS step, and the job concluded `failure`. Run 1866 is the positive control —
|
||||
# the classifier ran, posted its exemption, and this step found the marker at
|
||||
# `/tmp/h10-classifier-started-1866-1`, confirming the run-keyed path resolves.
|
||||
#
|
||||
# Had the runner dropped the remaining steps too, this guard could not work and the body would
|
||||
# have had to move into `scripts/`, where a one-line `run:` makes the class unreachable. It does
|
||||
# not, so it stays here. Re-measure if the runner is upgraded: this is the one assumption whose
|
||||
# failure is silent again.
|
||||
- name: Assert the classifier actually executed
|
||||
if: always()
|
||||
run: |
|
||||
set -euo pipefail
|
||||
marker="${RUNNER_TEMP:-${GITHUB_WORKSPACE:-/tmp}}/h10-classifier-started-${GITHUB_RUN_ID:-norunid}-${GITHUB_RUN_ATTEMPT:-1}"
|
||||
if [ ! -f "$marker" ]; then
|
||||
echo "::error::The classify step did not execute AT ALL — no start marker at ${marker}. The runner dropped it (an interpolation failure over the run: body does this and still reports the job green; see ersatztv#751) or it was skipped. review-verdict/h10 has NOT been posted for this head, so exempt PRs (Renovate, docs-only) are silently unmergeable. Failing the job so this is visible instead of green."
|
||||
exit 1
|
||||
fi
|
||||
echo "The classify step executed (start marker present at ${marker})."
|
||||
|
||||
@@ -10,6 +10,10 @@ project.lock.json
|
||||
# Claude Code
|
||||
.mcp/
|
||||
.mcp.json
|
||||
# Machine-local settings (DOTNET_ROOT and friends — see docs/local-lsp-tooling.md).
|
||||
# Ignored here rather than relying on a personal ~/.config/git/ignore, so a second
|
||||
# contributor following that doc cannot accidentally commit their own Homebrew paths.
|
||||
/.claude/settings.local.json
|
||||
.agents/
|
||||
plugins/
|
||||
nupkg/
|
||||
@@ -70,6 +74,11 @@ ErsatzTV/wwwroot/app/
|
||||
web/dist/
|
||||
web/node_modules
|
||||
|
||||
# Root-level link that makes `typescript` resolvable from the repo root, which is
|
||||
# the LSP workspace root — without it typescript-language-server refuses to start
|
||||
# (ersatztv#777). See docs/local-lsp-tooling.md.
|
||||
/node_modules/
|
||||
|
||||
# E2E / screenshot scratch (from Playwright/live-E2E runs) — never committed
|
||||
/*.png
|
||||
.playwright-mcp/
|
||||
|
||||
+3
-2
@@ -12,8 +12,9 @@ unset GIT_DIR GIT_WORK_TREE GIT_INDEX_FILE
|
||||
|
||||
# H11 (ersatztv#311): refuse to push a branch that is BEHIND origin/main — rebase, don't merge
|
||||
# main in (a merge drags in files you never touched, e.g. legacy-BOM .cs, and trips the format
|
||||
# hook on code that isn't yours). Fail-open; escape with ETV_SKIP_REBASE_CHECK=1.
|
||||
./.claude/hooks/prepush-rebase-check.sh || exit 1
|
||||
# hook on code that isn't yours). Fail-open; escape with ETV_SKIP_REBASE_CHECK=1. Exempts a
|
||||
# tag-only push (ersatztv#719) — forward the ref lines captured above so it can tell.
|
||||
printf '%s\n' "$_prepush_refs" | ./.claude/hooks/prepush-rebase-check.sh || exit 1
|
||||
|
||||
# H13 (ersatztv#416 session): refuse to push when a file in the pushed diff still has uncommitted
|
||||
# working-tree/index changes — the pushed commit wouldn't match what you built/reviewed (the #416
|
||||
|
||||
@@ -85,9 +85,11 @@ Every task that closes a Gitea issue MUST complete ALL of these before it is con
|
||||
**Merge-consent is derived from state, not asserted (`## Done-when` convention — ersatztv#303 H6 + H10).** Any issue whose PR will merge to `main` should carry a `## Done-when` section in its **issue body** — a checklist of completion criteria (always include an "adversarial review passed" box; add per-issue criteria like tests-green, docs-updated, live-E2E). Two hooks derive merge-consent from it so a premature merge is blocked *by construction*, not by memory:
|
||||
- `pretooluse-merge-consent.sh` (Claude PreToolUse on the Gitea merge tool) — **auto-grants** a merge (emits `permissionDecision: allow`, so **no** redundant mechanical prompt fires) only when the PR's CI is green **and** every `## Done-when` box on the linked issue (`fixes #N`) is ticked **and** a `Review-verdict:` comment references the PR's *current head sha* (**H10**); **denies** on an unticked box, red CI, or a stale/negative review verdict; **asks** (falls back to a human prompt) when it can't derive state (no linked issue, no `## Done-when` section, no `Review-verdict:` comment yet, no creds, Gitea down). On the auto-grant (satisfied) path the derived state **is** the consent — do not also ask conversationally to merge; a separate human confirmation is warranted only when the gate **asks** (ersatztv#314). **The H10 review-verdict convention**: after an adversarial/Codex review of a PR (or its latest fix commit), run **`scripts/post-review-verdict.sh <pr> <MERGEABLE|APPROVED|BLOCKED|NOT-MERGEABLE> [note]`** — it posts both the `Review-verdict: … @ <head-sha>` comment and the sha-bound `review-verdict/h10` commit status, proving the *latest* commit was reviewed rather than a stale earlier diff (ersatztv#242). Do not hand-write the comment: the **status** is the required check branch protection enforces, and a comment alone leaves it absent.
|
||||
- **The gate is enforced server-side, per sha (ersatztv#622).** `review-verdict/h10` is a required status check on `main`. Because a commit status belongs to one sha, a commit pushed *after* an auto-merge is scheduled clears it and blocks the merge — closing the hole where `merge_when_checks_succeed` froze consent at scheduling time and Gitea later merged an unreviewed head. Renovate-authored and docs-only PRs are auto-passed by `.gitea/workflows/review-verdict.yml`, **except** when they touch `.claude/`, `.codex/`, `.gitea/`, `.husky/`, `scripts/` or `docker/ci/`. See `docs/ci-cd.md` → Review-verdict gate.
|
||||
- `.husky/pre-push` → `prepush-donewhen.sh` — a fail-open backstop that blocks a direct `git push origin main` whose commits `fix #N` an issue with unticked boxes.
|
||||
- `.husky/pre-push` → `prepush-donewhen.sh` — a fail-open backstop that blocks a direct `git push origin main` whose commits `fix #N` an issue with unticked boxes. **Since ersatztv#743 that push can no longer happen at all** (see below), so this hook is now belt-and-braces for a path the server refuses.
|
||||
|
||||
Both need Gitea read creds in the env to enforce (**`ETV_GITEA_BASICAUTH=user:pass`** or `ETV_GITEA_TOKEN`; `ETV_GITEA_URL` overrides the base). Without them the merge hook asks and the push backstop is a no-op — the gate degrades to today's manual confirmation, never a silent pass. Docs-only PRs/pushes are exempt.
|
||||
**`main` is PR-only — there is no direct-push path any more (ersatztv#743, `release.main-direct-push-disabled`).** Branch protection carries `enable_push: false` **and** `block_admin_merge_override: true`: a direct `git push origin HEAD:main` is refused server-side at pre-receive for every account including a site admin, the contents API is refused too, and an admin cannot `force_merge` past a missing or red required context. This is what makes `review-verdict/h10` load-bearing rather than conventional — Gitea only evaluates `status_check_contexts` on the PR merge path, so before this the whole gate was skippable with no forgery. Practically: **every** change to `main` goes through a PR, including a one-line docs fix. Tag pushes are unaffected (separate mechanism), so the release cut is unchanged.
|
||||
|
||||
Both need Gitea read creds in the env to enforce (**`ETV_GITEA_BASICAUTH=user:pass`** or `ETV_GITEA_TOKEN`; `ETV_GITEA_URL` overrides the base). Without them the merge hook asks and the push backstop is a no-op — the gate degrades to today's manual confirmation, never a silent pass. Docs-only PRs are exempt from the *review-verdict* gate; the direct-push exemption is moot now that direct pushes are refused outright.
|
||||
|
||||
**The 7 mandatory completion steps and the `## Closing record` comment template** live in the
|
||||
`closing-an-issue` skill (`.claude/skills/closing-an-issue/SKILL.md`) — invoke it (or `/done`)
|
||||
|
||||
@@ -75,7 +75,7 @@
|
||||
<PackageVersion Include="RichTextKit.Stbear" Version="0.4.167.3" />
|
||||
<PackageVersion Include="Roslynator.Analyzers" Version="4.15.0" />
|
||||
<PackageVersion Include="Scalar.AspNetCore" Version="2.12.32" />
|
||||
<PackageVersion Include="Scriban.Signed" Version="7.2.5" />
|
||||
<PackageVersion Include="Scriban.Signed" Version="7.2.6" />
|
||||
<PackageVersion Include="Serilog" Version="4.3.0" />
|
||||
<PackageVersion Include="Serilog.AspNetCore" Version="10.0.0" />
|
||||
<PackageVersion Include="Serilog.Extensions.Hosting" Version="10.0.0" />
|
||||
|
||||
@@ -21,4 +21,16 @@
|
||||
<ProjectReference Include="..\ErsatzTV.Mcp\ErsatzTV.Mcp.csproj" />
|
||||
</ItemGroup>
|
||||
|
||||
<!--
|
||||
The generated OpenAPI document is the wire contract the MCP catalog wraps. Copying it into the
|
||||
test output lets ToolCatalogTests assert that every write tool declares exactly the request-body
|
||||
fields its endpoint accepts, so a new DTO property cannot drift out of a tool schema unnoticed
|
||||
(issue #754). Regenerated by scripts/update-openapi.sh.
|
||||
-->
|
||||
<ItemGroup>
|
||||
<Content Include="..\ErsatzTV\wwwroot\openapi\v1.json"
|
||||
Link="openapi\v1.json"
|
||||
CopyToOutputDirectory="PreserveNewest" />
|
||||
</ItemGroup>
|
||||
|
||||
</Project>
|
||||
|
||||
@@ -77,9 +77,22 @@ public class ToolCatalogTests
|
||||
// Enums must NOT be forced required (they have server-side defaults).
|
||||
createRequired.ShouldNotContain("streamingMode");
|
||||
|
||||
// Update carries the same body fields plus the route id.
|
||||
update.InputSchema.RootElement.GetProperty("properties").TryGetProperty("id", out _).ShouldBeTrue();
|
||||
update.InputSchema.RootElement.GetProperty("properties").TryGetProperty("showInEpg", out _).ShouldBeTrue();
|
||||
// Update carries the create body fields plus the route id...
|
||||
JsonElement updateProps = update.InputSchema.RootElement.GetProperty("properties");
|
||||
updateProps.TryGetProperty("id", out _).ShouldBeTrue();
|
||||
updateProps.TryGetProperty("showInEpg", out _).ShouldBeTrue();
|
||||
|
||||
// ...plus graphicsElementIds, which is on UpdateChannelRequest only. PUT is a full replace, so
|
||||
// while the tool could not express this field an agent following the tool's own "send the full
|
||||
// desired state" instruction silently detached every graphics element (issue #754).
|
||||
updateProps.TryGetProperty("graphicsElementIds", out JsonElement graphicsElementIds).ShouldBeTrue();
|
||||
graphicsElementIds.GetProperty("type").GetString().ShouldBe("array");
|
||||
graphicsElementIds.GetProperty("items").GetProperty("type").GetString().ShouldBe("integer");
|
||||
|
||||
// Create must NOT send it: CreateChannelRequest has no such property, and the tool schema is
|
||||
// additionalProperties:false. This is why it is declared on the update tool rather than in the
|
||||
// shared ChannelFields().
|
||||
createProps.TryGetProperty("graphicsElementIds", out _).ShouldBeFalse();
|
||||
}
|
||||
|
||||
[Test]
|
||||
@@ -256,4 +269,248 @@ public class ToolCatalogTests
|
||||
tool.QueryParameters.ShouldNotBeNull();
|
||||
tool.QueryParameters!.ShouldContain("deep");
|
||||
}
|
||||
|
||||
// #754: ToolCatalog declared 27 of UpdateChannelRequest's 28 properties. The missing one was
|
||||
// graphicsElementIds, and because PUT /api/v1/channels/{id} is a FULL REPLACE the omission was not
|
||||
// merely "one field you cannot set" — an agent that GET-edit-PUT the channel, exactly as the tool's
|
||||
// description tells it to, detached every graphics element (including the On Now/Next overlay) with
|
||||
// a 200 and no error. The same shape was live on ersatztv_update_schedule, which omitted
|
||||
// padToNearestMinute and silently cleared a configured pad.
|
||||
//
|
||||
// Neither is fixable by counting fields once: the defect is that nothing tied the tool schema to the
|
||||
// contract it wraps. So this test asserts the tie for EVERY write tool against the generated OpenAPI
|
||||
// document (the actual wire contract, linked into the test output by the csproj). A new property on
|
||||
// any request DTO now fails here until the catalog declares it.
|
||||
[Test]
|
||||
public void Every_Write_Tool_Should_Declare_Exactly_Its_OpenApi_Request_Body_Fields()
|
||||
{
|
||||
using JsonDocument spec = LoadOpenApiDocument();
|
||||
JsonElement paths = spec.RootElement.GetProperty("paths");
|
||||
|
||||
ToolDefinition[] writeTools = ToolCatalog.All
|
||||
.Where(t => t.HttpMethod == HttpMethod.Post
|
||||
|| t.HttpMethod == HttpMethod.Put
|
||||
|| t.HttpMethod == HttpMethod.Patch)
|
||||
.ToArray();
|
||||
|
||||
// Pin the covered set rather than trusting the filter. A tool that stopped being a write verb,
|
||||
// or a new write tool, must show up as a change here — a bare loop over a filtered set passes
|
||||
// just as happily when the set silently shrinks to nothing.
|
||||
string[] expectedWriteTools =
|
||||
[
|
||||
"ersatztv_add_collection_items",
|
||||
"ersatztv_create_channel",
|
||||
"ersatztv_create_collection",
|
||||
"ersatztv_create_playout",
|
||||
"ersatztv_create_schedule",
|
||||
"ersatztv_create_smart_collection",
|
||||
"ersatztv_enable_jellyfin_library_sync",
|
||||
"ersatztv_refresh_jellyfin_libraries",
|
||||
"ersatztv_reset_channel_playout",
|
||||
"ersatztv_scan_jellyfin_collections",
|
||||
"ersatztv_scan_library",
|
||||
"ersatztv_update_channel",
|
||||
"ersatztv_update_collection",
|
||||
"ersatztv_update_collection_custom_order",
|
||||
"ersatztv_update_playout",
|
||||
"ersatztv_update_schedule",
|
||||
"ersatztv_update_smart_collection"
|
||||
];
|
||||
|
||||
writeTools.Select(t => t.Name).OrderBy(n => n, StringComparer.Ordinal)
|
||||
.ShouldBe(expectedWriteTools.OrderBy(n => n, StringComparer.Ordinal));
|
||||
|
||||
foreach (ToolDefinition tool in writeTools)
|
||||
{
|
||||
paths.TryGetProperty(tool.PathTemplate, out JsonElement pathItem)
|
||||
.ShouldBeTrue($"{tool.Name}: {tool.PathTemplate} is not in the OpenAPI document");
|
||||
|
||||
string verb = tool.HttpMethod.Method.ToLowerInvariant();
|
||||
pathItem.TryGetProperty(verb, out JsonElement operation)
|
||||
.ShouldBeTrue($"{tool.Name}: {verb.ToUpperInvariant()} {tool.PathTemplate} is not in the OpenAPI document");
|
||||
|
||||
Dictionary<string, string> declared = DeclaredBodyArguments(tool);
|
||||
Dictionary<string, string> accepted = RequestBodyProperties(spec, operation, tool.Name);
|
||||
|
||||
// Compare name AND type. Names alone would let a field drift to the wrong JSON type: the
|
||||
// tool would advertise "string" for an int?, the agent would send "30", and the API would
|
||||
// 400 — green test, broken tool.
|
||||
declared.Select(p => $"{p.Key}: {p.Value}").OrderBy(s => s, StringComparer.Ordinal)
|
||||
.ShouldBe(
|
||||
accepted.Select(p => $"{p.Key}: {p.Value}").OrderBy(s => s, StringComparer.Ordinal),
|
||||
customMessage:
|
||||
$"{tool.Name} declares body fields that do not match {verb.ToUpperInvariant()} {tool.PathTemplate}. "
|
||||
+ "A field the endpoint accepts but the tool omits is silently dropped on a full-replace "
|
||||
+ "write (#754); a field the tool sends but the endpoint does not accept is rejected; "
|
||||
+ "a field declared with the wrong type is rejected at the API.");
|
||||
}
|
||||
}
|
||||
|
||||
// #757, the sibling of the body guard above. Query parameters drift the same way and are WORSE for
|
||||
// reads: ToolArgumentValidator rejects undeclared arguments, so a parameter the tool omits is not
|
||||
// merely undocumented, it is unreachable — the caller cannot pass it at all. That is how #616's
|
||||
// paging omission hard-capped two tools at the first page. This covers EVERY tool, not just the
|
||||
// write verbs, because the drift that existed when this was written was entirely on reads.
|
||||
[Test]
|
||||
public void Every_Tool_Should_Declare_Exactly_Its_OpenApi_Query_Parameters()
|
||||
{
|
||||
using JsonDocument spec = LoadOpenApiDocument();
|
||||
JsonElement paths = spec.RootElement.GetProperty("paths");
|
||||
|
||||
// Every tool is covered, so an emptiness guard is enough here — there is no filter to escape.
|
||||
ToolCatalog.All.Count.ShouldBeGreaterThan(30);
|
||||
|
||||
// Accumulate rather than throwing on the first mismatch, so one run reports the WHOLE drift set.
|
||||
// Failing fast here would hand back one tool at a time and invite fixing them one at a time,
|
||||
// which is how the #754 twin stayed hidden in the first place.
|
||||
List<string> drift = [];
|
||||
|
||||
foreach (ToolDefinition tool in ToolCatalog.All)
|
||||
{
|
||||
paths.TryGetProperty(tool.PathTemplate, out JsonElement pathItem)
|
||||
.ShouldBeTrue($"{tool.Name}: {tool.PathTemplate} is not in the OpenAPI document");
|
||||
|
||||
string verb = tool.HttpMethod.Method.ToLowerInvariant();
|
||||
pathItem.TryGetProperty(verb, out JsonElement operation)
|
||||
.ShouldBeTrue($"{tool.Name}: {verb.ToUpperInvariant()} {tool.PathTemplate} is not in the OpenAPI document");
|
||||
|
||||
IReadOnlySet<string> declared = tool.QueryParameters ?? new HashSet<string>(StringComparer.Ordinal);
|
||||
HashSet<string> accepted = QueryParameterNames(operation);
|
||||
|
||||
string[] missing = accepted.Except(declared, StringComparer.Ordinal).OrderBy(n => n, StringComparer.Ordinal).ToArray();
|
||||
string[] phantom = declared.Except(accepted, StringComparer.Ordinal).OrderBy(n => n, StringComparer.Ordinal).ToArray();
|
||||
|
||||
if (missing.Length > 0 || phantom.Length > 0)
|
||||
{
|
||||
drift.Add(
|
||||
$"{tool.Name} ({verb.ToUpperInvariant()} {tool.PathTemplate}): "
|
||||
+ $"unreachable={string.Join(",", missing)} phantom={string.Join(",", phantom)}");
|
||||
}
|
||||
}
|
||||
|
||||
// A parameter the endpoint accepts but the tool omits is UNREACHABLE, not merely undocumented:
|
||||
// ToolArgumentValidator rejects undeclared arguments, so the caller cannot pass it at all
|
||||
// (#616 hard-capped two paged tools exactly this way). A phantom is the reverse — the tool
|
||||
// advertises something the endpoint ignores.
|
||||
drift.ShouldBeEmpty();
|
||||
}
|
||||
|
||||
private static HashSet<string> QueryParameterNames(JsonElement operation)
|
||||
{
|
||||
if (!operation.TryGetProperty("parameters", out JsonElement parameters))
|
||||
{
|
||||
return [];
|
||||
}
|
||||
|
||||
return parameters.EnumerateArray()
|
||||
.Where(p => p.TryGetProperty("in", out JsonElement location)
|
||||
&& string.Equals(location.GetString(), "query", StringComparison.Ordinal))
|
||||
.Select(p => p.GetProperty("name").GetString())
|
||||
.OfType<string>()
|
||||
.ToHashSet(StringComparer.Ordinal);
|
||||
}
|
||||
|
||||
// The body is every declared argument that is not routed elsewhere — mirroring exactly how
|
||||
// ErsatzTvApiClient builds the request, so this test cannot disagree with the code it guards.
|
||||
// DELETE is not compared: ErsatzTvApiClient sets hasBody for POST/PUT/PATCH only, so a body
|
||||
// argument on a DELETE tool would be silently dropped. No DELETE tool has one today.
|
||||
private static Dictionary<string, string> DeclaredBodyArguments(ToolDefinition tool)
|
||||
{
|
||||
var pathParameters = Regex.Matches(tool.PathTemplate, @"\{([^}]+)\}")
|
||||
.Select(m => m.Groups[1].Value)
|
||||
.ToHashSet(StringComparer.Ordinal);
|
||||
|
||||
IReadOnlySet<string> queryParameters = tool.QueryParameters ?? new HashSet<string>(StringComparer.Ordinal);
|
||||
|
||||
if (!tool.InputSchema.RootElement.TryGetProperty("properties", out JsonElement properties))
|
||||
{
|
||||
return [];
|
||||
}
|
||||
|
||||
return properties.EnumerateObject()
|
||||
.Where(p => !pathParameters.Contains(p.Name)
|
||||
&& !queryParameters.Contains(p.Name)
|
||||
&& !string.Equals(p.Name, "ifMatch", StringComparison.Ordinal))
|
||||
.ToDictionary(p => p.Name, p => DeclaredType(p.Value), StringComparer.Ordinal);
|
||||
}
|
||||
|
||||
// The tool schema's own shape: a plain "type", plus the array element type where there is one.
|
||||
private static string DeclaredType(JsonElement property)
|
||||
{
|
||||
string type = property.GetProperty("type").GetString().ShouldNotBeNull();
|
||||
|
||||
return type == "array" && property.TryGetProperty("items", out JsonElement items)
|
||||
? $"array<{items.GetProperty("type").GetString()}>"
|
||||
: type;
|
||||
}
|
||||
|
||||
private static Dictionary<string, string> RequestBodyProperties(JsonDocument spec, JsonElement operation, string toolName)
|
||||
{
|
||||
// No request body at all (queue/scan POSTs) — the tool must send none either.
|
||||
if (!operation.TryGetProperty("requestBody", out JsonElement requestBody))
|
||||
{
|
||||
return [];
|
||||
}
|
||||
|
||||
JsonElement schema = requestBody
|
||||
.GetProperty("content")
|
||||
.GetProperty("application/json")
|
||||
.GetProperty("schema");
|
||||
|
||||
// Every request body in this document is a plain $ref to a component schema. Anything else
|
||||
// (allOf/inline/oneOf) is a contract shape this guard has not been taught to read, so fail
|
||||
// loudly rather than comparing against an empty set and reporting a false pass.
|
||||
schema.TryGetProperty("$ref", out JsonElement reference)
|
||||
.ShouldBeTrue($"{toolName}: request body schema is not a $ref; teach this test the new shape");
|
||||
|
||||
JsonElement schemas = spec.RootElement.GetProperty("components").GetProperty("schemas");
|
||||
string componentName = reference.GetString().ShouldNotBeNull().Split('/')[^1];
|
||||
|
||||
return schemas
|
||||
.GetProperty(componentName)
|
||||
.GetProperty("properties")
|
||||
.EnumerateObject()
|
||||
.ToDictionary(p => p.Name, p => SpecType(schemas, p.Value, toolName, p.Name), StringComparer.Ordinal);
|
||||
}
|
||||
|
||||
// Normalize the generator's shapes onto the catalog's vocabulary. Two forms appear in this
|
||||
// document: a nullable type as ["null", T] (the catalog has no nullable notion — optionality is
|
||||
// carried by `required`), and a $ref to a component, which for the enum fields is a string enum
|
||||
// and for `logo` is an object.
|
||||
private static string SpecType(JsonElement schemas, JsonElement property, string toolName, string fieldName)
|
||||
{
|
||||
if (property.TryGetProperty("$ref", out JsonElement reference))
|
||||
{
|
||||
string componentName = reference.GetString().ShouldNotBeNull().Split('/')[^1];
|
||||
return SpecType(schemas, schemas.GetProperty(componentName), toolName, fieldName);
|
||||
}
|
||||
|
||||
JsonElement type = property.GetProperty("type");
|
||||
|
||||
string[] types = type.ValueKind == JsonValueKind.Array
|
||||
? type.EnumerateArray().Select(t => t.GetString()).OfType<string>().Where(t => t != "null").ToArray()
|
||||
: [type.GetString().ShouldNotBeNull()];
|
||||
|
||||
// More than one non-null type is a shape this guard has not been taught to read; fail rather
|
||||
// than picking one and reporting a comparison that means nothing.
|
||||
types.Length.ShouldBe(1, $"{toolName}.{fieldName}: unexpected OpenAPI type union [{string.Join(", ", types)}]");
|
||||
|
||||
// The element schema is resolved through the same normalization: an array's items can itself be
|
||||
// a $ref to a component (ReplaceRemoteLibraryPreferencesRequest.libraries), which the catalog
|
||||
// declares as an object array.
|
||||
return types[0] == "array" && property.TryGetProperty("items", out JsonElement items)
|
||||
? $"array<{SpecType(schemas, items, toolName, fieldName)}>"
|
||||
: types[0];
|
||||
}
|
||||
|
||||
private static JsonDocument LoadOpenApiDocument()
|
||||
{
|
||||
string path = Path.Combine(AppContext.BaseDirectory, "openapi", "v1.json");
|
||||
|
||||
// A missing spec would make every assertion above vacuous, so it is an explicit failure.
|
||||
File.Exists(path).ShouldBeTrue(
|
||||
$"OpenAPI document not found at {path}; the test project links it from ErsatzTV/wwwroot/openapi/v1.json");
|
||||
|
||||
return JsonDocument.Parse(File.ReadAllText(path));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -29,14 +29,32 @@ public static class ToolCatalog
|
||||
Get("ersatztv_list_schedules", "List schedules.", "/api/v1/schedules"),
|
||||
Get("ersatztv_get_schedule", "Get a schedule by id.", "/api/v1/schedules/{id}", IdPath("Schedule id.")),
|
||||
Get("ersatztv_get_schedule_items", "Get a schedule's items. Emits the schedule ETag.", "/api/v1/schedules/{id}/items", IdPath("Schedule id.")),
|
||||
Get("ersatztv_list_playouts", "List playouts (paged).", "/api/v1/playouts", [], Page()),
|
||||
Get(
|
||||
"ersatztv_list_playouts",
|
||||
"List playouts (paged), optionally filtered by channel name.",
|
||||
"/api/v1/playouts",
|
||||
[],
|
||||
[
|
||||
Str(
|
||||
"query",
|
||||
"Optional case-insensitive substring match on the CHANNEL name (not the playout or schedule name); omit for all playouts.",
|
||||
arg: In.Query),
|
||||
.. Page()
|
||||
]),
|
||||
Get("ersatztv_get_playout", "Get a playout by id.", "/api/v1/playouts/{id}", IdPath("Playout id.")),
|
||||
Get(
|
||||
"ersatztv_get_playout_items",
|
||||
"Get upcoming items (and unscheduled gaps) for a playout (paged).",
|
||||
"/api/v1/playouts/{id}/items",
|
||||
[IdPath("Playout id.")],
|
||||
Page()),
|
||||
[
|
||||
Bool(
|
||||
"showFiller",
|
||||
"Include items whose filler kind is not None (pre/mid/post-roll, tail, fallback, guide-mode, deco); "
|
||||
+ "default false returns only non-filler items.",
|
||||
arg: In.Query),
|
||||
.. Page()
|
||||
]),
|
||||
Get("ersatztv_list_ffmpeg_profiles", "List FFmpeg profiles.", "/api/v1/ffmpeg/profiles"),
|
||||
Get("ersatztv_get_ffmpeg_profile", "Get an FFmpeg profile by id.", "/api/v1/ffmpeg/profiles/{id}", IdPath("FFmpeg profile id.")),
|
||||
Get(
|
||||
@@ -132,7 +150,8 @@ public static class ToolCatalog
|
||||
[Str("name", "Schedule name.", required: true), .. ScheduleFlags()]),
|
||||
Put(
|
||||
"ersatztv_update_schedule",
|
||||
"Update a program schedule's settings.",
|
||||
"Update a program schedule. Send the full desired state: every field is applied, so omitting "
|
||||
+ "padToNearestMinute CLEARS a configured pad (GET the schedule first to copy current values).",
|
||||
"/api/v1/schedules/{id}",
|
||||
[IdPath("Schedule id."), Str("name", "Schedule name.", required: true), .. ScheduleFlags()]),
|
||||
Delete("ersatztv_delete_schedule", "Delete a program schedule.", "/api/v1/schedules/{id}", IdPath("Schedule id.")),
|
||||
@@ -159,9 +178,22 @@ public static class ToolCatalog
|
||||
ChannelFields()),
|
||||
Put(
|
||||
"ersatztv_update_channel",
|
||||
"Update a channel. Send the full desired state; enum fields take the enum name (GET the channel first to copy current values).",
|
||||
"Update a channel. Send the full desired state; enum fields take the enum name (GET the channel first to copy current values). "
|
||||
+ "graphicsElementIds is part of that state: omitting it DETACHES every graphics element (e.g. the On Now/Next overlay), "
|
||||
+ "so copy it from ersatztv_get_channel unless you mean to clear it.",
|
||||
"/api/v1/channels/{id}",
|
||||
[IdPath("Channel id."), .. ChannelFields()]),
|
||||
[
|
||||
IdPath("Channel id."),
|
||||
.. ChannelFields(),
|
||||
|
||||
// Update-only: UpdateChannelRequest carries GraphicsElementIds, CreateChannelRequest does
|
||||
// not, so this cannot move into the shared ChannelFields() without making create send an
|
||||
// unknown property. PUT is a full replace, so omitting it detaches every attached element
|
||||
// with no error — issue #754.
|
||||
IntArray(
|
||||
"graphicsElementIds",
|
||||
"Ids of the graphics elements attached to the channel. Full replace: omit or send [] to detach all.")
|
||||
]),
|
||||
Post(
|
||||
"ersatztv_reset_channel_playout",
|
||||
"Queue a rebuild of a channel's playout (202 Accepted; 409 if a build is already running).",
|
||||
@@ -297,7 +329,12 @@ public static class ToolCatalog
|
||||
Bool("treatCollectionsAsShows", "Treat collections as shows."),
|
||||
Bool("shuffleScheduleItems", "Shuffle schedule items."),
|
||||
Bool("randomStartPoint", "Use a random start point."),
|
||||
Str("fixedStartTimeBehavior", "Fixed start-time behavior (enum name; GET a schedule to see valid values).")
|
||||
Str("fixedStartTimeBehavior", "Fixed start-time behavior (enum name; GET a schedule to see valid values)."),
|
||||
|
||||
// Both Create- and UpdateScheduleRequest carry this, so it belongs in the shared helper. The
|
||||
// update PUT is a full replace that writes the value unconditionally, so omitting it used to
|
||||
// clear a configured pad silently — the same #754 shape as channel graphicsElementIds.
|
||||
Int("padToNearestMinute", "Pad each item to the nearest N minutes; omit or send null for no padding.")
|
||||
];
|
||||
|
||||
// ---- Tool factories ----
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
using System.Collections;
|
||||
using System.Reflection;
|
||||
using System.Threading.Channels;
|
||||
using ErsatzTV.Application;
|
||||
using ErsatzTV.Application.ProgramSchedules;
|
||||
@@ -65,7 +67,8 @@ public class ScheduleItemResponseRoundTripTests
|
||||
await replaceHandler.Handle(new ReplaceProgramScheduleItems(scheduleId, reconstructed), CancellationToken.None);
|
||||
replaced.IsRight.ShouldBeTrue(replaced.LeftToSeq().HeadOrNone().Match(e => e.Value, () => "unknown"));
|
||||
|
||||
// GET again → envelope B; A and B must be semantically identical (ignoring regenerated row ids).
|
||||
// GET again → envelope B; A and B must be semantically identical INCLUDING row ids —
|
||||
// the handler reconciles by id and updates in place, it does not regenerate rows.
|
||||
ScheduleItemsResponseModel envelopeB = await GetItemsEnvelope(scheduleId);
|
||||
|
||||
envelopeB.Items.Count.ShouldBe(envelopeA.Items.Count);
|
||||
@@ -340,62 +343,97 @@ public class ScheduleItemResponseRoundTripTests
|
||||
r.PreferredSubtitleLanguageCode,
|
||||
r.SubtitleMode);
|
||||
|
||||
// ersatztv#779 (detector G): the compared field list is DERIVED from the DTO by reflection,
|
||||
// never hand-copied. The previous version was a hand-written run of `b.X.ShouldBe(a.X)` lines.
|
||||
// It was COMPLETE on the day it was written — every property but Id — and had no way
|
||||
// to report the day it stopped being: a field added to ScheduleItemResponseModel simply went
|
||||
// uncompared, and this "lossless round-trip" test kept passing while the round trip silently
|
||||
// dropped it. That is #754's mechanism exactly (a hand-maintained mirror drifting from a
|
||||
// 28-property DTO by one field, HTTP 200, no error), one altitude up — in the very test whose
|
||||
// job is to catch losses.
|
||||
//
|
||||
// Properties deliberately NOT compared. The set is EMPTY, and that is a finding rather than an
|
||||
// oversight. The first version exempted Id on the reasoning that "the PUT replaces the item
|
||||
// set, so B's rows are new rows with new ids". ReplaceProgramScheduleItemsHandler does not do
|
||||
// that for this fixture's payload: it forwards every Id, takes the id-based reconcile, and
|
||||
// updates rows in place. So Id compares equal, and the exemption was unnecessary.
|
||||
//
|
||||
// Two mutations of this fixture, both EXECUTED — recorded as results, with no account of why,
|
||||
// because three earlier drafts of this comment each supplied a confident mechanism for a
|
||||
// correct observation and two of them were contradicted by the code:
|
||||
//
|
||||
// ToReplaceCommand passes `null` for EVERY id -> test stays GREEN
|
||||
// ToReplaceCommand passes `null` for index 0 only -> test goes RED, "Id differs"
|
||||
//
|
||||
// So the Id comparison does discriminate; it is not decorative. What it is NOT is a substitute
|
||||
// for ReplaceProgramScheduleItemsReconcileTests, whose
|
||||
// Reorder_ById_Should_Move_State_With_The_Logical_Item_Not_The_Slot and
|
||||
// Insert_ById_In_Middle_Should_Keep_Existing_Ids_And_State pass real ids and pin that state
|
||||
// moves with the logical item rather than the slot. Those are the #252 tests; this is a
|
||||
// round-trip check that happens to also notice a lost row.
|
||||
//
|
||||
// Any name added here must still exist on ScheduleItemResponseModel (asserted below), so
|
||||
// renaming a field cannot leave a stale exemption silently exempting nothing.
|
||||
private static readonly System.Collections.Generic.HashSet<string> RoundTripExemptProperties =
|
||||
new(StringComparer.Ordinal);
|
||||
|
||||
private static void AssertSemanticallyEqual(ScheduleItemResponseModel a, ScheduleItemResponseModel b)
|
||||
{
|
||||
b.Index.ShouldBe(a.Index);
|
||||
b.StartType.ShouldBe(a.StartType);
|
||||
b.StartTime.ShouldBe(a.StartTime);
|
||||
b.FixedStartTimeBehavior.ShouldBe(a.FixedStartTimeBehavior);
|
||||
b.PlayoutMode.ShouldBe(a.PlayoutMode);
|
||||
b.CollectionType.ShouldBe(a.CollectionType);
|
||||
b.CollectionId.ShouldBe(a.CollectionId);
|
||||
b.MultiCollectionId.ShouldBe(a.MultiCollectionId);
|
||||
b.SmartCollectionId.ShouldBe(a.SmartCollectionId);
|
||||
b.RerunCollectionId.ShouldBe(a.RerunCollectionId);
|
||||
b.MediaItemId.ShouldBe(a.MediaItemId);
|
||||
b.PlaylistId.ShouldBe(a.PlaylistId);
|
||||
b.SearchTitle.ShouldBe(a.SearchTitle);
|
||||
b.SearchQuery.ShouldBe(a.SearchQuery);
|
||||
b.PlaybackOrder.ShouldBe(a.PlaybackOrder);
|
||||
b.MarathonGroupBy.ShouldBe(a.MarathonGroupBy);
|
||||
b.MarathonShuffleGroups.ShouldBe(a.MarathonShuffleGroups);
|
||||
b.MarathonShuffleItems.ShouldBe(a.MarathonShuffleItems);
|
||||
b.MarathonBatchSize.ShouldBe(a.MarathonBatchSize);
|
||||
b.FillWithGroupMode.ShouldBe(a.FillWithGroupMode);
|
||||
b.MultipleMode.ShouldBe(a.MultipleMode);
|
||||
b.MultipleCount.ShouldBe(a.MultipleCount);
|
||||
b.PlayoutDuration.ShouldBe(a.PlayoutDuration);
|
||||
b.TailMode.ShouldBe(a.TailMode);
|
||||
b.DiscardToFillAttempts.ShouldBe(a.DiscardToFillAttempts);
|
||||
b.CustomTitle.ShouldBe(a.CustomTitle);
|
||||
b.GuideMode.ShouldBe(a.GuideMode);
|
||||
b.PreRollFillerId.ShouldBe(a.PreRollFillerId);
|
||||
b.MidRollFillerId.ShouldBe(a.MidRollFillerId);
|
||||
b.PostRollFillerId.ShouldBe(a.PostRollFillerId);
|
||||
b.TailFillerId.ShouldBe(a.TailFillerId);
|
||||
b.FallbackFillerId.ShouldBe(a.FallbackFillerId);
|
||||
b.WatermarkIds.ShouldBe(a.WatermarkIds);
|
||||
b.GraphicsElementIds.ShouldBe(a.GraphicsElementIds);
|
||||
b.PreferredAudioLanguageCode.ShouldBe(a.PreferredAudioLanguageCode);
|
||||
b.PreferredAudioTitle.ShouldBe(a.PreferredAudioTitle);
|
||||
b.PreferredSubtitleLanguageCode.ShouldBe(a.PreferredSubtitleLanguageCode);
|
||||
b.SubtitleMode.ShouldBe(a.SubtitleMode);
|
||||
b.CollectionName.ShouldBe(a.CollectionName);
|
||||
b.MultiCollectionName.ShouldBe(a.MultiCollectionName);
|
||||
b.SmartCollectionName.ShouldBe(a.SmartCollectionName);
|
||||
b.RerunCollectionName.ShouldBe(a.RerunCollectionName);
|
||||
b.PlaylistName.ShouldBe(a.PlaylistName);
|
||||
b.PlaylistGroupId.ShouldBe(a.PlaylistGroupId);
|
||||
b.MediaItemName.ShouldBe(a.MediaItemName);
|
||||
b.PreRollFillerName.ShouldBe(a.PreRollFillerName);
|
||||
b.MidRollFillerName.ShouldBe(a.MidRollFillerName);
|
||||
b.PostRollFillerName.ShouldBe(a.PostRollFillerName);
|
||||
b.TailFillerName.ShouldBe(a.TailFillerName);
|
||||
b.FallbackFillerName.ShouldBe(a.FallbackFillerName);
|
||||
b.Watermarks.Select(w => (w.Id, w.Name)).ShouldBe(a.Watermarks.Select(w => (w.Id, w.Name)));
|
||||
b.GraphicsElements.Select(g => (g.Id, g.Name)).ShouldBe(a.GraphicsElements.Select(g => (g.Id, g.Name)));
|
||||
b.Name.ShouldBe(a.Name);
|
||||
b.DurationEstimate.ShouldBe(a.DurationEstimate);
|
||||
PropertyInfo[] properties = typeof(ScheduleItemResponseModel)
|
||||
.GetProperties(BindingFlags.Public | BindingFlags.Instance);
|
||||
|
||||
// A stale exemption is a silent hole: it would exempt nothing while reading as a reviewed
|
||||
// decision, and the property it once named would be compared or not by accident.
|
||||
foreach (string exempt in RoundTripExemptProperties)
|
||||
{
|
||||
properties.Any(p => p.Name == exempt).ShouldBeTrue(
|
||||
$"'{exempt}' is exempted from the round-trip comparison but is not a property of "
|
||||
+ $"{nameof(ScheduleItemResponseModel)}; remove the stale exemption or fix the name.");
|
||||
}
|
||||
|
||||
var compared = 0;
|
||||
foreach (PropertyInfo property in properties)
|
||||
{
|
||||
if (RoundTripExemptProperties.Contains(property.Name))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
object? expected = property.GetValue(a);
|
||||
object? actual = property.GetValue(b);
|
||||
|
||||
if (expected is IEnumerable expectedSequence and not string)
|
||||
{
|
||||
// Collection-valued members (WatermarkIds, Watermarks, GraphicsElementIds,
|
||||
// GraphicsElements). The elementwise walk still delegates to each element's Equals,
|
||||
// so it is value equality only because those elements are records
|
||||
// (NamedIdResponseModel) or value types (the int id lists); a future element type that
|
||||
// is neither would silently be compared by REFERENCE inside this loop. It is also order-sensitive, which is
|
||||
// correct for these ordered lists but would be wrong for an unordered type such as
|
||||
// a dictionary-valued property.
|
||||
actual.ShouldNotBeNull($"{property.Name} was null on the round-tripped item");
|
||||
var actualSequence = (IEnumerable)actual;
|
||||
actualSequence.Cast<object?>().ToList()
|
||||
.ShouldBe(expectedSequence.Cast<object?>().ToList(), $"{property.Name} differs");
|
||||
}
|
||||
else
|
||||
{
|
||||
actual.ShouldBe(expected, $"{property.Name} differs");
|
||||
}
|
||||
|
||||
compared++;
|
||||
}
|
||||
|
||||
// Anti-vacuity, as a PIN rather than a floor. A `>=` floor lets properties vanish silently,
|
||||
// which is the one-sided version of the both-directions rule this test is meant to embody.
|
||||
// Comparing against the reflected count minus exemptions would be tautological — both sides
|
||||
// come from the same reflection — so the expected number is written down and must be
|
||||
// bumped deliberately in the same change that adds or removes a DTO field.
|
||||
const int expectedComparedProperties = 55;
|
||||
compared.ShouldBe(
|
||||
expectedComparedProperties,
|
||||
$"{compared} properties were compared, expected {expectedComparedProperties}; update "
|
||||
+ "this pin in the same change that alters ScheduleItemResponseModel's field list");
|
||||
}
|
||||
|
||||
private async Task<int> SeedScheduleAndReferences(bool shuffleScheduleItems)
|
||||
|
||||
@@ -126,6 +126,70 @@ public class ApiKeyProviderTests
|
||||
key.ShouldMatch("^[0-9a-f]{64}$");
|
||||
}
|
||||
|
||||
// ---- Api:RequireKeyForReads, read through the REAL provider (ersatztv#779, detector F) ----
|
||||
//
|
||||
// Every other assertion about the read-gating posture goes through a hand-written
|
||||
// FakeApiKeyProvider that is HANDED the bool (ApiAuthorizationFilterTests,
|
||||
// ApiKeyEndpointRequiresKeyTests). Those fakes prove the FILTER reacts to the flag; they cannot
|
||||
// see the line that DERIVES it, because they never run it. Until these tests, nothing in the
|
||||
// suite constructed ApiKeyProvider at all, so a mistyped configuration key or a flipped default
|
||||
// would have left the whole suite green while shipping anonymous reads (#280/#282).
|
||||
//
|
||||
// Api:WriteKey is set in every case purely so ResolveKey returns before touching the real
|
||||
// FileSystemLayout.ApiKeyPath — the constructor would otherwise generate and persist a key into
|
||||
// the live config volume. It is deliberately NOT the subject of these tests.
|
||||
private static ApiKeyProvider ProviderWith(params (string Key, string Value)[] settings)
|
||||
{
|
||||
// The WriteKey entry is appended LAST so a caller cannot override it to empty. That is not
|
||||
// hypothetical tidiness: an empty Api:WriteKey sends ResolveKey down the real path, which
|
||||
// reads, generates and PERSISTS a key into the live config volume (FileSystemLayout
|
||||
// .ApiKeyPath) from a unit test.
|
||||
var withKey = new List<(string, string)>(settings) { (ApiKeyProvider.WriteKeyConfigurationKey, "test-key") };
|
||||
return new ApiKeyProvider(Config(withKey.ToArray()), NullLogger<ApiKeyProvider>.Instance);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void Read_Gating_Is_Required_When_The_Setting_Is_Absent()
|
||||
{
|
||||
// The shipped default, and the case a fixture that simply OMITS the field would test by
|
||||
// accident. Asserted explicitly so it is a pinned decision rather than a coincidence.
|
||||
ProviderWith().RequireKeyForReads.ShouldBeTrue();
|
||||
}
|
||||
|
||||
[TestCase("true")]
|
||||
[TestCase("True")]
|
||||
[TestCase("TRUE")]
|
||||
public void Read_Gating_Is_Required_At_The_Explicit_Production_Value(string configured)
|
||||
{
|
||||
// The DENY path at the production value, which is the half #756 showed can stay invisible:
|
||||
// the absent case behaving correctly says nothing about the configured one.
|
||||
ProviderWith((ApiKeyProvider.RequireKeyForReadsConfigurationKey, configured))
|
||||
.RequireKeyForReads.ShouldBeTrue();
|
||||
}
|
||||
|
||||
[TestCase("")]
|
||||
[TestCase("1")]
|
||||
[TestCase("yes")]
|
||||
public void A_Non_Boolean_Read_Gating_Value_Fails_Startup_Rather_Than_Reads(string configured)
|
||||
{
|
||||
// The fourth cell of the matrix, and the one an operator actually hits: `Api__RequireKeyForReads=`
|
||||
// with nothing after it in a compose file, or a habitual `1`/`yes`. ConfigurationBinder returns
|
||||
// the default ONLY for a null section value, so any present-but-unparseable string goes through
|
||||
// BooleanConverter and throws. That is fail-CLOSED — the app refuses to start rather than
|
||||
// quietly choosing a posture — and it is pinned here so a future switch to a lenient parse
|
||||
// (TryParse with a fallback) cannot silently turn a typo into anonymous reads.
|
||||
Should.Throw<InvalidOperationException>(() =>
|
||||
ProviderWith((ApiKeyProvider.RequireKeyForReadsConfigurationKey, configured)));
|
||||
}
|
||||
|
||||
[TestCase("false")]
|
||||
[TestCase("False")]
|
||||
public void Read_Gating_Is_Waived_Only_By_An_Explicit_Opt_Out(string configured)
|
||||
{
|
||||
ProviderWith((ApiKeyProvider.RequireKeyForReadsConfigurationKey, configured))
|
||||
.RequireKeyForReads.ShouldBeFalse();
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void Returns_A_Usable_Key_Even_When_Persist_Fails()
|
||||
{
|
||||
|
||||
@@ -24,6 +24,13 @@ doc below, or that changes which sections a task signal points to.**
|
||||
| Concurrency / optimistic-locking work | `docs/api-conventions.md` §7a/b/c + `docs/decisions/optimistic-concurrency.md` |
|
||||
| Auth / security-surface work | `docs/decisions/api-auth-security.md` |
|
||||
| CI / release pipeline work | `docs/ci-cd.md` + `docs/decisions/release-ci-governance.md` |
|
||||
| Proposing a new guard / CI check / regression test convention | `docs/defect-shapes-773.md` §4 (detector menu + the classes where no detector is plausible), then the three rules every guard must satisfy: `docs/decisions/records/testing/guard-derives-population-from-source.md`, `…/guard-ships-with-mutation-proof.md` and `…/mutation-claims-are-executed.md` (a `MUTATION` grade carries a DECLARED clause mutation that is re-run every suite) |
|
||||
| Testing a surface gated by config / an env var / a credential | `docs/decisions/records/testing/deny-path-at-production-config-value.md` — cover the setting absent, at its production value, and each opt-out, and assert the DENY branch |
|
||||
| Touching a full-replace write path or a hand-built request object | `docs/decisions/records/testing/full-replace-asserts-field-list.md` — derive the field list from the DTO and assert set equality; reconcile by id where child state exists |
|
||||
| Writing or editing any doc, or answering a review finding in prose | `docs/decisions/records/docs/no-session-narrative.md` — the doc records the END STATE; the path to it goes in the commit message. Apply the who-benefits test, and read the carve-out before you cut (dated measurements, stated snapshot boundaries and tested-and-rejected results stay) |
|
||||
| Adding / changing / deleting a guard file | `docs/guard-inventory.md` — every guard's row is machine-checked by `scripts/tests/test_guard_inventory.py`, so a new guard must acquire a row before the suite goes green, and a row graded `MUTATION` must also acquire a declared clause in `scripts/tests/mutation_manifest.py` |
|
||||
| Writing code that reads live Gitea/remote state and then acts on it | `docs/decisions/records/process/check-and-use-pins-a-version.md`, then `docs/remote-state-inventory.md` — a new executable under `scripts/` (**excluding `scripts/tests/`**), `.claude/hooks/`, `.husky/` or `.gitea/workflows/` must acquire a row there before `scripts/tests/test_remote_state_inventory.py` goes green |
|
||||
| Finding every site that references a symbol (multi-site fix/sweep) | `docs/local-lsp-tooling.md` — which surface answers, and why a delegated agent must be pointed at the `csharp-lsp` MCP tools rather than the `LSP` tool |
|
||||
| Live local run / Playwright-MCP verification | `docs/e2e-local.md` + `scripts/e2e-local.sh` |
|
||||
| Adding/changing a UI-E2E browser flow | `docs/e2e-local.md` → "UI-E2E harness" + `scripts/e2e-ui.sh` |
|
||||
| What does a test suite cover | `docs/testing.md` |
|
||||
@@ -66,6 +73,11 @@ bounds, what's mined per issue): `docs/handoffs/chicorytv-issue-queue.md` → "K
|
||||
- **`docs/spa-conventions.md`** — playbook for adding a screen to the ChicoryTV React SPA.
|
||||
- **`docs/e2e-local.md`** (+ `scripts/e2e-local.sh`) — how to run a live local instance for manual
|
||||
or Playwright-MCP verification.
|
||||
- **`docs/local-lsp-tooling.md`** — the code-intelligence surfaces (the `LSP` tool's three servers and
|
||||
the `csharp-lsp` MCP server): how each is configured, which one a **subagent** can actually reach,
|
||||
the traps (a cold server answers the first query with a confidently partial result), and
|
||||
`scripts/check-local-lsp.sh` to verify the preconditions. Read before briefing an agent to find
|
||||
every site referencing a symbol.
|
||||
- **`docs/testing.md`** — testing map: what each `*.Tests` project / `web` suite covers,
|
||||
golden-file nets, the timezone-independence rule, how to run subsets, the per-PR verification
|
||||
gate.
|
||||
@@ -98,6 +110,28 @@ bounds, what's mined per issue): `docs/handoffs/chicorytv-issue-queue.md` → "K
|
||||
that file's standing kickoff for the two concurrent tracks (orientation ‖ selection). ersatztv#237
|
||||
is a closed, archival historical tracker (superseded by `startup.parallel-orientation` in
|
||||
`docs/decisions.md`) — not a live pointer.
|
||||
- **`docs/defect-shapes-773.md`** — root-cause analysis of the recurring defect shapes across the
|
||||
whole closed-issue corpus (#773): the measured class ranking, the four families they consolidate
|
||||
into, the cheapest mechanical detector per class, the classes where **no** detector is plausible,
|
||||
and an audit of which configured hooks/MCP servers/LSPs are actually invoked. Read it before
|
||||
proposing a new guard or CI check — §4 is the detector menu, and it argues against enumerating
|
||||
cases one incident at a time.
|
||||
- **`docs/remote-state-inventory.md`** — every executable in `scripts/` (**excluding
|
||||
`scripts/tests/`**), `.claude/hooks/`, `.husky/` and `.gitea/workflows/` that reads live remote
|
||||
state and acts on that read, classified `PINNED` / `CAS` / `UNSAFE-KNOWN` / `N/A` with the window
|
||||
and what bounds it. Code outside those directories — C#/TypeScript guards, `web/`, and the test
|
||||
suites themselves — is out of scope, and the doc states that rather than implying coverage.
|
||||
The population is derived from `git ls-files` and compared for set equality by
|
||||
`scripts/tests/test_remote_state_inventory.py`, so a new script that talks to a remote service
|
||||
cannot ship unclassified. Read it with `process.check-and-use-pins-a-version`; it is that record's
|
||||
detector, since the class has no plausible linter (`docs/defect-shapes-773.md` §4 detector D).
|
||||
- **`docs/guard-inventory.md`** — every executable guard file, what it blocks, whether it is a
|
||||
`GUARD` or `TOOLING`, and whether it ships a mutation proof (`MUTATION` / `BEHAVIOUR-ONLY` /
|
||||
`NONE`) with a `file::function` ref. The population is derived from the filesystem and the
|
||||
workflow/hook call sites and compared for set equality by `scripts/tests/test_guard_inventory.py`,
|
||||
so a new guard cannot ship unclassified and a renamed test cannot leave a row claiming coverage it
|
||||
has lost. Guards implemented inline in workflow YAML are deliberately outside that population —
|
||||
the doc states the limit rather than implying coverage.
|
||||
- **`docs/tracker-retrofit-triage-237.md`** — audit trail for the #524 triage of ersatztv#237's 111
|
||||
comments (method, per-comment classification, totals). Evidence for the
|
||||
`docs.tracker-comment-retrofit` decision; read it only when triaging another over-cap tracker.
|
||||
|
||||
+540
-18
@@ -114,6 +114,24 @@ commit in the range (see the `decisions.md` header) — routine lifecycle metada
|
||||
> `DeployStack media-servers` silently targets the **dead** stack. Confirm with
|
||||
> `/read ListStacks` before deploying.
|
||||
>
|
||||
> **A third stack, literally named `ersatztv`, is also live and is NOT prod**: it owns
|
||||
> `ersatztv-test`, tracks the floating `:latest` tag, and runs with `auto_update: true`. (Stack
|
||||
> name, compose project, config-file path and image tag re-confirmed from the container labels on
|
||||
> jazz 2026-08-11; the `auto_update` flags and the bumblebee `unhealthy` status are carried from
|
||||
> the earlier checks noted above, not re-measured that day.)
|
||||
> `DeployStack ersatztv` succeeds and looks healthy — it just promotes nothing to prod. Three names,
|
||||
> one target:
|
||||
>
|
||||
> | Stack name | project_name | auto_update | owns |
|
||||
> |---|---|---|---|
|
||||
> | `jazz-media` | `media-servers` | `false` | `ersatztv` (**prod**, tracks `:prod`) |
|
||||
> | `ersatztv` | `ersatztv` | `true` | `ersatztv-test` (tracks `:latest`) |
|
||||
> | `media-servers` | — | — | dead stack on bumblebee, `unhealthy` |
|
||||
>
|
||||
> Don't resolve the prod stack by name at all — resolve it by the **container's**
|
||||
> `com.docker.compose.project.config_files` label (e.g. via `docker inspect`), which points at the
|
||||
> actual compose file path and can't be confused by a same-named stack.
|
||||
>
|
||||
> **There is no Global Auto Update fallback anymore**: `jazz-media` has `auto_update: false`
|
||||
> (`poll_for_updates: true` only), so nothing promotes `:prod` on a timer — promotion is manual,
|
||||
> full stop. The old "don't cut a tag near the 03:00 run" caveat no longer applies.
|
||||
@@ -182,9 +200,17 @@ rather than in `docker-build.yml` — see that section (ersatztv#535).
|
||||
|
||||
**`small` is git-only, and that is load-bearing (server-management#639).** Everything in
|
||||
the lane is a checkout plus a `git diff`: `decisions-guard`, `ci-image-pin`,
|
||||
`docs-reminder` — plus `script-tests`, which is a checkout plus a `pytest` run needing only
|
||||
`docs-reminder` — plus `prove-fix` (ersatztv#794), the heaviest member: per commit carrying a
|
||||
`Proves:` trailer it makes two `git worktree add`s and runs an arbitrary pytest selection twice,
|
||||
so a PR claiming many proofs costs proportionally more than the rest of the lane combined — plus
|
||||
`script-tests`, which is a checkout plus a `pytest` run needing only
|
||||
`pytest` and `pyyaml` (ersatztv#631; it is NOT stdlib-only — that assumption is what turned the
|
||||
job red on its first CI run, see below). Nothing there runs a compiler or a `docker build`, which is why the lane
|
||||
job red on its first CI run, see below) — plus **`scan`** (ersatztv#767), the same
|
||||
lightweight-Python shape. `scan` is the lane member to think hardest about before changing anything
|
||||
here: it is the only one that lives in `docker-build.yml` rather than `pr-checks.yml`, so the only
|
||||
one that runs on a **tag push**, and the only one whose failure does not merely redden a status but
|
||||
**skips `build`** — an OOM or a wedge there yields no release image at all.
|
||||
Nothing there runs a compiler or a `docker build`, which is why the lane
|
||||
can be capped at 1 GiB per job. The lightweight-Python jobs are the deliberate edge of the
|
||||
"git-only" rule, not an exception to it: `setup-python` + `pip install pytest` + a suite whose
|
||||
heaviest allocation is a handful of temp-dir git repos stays far under the cap. Route a heavy job here and it will OOM — give it
|
||||
@@ -242,7 +268,7 @@ thing that ran in the lane) — and their 10 GiB requirement set the lane's per-
|
||||
on a 25 GiB host permits exactly one slot. Moving both to `ubuntu-latest` made the lane
|
||||
genuinely tiny, so it could widen to **4 slots across two hosts while committing less RAM to
|
||||
CI than the single slot did**. `docker-build.yml`'s `build` does not re-create #574's
|
||||
skip-task queueing, because `needs: [test, migrations]` means it cannot be dispatched until
|
||||
skip-task queueing, because `needs: [test, migrations, scan]` means it cannot be dispatched until
|
||||
the lane it would queue behind has already drained.
|
||||
|
||||
### CI build memory: no persistent compiler servers (ersatztv#406)
|
||||
@@ -413,6 +439,14 @@ the image build.
|
||||
`insecure-registries`**, so without this, cache/base-image/push over the HTTP
|
||||
registry fails (`http: server gave HTTP response to HTTPS client`).
|
||||
3. `docker/login-action` with repo secrets `REGISTRY_USER` / `REGISTRY_PASSWORD`.
|
||||
**`REGISTRY_PASSWORD` is a scoped PAT (`write:package` + `read:repository`), not an account
|
||||
password** — deliberately, so head-resolved PR code cannot use it to forge a commit status
|
||||
(`ci.actions-credential-scoping`, ersatztv#697). If a job ever fails with `token does not have at
|
||||
least one of required scope(s)`, the fix is to narrow what the job does, **never** to widen the
|
||||
token to `write:repository` or to put the admin password back. Note what the scope still reaches:
|
||||
`write:package` covers `ersatztv:prod` (the tag prod's stack follows) and `ersatztv-ci:<sha>` (the
|
||||
toolchain image five `container:` jobs execute), so this is the deployment supply chain, not an
|
||||
inert endpoint — see `ci.actions-credential-scoping`.
|
||||
4. `docker/build-push-action@v6`: amd64-only, `docker/Dockerfile`, `INFO_VERSION`
|
||||
build-arg, registry layer cache (`type=registry,ref=…:buildcache`,
|
||||
`cache-to … ignore-error=true`).
|
||||
@@ -597,14 +631,243 @@ CI-validated, so the tree-match check correctly declines. So the skip is a genui
|
||||
win (clean, up-to-date, un-rebased merges in quiet periods) — correct-but-conservative by
|
||||
construction, not a general dedup. It never fires unsafely; when in doubt it runs the full matrix.
|
||||
|
||||
### Dropped-step guard on the required jobs (ersatztv#756)
|
||||
|
||||
`test` and `migrations` write the only two `docker-build.yml` contexts branch protection requires on
|
||||
`main`. A step the runner declines to interpolate is **dropped, and the job still concludes
|
||||
`success`** (ersatztv#751), so in these two jobs that failure is **fail-OPEN**: a required check
|
||||
reports green having done no work. In `review-verdict.yml` the same drop is fail-closed — the status
|
||||
is simply absent and the merge is blocked — which is why #751 fixed the safe direction first.
|
||||
|
||||
Two independent mechanisms hold it, and neither is redundant:
|
||||
|
||||
- **A static ban on expression delimiters** in any `run:` body of `test`, `migrations` **and
|
||||
`build`**. The drop mechanism *requires* an opener in the scalar, so this makes the class
|
||||
unreachable rather than merely detected — and it is the raw `${{` opener that is banned, not a
|
||||
well-formed pair, because an unclosed one triggers the same rewrite. When a step genuinely needs a
|
||||
value, pass it through the step's `env:` block, which is interpolated **per value**, so a bad
|
||||
payload there cannot take the body with it.
|
||||
|
||||
**Why `build` is in the ban although it is not a required context.** Its one delimiter-bearing body
|
||||
was `Smoke + IPTV E2E`, which runs *after* `Build and push` — so on a `v*` tag the image is already
|
||||
in the registry as the release candidate and that step is what decides whether the candidate was
|
||||
ever booted. A drop there publishes an unsmoked candidate, reports green, and `DeployStack
|
||||
jazz-media` promotes exactly that image. Its two payloads moved into the step's `env:`, so the ban
|
||||
cost nothing.
|
||||
|
||||
**The ban is re-checked on the release path itself (ersatztv#767).** It used to be enforced only by
|
||||
the `script-tests` job, which lives in `pr-checks.yml` (`on: pull_request`) and is **not** a
|
||||
required context — a *review-time* check on the PR that would introduce a delimiter, not a gate on
|
||||
the release. `pr-checks.yml` does not run on a `v*` tag push at all, so a delimiter that ever
|
||||
reached `main` would still drop `Smoke` on the tag build and go green; `main` being PR-only (#743)
|
||||
meant such a change had to pass through a PR where `script-tests` reddens, but a red on a
|
||||
non-required check does not block the merge server-side.
|
||||
|
||||
There is now a **`scan` job** (`Delimiter ban (release path)`) that runs the PyYAML-based ban test,
|
||||
and **`build` lists it in `needs:`**. That single edge is the fail-closed property: a red `scan`
|
||||
means `build` is skipped outright, so the image is never built, let alone pushed.
|
||||
|
||||
**Why a job and not a step inside `build`.** A step cannot protect the job it lives in. `build` is
|
||||
what publishes, so a guard step there fails **open** if the runner drops it — and the defence
|
||||
("the guard's own body has no opener, so it cannot be dropped") is circular when the only thing
|
||||
enforcing that property is the same PR-only test being backstopped. This was the first design and
|
||||
two independent reviews rejected it for exactly that.
|
||||
|
||||
**Why it runs the real pytest and not a bespoke scanner.** The same first cut hand-parsed the
|
||||
workflow YAML in stdlib Python, to avoid provisioning PyYAML on `build`'s bare runner. Review found
|
||||
~10 **false negatives** in that parser in one round — flow mappings (`{run: …}`), a quoted
|
||||
`"run":` key, aliases, multiline quoted scalars — making it strictly *weaker* than the check it
|
||||
backstopped, in the only direction that matters for a security gate. Running the existing test
|
||||
needs no second definition of "what is a `run:` body", so it has no drift surface at all. `scan`
|
||||
runs on `small` and provisions Python the same way `script-tests` does.
|
||||
|
||||
The wiring is held by `scripts/tests/test_ci_release_path_scan_job.py` — `build` depends on it, it
|
||||
carries **no job-level `if:`** (one that excluded the tag push would restore the hole; one that
|
||||
skipped the job would skip `build` too), no step is `continue-on-error`, and every one of its own
|
||||
`run:` bodies is delimiter-free. Its steps also carry #756 markers and a trailing assert, so a drop
|
||||
*inside this job* is caught as well.
|
||||
|
||||
The gate's guarantee is one **execution probe**, not a set of assertions about the command: it runs
|
||||
the scan step's real `run:` body — with the workflow, job and step `env:` tiers layered as the
|
||||
runner would — against a copy of the repo whose `Smoke` body carries an injected delimiter, and
|
||||
requires a non-zero exit, with a clean-tree negative control. Shape assertions were tried twice and
|
||||
lost twice (nine disarms across two rounds, from `echo`ing the command to `PYTEST_ADDOPTS` one env
|
||||
tier up), so **do not replace the probe with cheaper checks about the command.** One tier is
|
||||
unreachable by any static reconstruction — a step writing to `$GITHUB_ENV` injects into later steps
|
||||
at runtime — and is therefore **banned** in this job rather than modelled.
|
||||
|
||||
**And one check deliberately is not a test at all.** The tests above prove the ban *holds*; they
|
||||
cannot prove it would *notice*, because pytest's own configuration outranks them — a repo-root
|
||||
`pytest.ini` with `addopts = -k "not delimiter_banned"`, or a `conftest.py` with
|
||||
`pytest_collection_modifyitems`, deselects the ban test and every test guarding it, leaving the
|
||||
whole suite green with a delimiter sitting in `Smoke`. So `scan`'s last working step runs
|
||||
**`scripts/ci-prove-ban-detects.sh`**, which poisons the checked-out workflow, re-runs the ban
|
||||
test, and requires **the ban test's `build` parametrisation to be the failure**. Not "the same
|
||||
command" (the step also runs the wiring tests; the script runs only the ban file) and not merely
|
||||
"fails if it passes": any other outcome — exit 5 because everything was deselected, exit 2 on a
|
||||
collection error, or a failure in some unrelated test — is a **refusal**, not a pass. That
|
||||
exhaustiveness is the load-bearing part, because each weaker reading was a live bug in an earlier
|
||||
draft: "non-zero means it noticed" vouched for the gate on a collection error, and the deselection
|
||||
disarm this script exists to catch exits **5**, not 1.
|
||||
|
||||
It runs in the **real checkout** — an isolated copy does not inherit the repo-root config a disarm
|
||||
would live in, which is exactly how the first version of that script reported healthy while the
|
||||
real invocation was deselected. The workflow file is
|
||||
restored by an `EXIT` trap, and the step carries a `selfcheck` marker so dropping it reddens the
|
||||
job.
|
||||
|
||||
Two further tests exist because review disarmed the gate in ways an obvious test misses. **The
|
||||
pytest invocation is checked as a command, not as a substring**: the ban test's path must appear as
|
||||
a bare token on the real `python3 -m pytest` line, that line may carry none of
|
||||
`-k`/`-m`/`--deselect`/`--ignore`/`--collect-only`, and its exit status may not be swallowed by
|
||||
`||`/`&&`/`;`/`|`. Commenting the invocation out and echoing instead, or appending
|
||||
`-k 'not delimiter_banned'`, each left the suite green with the release path unguarded. And **the
|
||||
ban's SCOPE is pinned** — `DELIMITER_BAN_JOBS` must still contain `build`, because dropping that
|
||||
one literal makes `scan` pass while checking nothing about the job that publishes the image.
|
||||
|
||||
Two operational notes. `scan` is a `small`-lane job on the tag path, which ersatztv#535 moved away
|
||||
from after such jobs wedged in act's setup phase on the v26.12.0 tag; that is safe again only
|
||||
because server-management#639 took `build` off the lane. If it ever returns, move `scan` to
|
||||
`ubuntu-latest` — never weaken the `needs:` edge. And `scan` installs from PyPI, so a network blip
|
||||
now sits between a `v*` tag and its image; fail-closed and loud, but real.
|
||||
|
||||
What this does **not** claim: that no step can ever fail to run for a reason other than the
|
||||
interpolation drop. It moves the terminal assumption — to fail open you must now drop the pytest
|
||||
step **and** the assert step, rather than either one alone.
|
||||
|
||||
Measuring a guard on this path does **not** require cutting a release, and an earlier draft here
|
||||
claiming it would was simply wrong: `build` runs on every push to `main`
|
||||
(`if: github.event_name != 'pull_request'`), and a `workflow_dispatch` on any other ref runs the
|
||||
job while `Build and push` publishes nothing (its `push:` is gated on `main`/`v*`). That is how
|
||||
#767 was verified — see the decision record for the run ids.
|
||||
|
||||
`functional-e2e` is delimiter-free too but is deliberately **not** banned: it is
|
||||
advisory by declaration, and the rule is "ban where a drop is consequential", not "ban wherever it
|
||||
is currently free". `api-docs` and `format` keep one `github.base_ref` each in a detect step and
|
||||
gate nothing that ships.
|
||||
- **Runtime per-step markers**, for a step that fails to run for any *other* reason. Every `run:`
|
||||
step that is not `continue-on-error: true` calls
|
||||
`"$GITHUB_WORKSPACE/scripts/ci-step-ran.sh" mark <key>` as its **first act**, and the job's last
|
||||
step calls `ci-step-ran.sh assert --always … --gated …`, which fails the job when an expected key
|
||||
was never recorded.
|
||||
|
||||
**Per step, not per job.** A marker written by the first step only proves the job *began*, which was
|
||||
never in doubt. The drop that costs something is `Test`, `Build` or a migration replay — all well
|
||||
past step one — so a job-level marker would have been a guard that cannot see the case it exists for.
|
||||
|
||||
**The guard carries no `if:`, and that is deliberate.** The #751 guard uses `if: always()` because its
|
||||
job has one real step. These have a dozen, and a genuine failure in an early step legitimately skips
|
||||
every later one — an `always()` guard would then announce a false *"these steps never executed:
|
||||
typecheck web-test build dotnet-test"* on top of every ordinary red build, and a guard that cries wolf
|
||||
gets deleted. (`migrations` is smaller — six marked steps — but the same argument applies, and its
|
||||
guard comment is worded for its own keys rather than copied from `test`'s.) The default `if:` is `success()`, which is the wanted condition, and the invariant that
|
||||
makes relying on it safe rather than lucky is: **the guard is skipped only when an earlier step
|
||||
failed, and that already fails the job**. So *guard skipped ⇒ job red*, and every path to a green job
|
||||
runs the guard. A dropped step is invisible precisely *because* it concludes `success` — which keeps
|
||||
the job green and therefore reaches the guard.
|
||||
|
||||
That invariant has one path where it could plausibly be false and where being wrong would be silent:
|
||||
a step marked `continue-on-error: true` that FAILS. If that flipped `success()`, the guard would be
|
||||
skipped on a job that still concluded green — the guard rendered a no-op by exactly the failure mode
|
||||
it exists to catch, with no signal. The `test` job has three `continue-on-error` steps and two of them sit immediately before the guard,
|
||||
so this is a live path, not a theoretical one. **Measured** (scratch PR #766, run 1913 job 8075): the last
|
||||
advisory step was made to `exit 1`, the log carries `❌ Failure - Main Report peak container memory`,
|
||||
and the guard **still ran**, reported `All 12 expected step(s) executed`, and the job concluded
|
||||
`success`. A failing `continue-on-error` step does not flip `success()` on this runner, so the
|
||||
invariant holds where it mattered most. That run is also the `test` job's full twelve-key positive
|
||||
control on the build lane.
|
||||
|
||||
**Adding a step to either job?** Mark it, and add its key to that job's guard list in the right
|
||||
bucket (`--always` for the two detect steps, `--gated` for anything carrying the docs-only /
|
||||
already-validated `if:`). `scripts/tests/test_ci_dropped_step_guard.py` derives the expected set from
|
||||
the workflow, so an unmarked step or a bucket mismatch is a red — it does not rely on anyone
|
||||
remembering. One caveat, since this section is careful about it elsewhere: that red is `script-tests`,
|
||||
the same non-required, PR-only check discussed above. For the delimiter ban on `test`/`migrations`
|
||||
that hardly matters, because the runtime guard is the fail-closed backstop — but a **newly added,
|
||||
unmarked** step is caught by the static test *alone*, since the runtime guard cannot expect a key
|
||||
nobody declared.
|
||||
|
||||
**The marker path is keyed on job + run id + attempt** — and be precise about why, because the
|
||||
obvious justification is a #751 measurement that does *not* transfer. #751 found `RUNNER_TEMP` to be
|
||||
`/tmp` and called it "not a private per-job directory", but that was taken on `review-verdict.yml`,
|
||||
which runs *without* a `container:`. These two jobs run **inside** the CI toolchain image, so their
|
||||
`/tmp` is the job container's own and starts empty. The fresh container is therefore what actually
|
||||
rules out a stale marker here; the keying is defence in depth against a lane change nobody would
|
||||
think to re-check this against. `GITHUB_JOB` and `GITHUB_RUN_ID` are *measured* present and the
|
||||
script refuses without them rather than falling back to a name other runs share.
|
||||
`GITHUB_RUN_ATTEMPT` is required too — but **how** that was established is the part worth keeping,
|
||||
because the first two attempts at it were both worthless. Grepping a job log for the variable *name*
|
||||
proves nothing: logs do not dump the environment. Inferring it from the *absence* of the script's
|
||||
"not set" warning proves nothing either, because that warning goes to **stderr**, and whether step
|
||||
stderr reaches a job log here was itself never established — the control offered for that turned out
|
||||
to be an `::error::` line this script writes to *stdout*. So the script was made to **report its
|
||||
resolved identity on stdout**, where capture is not in question, and the answer was simply read off
|
||||
this change's own run: `Marker identity: job=test run=1916 attempt=1 (from the runner)`, and the same
|
||||
for `migrations`. Both required jobs, on the lane that matters.
|
||||
|
||||
That measurement is what promoted it from warn-and-default to required, and it is why the residual
|
||||
this paragraph used to describe — a rerun inheriting attempt 1's markers — no longer exists. The
|
||||
identity line stays, as the standing evidence a future reader checks first if the keying is ever
|
||||
doubted again.
|
||||
|
||||
**The premise was re-measured on the build lane.** The whole thing rests on the runner still executing
|
||||
a later step after dropping an earlier one. #751 established that on the `small` lane; these jobs run
|
||||
in a `container:` on `ubuntu-latest`, so it was measured there rather than assumed — scratch PR #765
|
||||
(Gitea 1.27.1, 2026-08-10) reintroduced the exact #751 defect in the `test` job's `revalidate` step.
|
||||
Recorded outcome (job `test`, run 1910, 20:03:49→20:13:31Z — a full 9m42s heavy run, so `Build` and
|
||||
`Test` really executed):
|
||||
|
||||
- `Unable to interpolate expression 'format('# PROBE ONLY … {0}\n…', pr number)'` at 20:04:06 — the
|
||||
step was **dropped**, exactly as #751 describes, and it reported conclusion `success`.
|
||||
- **Every other marked step still ran** — eleven markers were recorded, ten of them AFTER the drop
|
||||
(`restore npm-ci check-api lint typecheck web-test web-build strip-scanner build dotnet-test`),
|
||||
`detect` being the eleventh and earlier. The premise holds on this lane.
|
||||
- The guard ran at 20:13:29, reported `These steps of job 'test' never executed: revalidate`, and was
|
||||
the **only** ❌ in the entire job log — every other step succeeded. Without it this run would have
|
||||
concluded `success` having never executed that step, which is precisely the fail-open being closed.
|
||||
- Incidental but kept: the dropped step's output arrived as `ETV_REVALIDATE_SKIP:` **empty**, not
|
||||
`false` — the case the guard must read as "widen what is required", never as a skip.
|
||||
|
||||
**The positive control is the same run's `migrations` job**, which the probe did not touch: it marked
|
||||
all six steps, the guard reported `All 6 expected step(s) executed: detect revalidate restore build
|
||||
sqlite mysql`, and the job concluded **success**. So one run demonstrates both directions on the build
|
||||
lane — a drop caught and reddened, and a clean job passing. The twelve-step `test` positive control is
|
||||
this change's own CI run.
|
||||
|
||||
Full rationale: `docs/decisions/records/ci/required-job-step-execution-markers.md`.
|
||||
|
||||
### `docs-reminder` job (non-blocking, PR-only — in `pr-checks.yml`)
|
||||
|
||||
A lightweight nudge that enforces the CLAUDE.md "docs-update is part of done" rule for the
|
||||
Two lightweight nudges, both `::warning::`-only. Neither can fail the build — they are reminders,
|
||||
not gates; prose-doc gates get gamed with token edits.
|
||||
|
||||
**1. The parity-doc reminder** enforces the CLAUDE.md "docs-update is part of done" rule for the
|
||||
one case that's easy to forget and easy to detect: a PR that touches a SPA screen
|
||||
(`web/src/screens/*.tsx`) or `ErsatzTV/LegacyUiRedirects.cs` but **does not** update
|
||||
`docs/blazor-route-parity.md`. It diffs the PR against its base branch and emits a
|
||||
`::warning::` annotation (never fails the build — it's a reminder, not a gate; prose-doc
|
||||
gates get gamed with token edits). Deliberately has **no** `setup-dotnet`/`setup-node` (and
|
||||
`::warning::` annotation.
|
||||
|
||||
**2. The session-narrative reminder** (ersatztv#784) runs `scripts/check-doc-narrative.py --diff`
|
||||
over the lines this PR **adds** to `docs/**/*.md` (minus `docs/decisions/**`, exempt wholesale) and
|
||||
root-level `*.md`, flagging text that narrates the document's own revision history —
|
||||
`docs.no-session-narrative`. It is advisory **by design and permanently**: a narrative detector is a
|
||||
string predicate over prose, the class `docs/defect-shapes-773.md` §4 argues must never be
|
||||
load-bearing, so the script exits 0 on every path including a bad argument or an unresolvable base
|
||||
ref. It does NOT rest on prose alone, and it does not rest wholly on tests either: the record's
|
||||
COVERAGE BOUNDARY names which clauses carry a mutation proof and which are defensive and unproven
|
||||
(the unhandled-exception arm among them). Every argument shape is asserted per shape in
|
||||
`scripts/tests/test_check_doc_narrative.py` rather than only in prose. When it cannot resolve the
|
||||
base it prints `SCANNED NOTHING` instead of a clean-looking line, because a silent zero-file scan is
|
||||
indistinguishable from a clean one — the same failure `ci.required-job-step-execution-markers`
|
||||
exists for.
|
||||
|
||||
Because this step needs an interpreter, the job DOES carry `actions/setup-python` — the one
|
||||
exception to the no-setup-actions note below, since `python3` is not guaranteed on the bare `small`
|
||||
lane. Both it and the script step carry `continue-on-error: true`: a script that exits 0 does not by
|
||||
itself keep the JOB green, and a setup-action download failure would redden an advisory check just
|
||||
as effectively as a hit would.
|
||||
|
||||
The job deliberately has **no** `setup-dotnet`/`setup-node` (and
|
||||
thus no `actions/cache`), so it can't hit the cache-save hangs seen on the VM-127 runner
|
||||
(server-management#570). It does not cover the remaining doc obligations in the CLAUDE.md table
|
||||
(domain-model, spa-conventions) — those stay on the author. (The API contract is mechanized by the
|
||||
@@ -635,14 +898,44 @@ compiler/docker build), so it doesn't violate the "small is git-only" lane rule.
|
||||
`docs-reminder`, otherwise a seconds-long `git diff` + parse with no dotnet/node setup
|
||||
(`runs-on: small`).
|
||||
|
||||
### `script-tests` job (`Script tests (pytest)`, PR-only — in `pr-checks.yml`)
|
||||
### `prove-fix` job (`Fix proofs (Proves trailers)`, PR-only — in `pr-checks.yml`)
|
||||
|
||||
Runs `scripts/prove-fix.sh` for **every commit in the PR that carries a `Proves: <pytest selector>`
|
||||
trailer**, and fails the PR if a claimed proof does not hold. The rule and its rationale are
|
||||
`testing.fix-ships-a-witnessed-red-test`; this section is the CI-side contract.
|
||||
|
||||
**Opt-in by trailer, enforced when present.** Requiring `Proves:` on every commit would block docs,
|
||||
CI and refactor commits that have no code side to revert, and a gate that blocks ordinary work gets
|
||||
switched off — which is how a check ends up running nowhere (ersatztv#631). So the trailer is the
|
||||
author's *claim* and this job checks claims. **When a PR carries none, the job emits a `::notice::`
|
||||
saying it asserted nothing** — its green must not be read as fix coverage.
|
||||
|
||||
**It needs full history** (`fetch-depth: 0`): the script reverts each commit against its PARENT, and
|
||||
a shallow clone cannot resolve `<sha>^`.
|
||||
|
||||
Three refusals worth knowing before you write a trailer:
|
||||
- a **merge** commit is rejected — several parents means "the code before this change" is ambiguous;
|
||||
put the trailer on the commit carrying the fix;
|
||||
- **more than one** `Proves:` trailer is rejected — only the first would be checked, so the rest
|
||||
would read as proven without ever running (the job checks this itself, because passing the
|
||||
selector explicitly bypasses the script's own guard);
|
||||
- a **test-only** commit is rejected — there is no code side to revert.
|
||||
|
||||
**Only pytest exit 1 counts as red.** 2 is an interrupted collection, 3 internal, 4 usage, 5 nothing
|
||||
collected, 143 a signal; `--continue-on-collection-errors` converts a genuine collection failure to
|
||||
1 so add-a-file fixes stay provable. The job inherits that. Note the direction: a wobble DOES redden
|
||||
this job (`prove-fix.sh` exits 5 on a harness/git failure or a signal, and the job turns that into an
|
||||
error), so what the exit-code discipline buys is the other way round — a **green** here means a claim
|
||||
was witnessed, never that a run was cancelled or broke.
|
||||
|
||||
### `script-tests` job (`Script lint and tests (ruff + pytest)`, PR-only — in `pr-checks.yml`)
|
||||
|
||||
> Reddens the run on failure, but like the other `pr-checks.yml` gates it is **not** one of the
|
||||
> three required status checks on `main` (`Build & test (.NET)`, `EF migration integrity`,
|
||||
> `review-verdict/h10`). Promoting it to required is a branch-protection change, tracked separately.
|
||||
|
||||
Runs the repository's Python test suite: `PYTHONPATH=. python3 -m pytest scripts/tests -q`
|
||||
(~190 tests at time of writing, ~10s; the suite grows, so treat the figure as indicative). It covers the decision-corpus parser/validator/catalog builder, the ersatztv#610
|
||||
(773 tests at `706674272`, ~4.5 min; the suite grows fast — it was ~190 tests / ~10s when this job landed in #631 — so treat the figure as a dated snapshot, not a budget). It covers the decision-corpus parser/validator/catalog builder, the ersatztv#610
|
||||
migration-equivalence harness, the merge-consent exemption logic and the ersatztv#622 review-verdict
|
||||
poster.
|
||||
|
||||
@@ -663,7 +956,33 @@ name keeps a real failure unambiguous.
|
||||
input set spans more than one directory — `test_post_review_verdict.py` and
|
||||
`test_merge_consent_exemption.py` execute the real `scripts/post-review-verdict.sh` and
|
||||
`.claude/hooks/pretooluse-merge-consent.sh` — so a `scripts/**` filter would silently miss a
|
||||
`.claude/hooks/**` edit. At ~10s, a filter buys nothing but drift.
|
||||
`.claude/hooks/**` edit. The reason is the input set, not the cost: the suite was ~10s when that was
|
||||
decided and is now ~4.5 min, and it would still be wrong to filter on `scripts/**`.
|
||||
|
||||
**It also lints (ersatztv#780).** Early in the job it installs a **pinned** `ruff==0.12.11` and runs
|
||||
`ruff check` and `ruff format --check` against the repo-root `ruff.toml`. Five things are deliberate:
|
||||
|
||||
- The config is **committed**. Without it ruff falls back to whatever `~/.config/ruff/ruff.toml` the
|
||||
operator's machine has, so a second machine lints this repo differently or not at all.
|
||||
- The version is **pinned** — an unpinned install makes the verdict a function of when the job ran,
|
||||
the same divergence one layer up, and the same argument as the `jq` pin below. `pytest`/`pyyaml`
|
||||
stay unpinned on purpose: a pytest release does not add assertions to your suite, a ruff release
|
||||
adds rules to your lint.
|
||||
- Lint runs **before the jq preflight**, and after the `git` one. `Preflight jq version` is a hard
|
||||
`--expect` tripwire; a lint sitting behind it goes dark for as long as the jq contract is broken,
|
||||
under a red that says "jq". `Preflight external tools` stays ahead, because the lint steps consume
|
||||
`git` — without it a missing git reaches them as an empty population and they blame the glob.
|
||||
- Neither step is `ruff check .`. Both pass an **explicit population** from
|
||||
`git ls-files -z '*.py' '*.pyi' '*.ipynb'` with `--no-force-exclude`, and fail if that list is
|
||||
empty. Discovery-based invocation is silently emptied by an `exclude` in the right config scope —
|
||||
top level empties both commands, `[lint]` empties `check`, `[format]` empties `format --check`
|
||||
(and `[format]` is where an appended line lands) — and
|
||||
`ruff check .` over zero files exits **0** with only a stderr warning, so the failure mode is a
|
||||
green gate. The measured matrix is in `ci.python-lint-ruff-config-committed`.
|
||||
- `RUF100` is selected, so a `# noqa` that no longer matches anything is itself a finding.
|
||||
|
||||
`pyright` is not gated; the reasoning and the exemption list are in
|
||||
`ci.python-lint-ruff-config-committed`.
|
||||
|
||||
**Dependencies: `pytest` and `pyyaml`** — the complete third-party set across `scripts/`, established
|
||||
by an AST import scan rather than by reading the files that looked relevant. PyYAML does **not**
|
||||
@@ -728,7 +1047,7 @@ assumed.
|
||||
|
||||
**File:** `.gitea/workflows/pr-checks.yml` — `on: pull_request` only.
|
||||
|
||||
The four git-only PR gates — `ci-image-pin`, `docs-reminder`, `decisions-guard`, `script-tests`
|
||||
The five PR gates — `ci-image-pin`, `docs-reminder`, `decisions-guard`, `script-tests`, and `prove-fix` (the one member that is NOT merely checkout + `git diff`: it installs pytest and creates worktrees)
|
||||
(all described above) — live here, **not** in `docker-build.yml`, and that separation is the fix
|
||||
for **ersatztv#535**.
|
||||
|
||||
@@ -765,6 +1084,40 @@ from `Build ErsatzTV Image / …` to `PR Gates / …`) does not affect merges. T
|
||||
unreviewed commit from merging** (ersatztv#622). It is not produced by a job's success/failure; it
|
||||
is a commit status that `scripts/post-review-verdict.sh` POSTs onto one specific sha.
|
||||
|
||||
**`main` is PR-only AND admin-override-proof, and it takes both to make the check load-bearing**
|
||||
(ersatztv#743, `release.main-direct-push-disabled`). Gitea evaluates `status_check_contexts` when it
|
||||
**merges a PR** — a direct `git push origin HEAD:main` never consults them. So until 2026-08-05 the
|
||||
entire gate was skippable with no forgery at all, which was cheaper than every route enumerated in
|
||||
#697. `main` now carries **two** fields, and citing either alone is a mistake:
|
||||
|
||||
- `enable_push: false` — a direct push is refused server-side at pre-receive (`Not allowed to push to
|
||||
protected branch main`), for every account including a site admin. The contents API is refused too
|
||||
— measured, HTTP 403 `user cannot commit to repo`. The web editor, upload, apply-patch, revert and
|
||||
cherry-pick paths share that same `CanUserPush` predicate and are therefore expected to refuse as
|
||||
well, but were not probed (source-attested only).
|
||||
- `block_admin_merge_override: true` — without it (the default is `false`), a repo admin could
|
||||
`POST /pulls/{n}/merge` with `force_merge: true` and merge straight past a missing or red
|
||||
`review-verdict/h10`. Disabling push alone just moves the bypass from the push path to the merge
|
||||
path, since `timothy` is admin and is the identity every session already uses. **Source-attested,
|
||||
not probed** (Gitea 1.27 `CanBypassBranchProtection`): verifying it by experiment means merging an
|
||||
unreviewed PR, so the field was set rather than measured. Setting it is safe under either
|
||||
semantics; re-confirming the bypass itself rides with ersatztv#747.
|
||||
|
||||
**Operator recovery when a required context gets stuck.** `block_admin_merge_override: true` removes
|
||||
the "Merge (admin)" / `force_merge: true` escape that used to unstick a PR whose required context was
|
||||
absent or wrongly red — a recurring situation here (a killed run overwriting a newer green, an
|
||||
advisory red counted into the combined status, a gate workflow that cannot post). That escape is gone
|
||||
*by design*: it was also the bypass. The supported recovery is to fix the status
|
||||
(re-run the job, or re-post the verdict with `scripts/post-review-verdict.sh`); the last resort is to
|
||||
`PATCH .../branch_protections/main` setting `block_admin_merge_override: false`, merge, and set it
|
||||
straight back. Do the last one deliberately and say so in the PR — it is the one action that
|
||||
re-opens the hole this section exists to close.
|
||||
|
||||
Practical consequences: **every** change to `main` goes through a PR, including a one-line docs fix;
|
||||
and the client-side Husky guards (H6/H11/H13) remain useful friction but were never the control —
|
||||
they are fail-open and `--no-verify` bypasses them. Tag pushes are unaffected (separate mechanism;
|
||||
`tag_protections` is empty), so the release cut in "Cutting a release" still works unchanged.
|
||||
|
||||
**The hole it closes.** `pretooluse-merge-consent.sh` proves its three consent conditions at the
|
||||
moment the merge tool is called. Pass `merge_when_checks_succeed=true` and Gitea performs the merge
|
||||
*later*, against whatever head is green then — while the Done-when and review-verdict checks were
|
||||
@@ -834,7 +1187,18 @@ as a human verdict — a non-null `.creator.login` **and** a `Review-verdict:` d
|
||||
**re-derived** rather than inherited. (Measured: a status POSTed with a user credential carries a
|
||||
creator; one POSTed by an Actions job carries `"creator": null`.) Without this, an exemption obtained
|
||||
once was accepted unchanged on every later run. This is a *provenance* check, not an authentication
|
||||
one — someone who can POST statuses directly can still impersonate a verdict, which is ersatztv#697.
|
||||
one — someone who can POST statuses directly can still impersonate a verdict (ersatztv#697). That
|
||||
provenance asymmetry is *why* the credential scoping in `ci.actions-credential-scoping` mattered: a
|
||||
forgery through a **user** credential inherits as a human verdict, while one through a job's
|
||||
`GITEA_TOKEN` carries `creator: null` and is re-derived, so it must win a race. CI's registry secret
|
||||
was a user credential — the admin account — and no longer carries status-write. **`RENOVATE_TOKEN`
|
||||
still is one** (`write:repository`, a real bot account), and secrets are a per-repo store any
|
||||
PR-added workflow can reference, so that route is narrowed rather than closed; tightening this check
|
||||
from "non-null creator" to an allow-list of approved reviewers is what would close it
|
||||
(ersatztv#742). A collaborator's own personal token still can, and no repo-side change closes that.
|
||||
Note also that re-derivation is **not** a race the attacker can lose: it fires only on the trigger's
|
||||
`types`, and posting a status is not one of them, so a POST timed after the last PR event stands
|
||||
until the next one.
|
||||
|
||||
Deciding either exemption requires the PR's **complete** changed-file list, which the workflow does
|
||||
not compute itself: it calls `scripts/pr-changed-files.sh`, the single shared implementation also
|
||||
@@ -901,12 +1265,24 @@ as establishing that the gate cannot be forged (see the residual below, and ersa
|
||||
here **only** because this job never checks out or executes head-supplied code. Verified on this
|
||||
instance with four scratch PRs rather than inferred from GitHub; full rationale in
|
||||
`docs/decisions/records/ci/gate-trigger-base-resolved.md`. **This closes the rewrite route through
|
||||
this workflow, not the class:** `docker-build.yml` is also head-resolved and its `ETV_STATUS_AUTH`
|
||||
credentials can write statuses, so it can still forge `review-verdict/h10` — it must stay on
|
||||
`pull_request` because it builds the PR's code, so it needs a read-only status identity instead
|
||||
(ersatztv#697) — and the inventory is every workflow, not that one, because Gitea injects a
|
||||
write-capable `GITEA_TOKEN` into every job and branch protection binds the *context*, not its
|
||||
issuer. The exemption path has separate defects of its own (ersatztv#698). One operational
|
||||
this workflow, not the class:** `docker-build.yml` is also head-resolved and must stay on
|
||||
`pull_request` because it builds the PR's code, so it got the read-only status identity instead —
|
||||
its `ETV_STATUS_AUTH` is now a PAT scoped `write:package` + `read:repository`, which the status
|
||||
endpoint refuses (`ci.actions-credential-scoping`, ersatztv#697). The inventory was never that one
|
||||
workflow, though: Gitea injects a write-capable `GITEA_TOKEN` into every job and branch protection
|
||||
binds the *context*, not its issuer. Gitea >=1.26 with the Actions default set to **Restricted**
|
||||
(server-management#714) binds the injected token, but does not close the class either — not against
|
||||
a personal token, and not against `RENOVATE_TOKEN` (ersatztv#742). **And none of it was necessary:
|
||||
direct pushes to `main` were server-side permitted, so the gate could be skipped without any forgery
|
||||
(ersatztv#743). That is now CLOSED — `main` carries `enable_push: false` **and**
|
||||
`block_admin_merge_override: true`, so it is reachable only through the PR merge path, the one path
|
||||
on which Gitea evaluates `status_check_contexts`, and an admin cannot `force_merge` past them
|
||||
(`release.main-direct-push-disabled` — neither field is citable alone).** Note the fix is *disabling* push, not whitelisting it: a
|
||||
push whitelist naming `timothy` was measured to still admit the push, and `timothy` is the identity
|
||||
every session, PAT and injected `GITEA_TOKEN` already acts as, so the whitelist form would have
|
||||
closed nothing. The block binds a site admin at pre-receive but not a credential that can first
|
||||
PATCH branch protection off — an accepted residual, recorded in that decision. The
|
||||
exemption path has separate defects of its own (ersatztv#698). One operational
|
||||
consequence of the trigger change: a PR whose base is not `main` now gets **no**
|
||||
`review-verdict/h10` at all. That is fail-closed. `edited` **is** now among the trigger's `types`
|
||||
(ersatztv#698), so a PR retargeted onto `main` reclassifies instead of staying statusless until its
|
||||
@@ -937,7 +1313,153 @@ trust the editing PR's own checks. Verify the way ersatztv#672 did:
|
||||
branches.
|
||||
|
||||
The same shape is what makes a `branches:`/`types:` change verifiable at all, since neither can be
|
||||
observed from the editing PR.
|
||||
observed from the editing PR. Note step 2 requires the scratch **base**'s own `branches:` filter to
|
||||
name that base — the definition comes from the base, so a base the filter does not admit produces no
|
||||
run at all.
|
||||
|
||||
⚠️ **Never write an expression delimiter inside a `run:` body here — a comment is NOT inert**
|
||||
(ersatztv#751, `ci.workflow-run-body-no-expressions`). A `run:` body is not shell when the runner
|
||||
reads it. The runner scans the whole scalar for the expression opener and, on finding one, rewrites
|
||||
the **entire** body into a single `format(...)` call so the result can be spliced back in. That
|
||||
rewrite is all-or-nothing: a payload that does not evaluate fails the interpolation of the whole
|
||||
scalar, and **the runner then drops the step and concludes the job `success`**.
|
||||
|
||||
That is not hypothetical. From 8f6d4f443 (2026-08-03) to 2026-08-06 the classify step **never ran**.
|
||||
The #706 note above, explaining why a concurrency group does not work here, quoted a `concurrency:`
|
||||
snippet containing a PR-number expression *as an illustration*, in a shell comment. `pr number` is not
|
||||
a valid expression. So `review-verdict/h10` was posted by nothing but a human hand for three days,
|
||||
both exemption classes silently stopped working, and every run reported success. The prose documenting
|
||||
a fix disabled the fix.
|
||||
|
||||
**The silent green is the real defect.** An absent required status reads as "not reviewed yet", which
|
||||
is indistinguishable from the correct pending state — so an ordinary PR looked ordinary while the gate
|
||||
was dead, and the cost landed only where no human was in the loop. PR #739 (docs-only) merged
|
||||
2026-08-05 with **zero** commit statuses on its head, and got in only because admin force-merge was
|
||||
still enabled; ersatztv#743 removed that escape the next day, so a docs-only or Renovate-manifest PR
|
||||
arriving after that would simply have been stuck with no bypass. The two Renovate PRs in the window
|
||||
escaped by timing, merging minutes before the bad commit.
|
||||
|
||||
Three things now hold the line, and they are deliberately different in kind:
|
||||
|
||||
- **The prose names expressions instead of quoting them** — write "a
|
||||
`github.event.pull_request.number` expression", not the delimiters. Pass values in through the
|
||||
step's `env:` block, which is interpolated per value, so a bad payload there cannot take the body
|
||||
with it.
|
||||
- **A start-marker guard turns a dropped step RED.** The classifier writes a marker as its first act
|
||||
and an `if: always()` step fails the job when it is missing. It asserts execution *started*, never
|
||||
that it completed — the classifier has several legitimate `exit 0` abstention paths. The guard's own
|
||||
body must stay expression-free, or the mechanism it guards against can delete the guard too, and
|
||||
that absence would be silent as well.
|
||||
- **Two static guards**, in `scripts/tests/test_pr_changed_files.py`: no delimiter in *any* `run:`
|
||||
body of this file (absolute — a dropped step here is a dead merge gate, and its bodies are ~700
|
||||
lines of prose), and repo-wide, every expression payload's **head token** must name a context or
|
||||
function the runner can resolve (permissive, because the other workflows interpolate into `run:`
|
||||
legitimately — 5 occurrences today, in `ci-image.yml`, `docker-build.yml`'s `api-docs`/`format`
|
||||
and `pr-checks.yml`'s two git-diff gates; #756 removed `build`'s two and banned that job as
|
||||
well, so the ban now covers `test`, `migrations` and `build`). Be precise about the second
|
||||
one's reach: it catches the
|
||||
historical defect (`pr number`) and a nonexistent context, but **not** a syntactically invalid
|
||||
payload whose tokens are all known (`${{ github.ref == }}` passes), nor a renamed output
|
||||
(`steps.metadata.outputs.shortsha` passes — every token after the first is preceded by `.` and is
|
||||
skipped), nor an unclosed opener. Catching those needs an expression parser. An earlier draft of
|
||||
this section claimed it caught "a payload that cannot evaluate, wherever it sits"; that was false,
|
||||
and the corrected claim is the one to rely on.
|
||||
|
||||
Worth knowing why nothing caught this for three days: every *other* workflow-shape test in that file
|
||||
reads `_code_lines()`, which strips comments. That is correct for what it was for, but it encodes the
|
||||
assumption this bug falsifies. The strict test reads the raw scalar, and must never adopt
|
||||
`_code_lines`.
|
||||
|
||||
⚠️ **A page past the end of `/issues/{n}/timeline` is JSON `null`, not `[]`** — and this instance is
|
||||
not consistent between endpoints (`/issues/{n}/comments` returns `[]` when empty). The retarget
|
||||
fence's `count_retargets` gated on `type == "array"`, so it read the real terminator as *unreadable*:
|
||||
the walk never reached a validated empty page, `rt_ok` was never `yes` for **any** PR, and the fence
|
||||
therefore withheld **every** exemption `success`. Renovate and docs-only PRs got no status at all —
|
||||
the same user-visible outcome as the dropped step above, by a completely unrelated route. So fixing
|
||||
the interpolation alone would not have restored the exemptions.
|
||||
|
||||
Two things kept it invisible, and both are worth generalising:
|
||||
|
||||
- It shipped in the **same commit** (8f6d4f443) that stopped the step executing, so the fence had
|
||||
never once run in production. A guard's first real execution is not the same event as its merge.
|
||||
- The **test double asserted the wrong shape while claiming measured fidelity.** Its comment read
|
||||
"Real shapes, measured on this instance and deliberately mirrored" and it printed `[]` for a page
|
||||
past the end. Every fence test was green against a response the server never produces, so the
|
||||
`array`-only gate was never exercised by the suite either. With the double corrected and the old
|
||||
gate restored, **most of the fence suite fails** — 18 tests when first measured at `c710db4a1`, 21
|
||||
once three more fence-dependent tests existed. The invariant is the point, not the count: they had
|
||||
all been passing for the wrong reason. (Given as a range on purpose — an earlier draft cited a bare
|
||||
"18", which was stale two commits later, inside a section about stale claims.) When a double claims
|
||||
fidelity, that claim is a test assertion and needs re-measuring like any other.
|
||||
|
||||
The type is now read as a value (`case` over `jq -r 'type'`) rather than through `jq -e`, whose
|
||||
exit-status semantics already bit this workflow once at jq 1.6, and both `null` and `[]` terminate the
|
||||
walk. The regression test is parameterised over both shapes because both are live on this server.
|
||||
`null` is accepted as exhaustion only from **page 2 on** — every real PR's first page carries events
|
||||
(spot-checked non-empty across #752/#753/#749/#739/#717; the counts are deliberately not recorded here
|
||||
because timelines grow and an earlier draft's five figures were stale within days), so a `null` first
|
||||
page is anomalous rather
|
||||
than empty, and the walk should not certify "no retarget happened" from a response it cannot explain.
|
||||
|
||||
**The same nil-slice shape bites `/commits/{sha}/status`** — a third instance, found by cold review of
|
||||
the fix for the second. A head with no statuses yet returns
|
||||
`{"state":"pending","total_count":0,"statuses":null}` (measured on PR #739's head). `read_existing_verdict`
|
||||
gated on `.statuses | type == "array"`, so it hit its `exit 1` and posted nothing at all — fail-closed,
|
||||
same user-visible outcome. `null` is now accepted there only when `total_count` is 0, so a body that
|
||||
merely lost its array is still refused and an existing verdict is still protected from a transient
|
||||
error. `scripts/pr-changed-files.sh` was swept and is unaffected (`pulls/{n}/files` returns `[]`).
|
||||
**The generalisable rule: a nil Go slice serialises to `null`, so every list-shaped field on this API
|
||||
is suspect and only a per-endpoint measurement settles it.**
|
||||
|
||||
**Establishing that "no verdict exists" needs a second page, and both arithmetic guards for it are
|
||||
no-ops here.** `read_existing_verdict` concluding absence is what licenses posting an exemption over a
|
||||
verdict the job cannot see, so that conclusion has to be earned. Two obvious checks were tried and both
|
||||
proved empty:
|
||||
|
||||
- **`.statuses | length` vs `.total_count`** — `total_count` is the count for the **page returned**, not
|
||||
for the commit. Measured at 1.27.1 on `3aed43c6` (6 contexts): `?limit=1` returns
|
||||
`len=1, total_count=1`, `?limit=3` returns `len=3, total_count=3`. Equal by construction, so the check
|
||||
reads as a completeness proof while proving nothing.
|
||||
- **"refuse when the page comes back full at the requested `limit=100`"** — this instance caps `limit`
|
||||
at the server-wide `MAX_RESPONSE_ITEMS`, **measured at 50** (`/issues?limit=100` returns 50). A
|
||||
response can therefore never carry 100 rows, and the comparison was **dead code**. The repo already
|
||||
documented that cap in `scripts/pr-changed-files.sh`, two test files and `ci.script-tests-job`; the
|
||||
guard was written against 100 anyway, and a cold review caught it. Hardcoding 50 instead would
|
||||
re-break the day the setting changes.
|
||||
|
||||
So the job **asks the server, and only when it matters**: if the `review-verdict/h10` row is on page 1
|
||||
there is nothing further to learn (this endpoint returns the latest status per *context*, and a context
|
||||
cannot recur on a later page). When the row is absent it reads **page 2** — any rows there mean the list
|
||||
runs longer than one page and a verdict could be beyond it, so it refuses instead of concluding absence.
|
||||
Cap-independent by construction. Paging is real here: measured `?limit=3&page=2` returning three further
|
||||
rows, and `page=9` returning the same `statuses: null` terminator.
|
||||
|
||||
The `total_count` zero-check also requires the JSON **type** to be a number: `jq -r` renders `0` and
|
||||
`"0"` identically, so a text compare would accept a schema-corrupted `"total_count": "0"` as "no
|
||||
statuses".
|
||||
|
||||
**The repo-wide expression guard scans PARSED scalars, not raw file text.** A delimiter in an ordinary
|
||||
top-level YAML comment is inert — the runner never evaluates it — so redding on it is a false positive,
|
||||
and this file has now produced that false red twice. PyYAML drops those comments. A `run:` body is
|
||||
itself a scalar and keeps its *shell* comments, which is the point: inside a `run:` scalar a comment is
|
||||
not inert. Verified both directions by mutation — an inert top-level comment passes; the same payload
|
||||
in a run-body comment still reds.
|
||||
|
||||
**`CLAUDE.md` and `AGENTS.md` are now PROTECTED paths.** `DOCS_ONLY` matched them, so the documents
|
||||
that *define* the completion protocol, the merge-consent convention and the H10 rule were themselves
|
||||
docs-only-exemptible while `.claude/` was protected — the same self-exemption the gate rules out, one
|
||||
directory over. Driving the real classify body with a lone `CLAUDE.md` change produced
|
||||
`review-verdict/h10=success`. It is fixed here rather than deferred because restoring the exemptions is
|
||||
what makes it reachable: no exemption `success` was writable at all while the classify step was
|
||||
dropped. `README.md` is deliberately not listed — ordinary prose, no enforcement. For the same reason,
|
||||
#706's known residual returns with the working fence: while `rt_ok` was never `yes`, route 1 was closed
|
||||
by accident.
|
||||
|
||||
**That gap is now closed** — `docker-build.yml`'s `test` and `migrations` jobs are also required
|
||||
contexts, and there a dropped step is **fail-OPEN**: the required check goes green having done no work,
|
||||
which is strictly worse than an absent status (compare #684). ersatztv#756 gave those two jobs
|
||||
per-**step** execution markers and extended the delimiter ban to them; see
|
||||
"Dropped-step guard on the required jobs" above.
|
||||
|
||||
It lives in its **own workflow file** on purpose: `pr-checks.yml` sets `cancel-in-progress: true`,
|
||||
and a cancelled run there would leave an exempt PR with no status and no further push to
|
||||
|
||||
@@ -117,7 +117,7 @@ talks exclusively to the REST API. Every former Blazor route now 302-redirects t
|
||||
`io.open(..., encoding='utf-8-sig')` *writes one back*, and sed/perl round-trips keep it. This cost
|
||||
two sessions a red CI job on 2026-07-17 alone (PR #405, 6 files; PR #402, 19), which is what the
|
||||
guard is for. To check by hand:
|
||||
`for f in $(git diff --name-only origin/main...HEAD --cached -- '*.cs'; git diff --name-only -- '*.cs') ; do head -c3 "$f" | xxd -p | grep -q '^efbbbf' && echo "BOM: $f"; done` (covers branch + staged + dirty — a brand-new staged file is absent from `origin/main...HEAD`)
|
||||
`for f in $(git diff --name-only origin/main...HEAD --cached -- '*.cs'; git diff --name-only -- '*.cs') ; do [ "$(od -A n -t x1 -N 3 < "$f" | tr -d ' \n')" = efbbbf ] && echo "BOM: $f"; done` (covers branch + staged + dirty — a brand-new staged file is absent from `origin/main...HEAD`)
|
||||
(note `dotnet format --include` needs **bash** — `mapfile` is bash-only, and under zsh the file list
|
||||
comes out empty, which looks exactly like the tool silently doing nothing). (A one-time repo-wide normalization
|
||||
is a separate, unmade decision; the touched-file rule is the standing one.)
|
||||
|
||||
@@ -36,17 +36,18 @@ the link for rationale. Superseded/retired history lives in `archive/`. Regenera
|
||||
| `blazor.rollback-tag` | The commit immediately preceding the Blazor-removal merge is tagged `blazor-final` (not a `v*` tag, so it doesn't trigger a prod release build) as the documented rollback/restore path. | 2026-07-11 | [link](records/blazor/rollback-tag.md) |
|
||||
| `blazor.ui-removed` | The legacy Blazor Server UI (`Pages/`, `Shared/`, `ViewModels/`, `Validators/`, MudBlazor + 8 other packages, Blazor Startup wiring) is fully deleted now that the SPA has parity; the legacy `MapWhen` branch is kept only for controllers/docs/OpenAPI/`LegacyUiRedirects`, and the catch-all fallback 302s any unmatched non-api/artwork/docs/openapi path to `/app`. | 2026-07-11 | [link](records/blazor/ui-removed.md) |
|
||||
| `channel.origin-marker` | A new `Channel.Origin` (`ChannelOrigin` enum — `Unknown`/`UserCreated`/`AutoTuned`) records how a channel row was created and is stamped exactly once at insert (`AutoTuned` in `CreateChannelFromLineupHandler`, `UserCreated` in `CreateChannelHandler`), and is never mutated on a later edit. It is surfaced as a raw `origin` field on `ChannelResponseModel`; the SPA badges only `AutoTuned`. Rows predating the column read `Unknown` — provenance is **not** back-filled. | 2026-07-23 | [link](records/channel/origin-marker.md) |
|
||||
| `ci.actions-credential-scoping` | Any credential reachable from an Actions job is scoped to what that job needs. The container-registry secret `REGISTRY_PASSWORD` is a personal access token scoped `write:package` + `read:repository` — never an account PASSWORD. This matters because Gitea has NO `status` token scope: `POST /repos/{o}/{r}/statuses/{sha}` is gated by `reqRepoWriter(unit.TypeCode)`, so ANY credential that can write the repository can forge `review-verdict/h10`, the required context that is supposed to make merge-consent derived rather than assertable. Package-write IS a separate scope, so the registry credential can be made status-incapable at no cost: `scripts/ci-detect-already-validated.sh` only GETs. Do NOT add a `permissions:` key to constrain the injected `GITEA_TOKEN` on the assumption that it binds — below Gitea 1.26.0 it is silently a NO-OP, which is worse than absent because it reads in review as a constraint. That version precondition NO LONGER HOLDS: this instance was upgraded 1.25.4 -> 1.27.1 on 2026-08-05. What has NOT changed is that the consequence is unverified — whether `permissions:` is honored here, and what this instance's default Actions token permission is, were both left UNPROBED (there is still no API surface: `/api/v1/settings/actions` 404s at 1.27.1). Probe before relying on it; do not read the upgrade alone as the constraint now working. Scoping is necessary and not sufficient: it bounds what a job may DO, never whether attacker YAML runs at all, so a self-referencing trigger needs its own filter (`ci-image.yml`, tracked in #744 — deliberately NOT bundled here, because editing that file re-points `ci-image-pin` at the editing commit and reddens a blocking job). This record closes ONE route. It does not close the class, and four later sections say exactly what survives — read them before citing this record as a mitigation. | 2026-08-05 | [link](records/ci/actions-credential-scoping.md) |
|
||||
| `ci.batch-pushes-no-cancel-route` | Hold review fixes, doc corrections and format fixes locally and push **once** — a superseded run cannot be cancelled from the agent side and holds a runner slot until it finishes. | 2026-07-21 | [link](records/ci/batch-pushes-no-cancel-route.md) |
|
||||
| `ci.build-once-rejected` | CI build-once (a shared compile artifact across jobs) was implemented, measured, and rejected for a 40-85% wall-clock regression; keep the #420 cross-run tree-identity skip instead. | 2026-07-18 | [link](records/ci/build-once-rejected.md) |
|
||||
| `ci.cancelled-is-not-a-verdict` | Treat a `cancelled` conclusion as "no verdict" — never as pass or fail — and report FAILED and CANCELLED counts separately in any CI monitor. | 2026-07-21 | [link](records/ci/cancelled-is-not-a-verdict.md) |
|
||||
| `ci.cancelled-is-not-a-verdict` | Treat a `cancelled` conclusion as "no verdict" — never as pass or fail — and report FAILED and CANCELLED counts separately in any CI monitor. THE COMBINED COMMIT-STATUS ENDPOINT CANNOT EXPRESS THIS: `GET /repos/{o}/{r}/commits/{sha}/status` has states `success`/`failure`/`pending`/`error` and NO `cancelled`, so it reports a cancelled job as `failure`. Anything polling that endpoint — which is what a CI monitor naturally polls, because it is the per-sha view the merge gate reads — must resolve the job-level `conclusion` via `actions/runs/{id}/jobs` before reporting a red. | 2026-07-21 | [link](records/ci/cancelled-is-not-a-verdict.md) |
|
||||
| `ci.decisions-edit-trailer` | The body-diff exemption is armed by an affirmative `Decisions-Edit:` **git trailer** (`yes`/`true`/`1`, case-insensitive, read with `unfold`) on some NON-MERGE commit in the PR's merge-base range — never by a substring search over the message text. A non-affirmative value (`no`) does not arm it, the retired `[decisions-edit]` substring arms nothing (the validator emits a `::warning::` nudge when it sees one without a trailer), and a git error leaves the guard ON. | 2026-07-25 | [link](records/ci/decisions-edit-trailer.md) |
|
||||
| `ci.decisions-lifecycle-flake` | When `decisions lifecycle` is the **only** red job, do not investigate and do not create a new run to clear it — no rebase, no `--amend`, no no-op push; the operator reruns that single job from the Gitea UI. | 2026-07-21 | [link](records/ci/decisions-lifecycle-flake.md) |
|
||||
| `ci.docs-only-detect-shallow-safe` | The docs-only detect script must diff against `FETCH_HEAD` (always resolves after `git fetch`, even shallow) using a two-dot tree diff — not `origin/<base>` with three-dot — because a `fetch-depth: 1` shallow clone has no remote-tracking ref and no merge-base, which silently fails the original detect into `docs_only=false` (full matrix, no functional error). A CI-behavior change must be verified by measuring the effect (job durations), not just a green check. | 2026-07-17 | [link](records/ci/docs-only-detect-shallow-safe.md) |
|
||||
| `ci.docs-only-skip-steps` | A docs-only change must still run every required job (`test`, `migrations`) so their commit-status contexts always report; each heavy job runs `scripts/ci-detect-docs-only.sh` first and gates its real STEPS on `if: steps.detect.outputs.docs_only != 'true'`, never `if:`-skips the whole job (an `if:`-skipped job reports `skipped`, not `success`, which branch protection may never unblock on). Detection biases toward running more on any doubt. | 2026-07-17 | [link](records/ci/docs-only-skip-steps.md) |
|
||||
| `ci.exemption-provenance` | The three inputs the exemption decision rests on must each be bound to something the judged PR cannot mutate. (1) BASE — `scripts/pr-changed-files.sh` takes the expected base BRANCH as a REQUIRED 5th argument and re-reads it before and after paging, because `/pulls/{n}/files` diffs against the PR's live base and retargeting moves the answer without moving the head sha; the workflow passes `github.event.pull_request.base.ref` from the `pull_request_target` payload, which a retarget cannot rewrite. (2) BOT EXEMPTION — an author match is necessary but never sufficient: `pull_request.user.login` is the PR's immutable CREATOR while its head is not, so the exemption additionally requires EVERY changed path to be a dependency manifest (`Directory.Packages.props` or `.config/dotnet-tools.json`, and ONLY those — the npm manifests are excluded because `package.json` `scripts` are executed by CI). (3) INHERITED SUCCESS — the never-overwrite short-circuit fires only for a status POSITIVELY identified as a human verdict for THIS base, meaning a non-null `.creator.login` AND a `Review-verdict:` description AND, when that description records a base (`(base: …)`, `release.verdict-status-check`), a base matching the PR's — tested by requiring the description to END with the exact literal `(base: <base>)` and to contain exactly ONE such marker, never by extracting a value (see below); a present-but-different base is rejected, an absent one is not, since verdicts predating that convention carry none; every other shape, including any unrecognised one, is re-derived rather than trusted. The bot and docs-only exemptions are evaluated as INDEPENDENT predicates and the decision made afterwards, never as an `elif` chain. `edited` is in the workflow's `types:` so a retarget reclassifies — which gives DETECTION, not atomicity: status writes are not serialized, so a stale run can still post over a fresher one. That residual is now FENCED rather than merely tracked — the job refuses to write at all if the PR's timeline retarget COUNT moved while it was classifying (`ci.verdict-write-retarget-fence`, #706) — leaving only the sub-round-trip window that no API without compare-and-set can close. The PROTECTED path list additionally covers `.codex/` (#711), which mirrors `.claude/hooks/` byte for byte including the merge-consent hook — latent while that directory is untracked, live the moment it is tracked; the list stays ENUMERATIVE rather than derived, because a derived rule would have to be evaluated against the very file list being classified. Path predicates are evaluated by COUNTING with `grep -c`, never `\| grep -q` (SIGPIPE inversion) and never a here-string (temp-space failure) — see `ci.grep-q-pipefail-inversion`. | 2026-07-29 | [link](records/ci/exemption-provenance.md) |
|
||||
| `ci.exemption-provenance` | The three inputs the exemption decision rests on must each be bound to something the judged PR cannot mutate. (1) BASE — `scripts/pr-changed-files.sh` takes the expected base BRANCH as a REQUIRED 5th argument and re-reads it before and after paging, because `/pulls/{n}/files` diffs against the PR's live base and retargeting moves the answer without moving the head sha; the workflow passes `github.event.pull_request.base.ref` from the `pull_request_target` payload, which a retarget cannot rewrite. (2) BOT EXEMPTION — an author match is necessary but never sufficient: `pull_request.user.login` is the PR's immutable CREATOR while its head is not, so the exemption additionally requires EVERY changed path to be a dependency manifest (`Directory.Packages.props` or `.config/dotnet-tools.json`, and ONLY those — the npm manifests are excluded because `package.json` `scripts` are executed by CI). (3) INHERITED SUCCESS — the never-overwrite short-circuit fires only for a status POSITIVELY identified as a human verdict for THIS base, meaning a non-null `.creator.login` AND a `Review-verdict:` description AND, when that description records a base (`(base: …)`, `release.verdict-status-check`), a base matching the PR's — tested by requiring the description to END with the exact literal `(base: <base>)` and to contain exactly ONE such marker, never by extracting a value (see below); a present-but-different base is rejected, an absent one is not, since verdicts predating that convention carry none; every other shape, including any unrecognised one, is re-derived rather than trusted. The bot and docs-only exemptions are evaluated as INDEPENDENT predicates and the decision made afterwards, never as an `elif` chain. `edited` is in the workflow's `types:` so a retarget reclassifies — which gives DETECTION, not atomicity: status writes are not serialized, so a stale run can still post over a fresher one. That residual is now FENCED rather than merely tracked — the job refuses to write at all if the PR's timeline retarget COUNT moved while it was classifying (`ci.verdict-write-retarget-fence`, #706) — leaving only the sub-round-trip window that no API without compare-and-set can close. The PROTECTED path list additionally covers `CLAUDE.md` and `AGENTS.md` (#751) — they are not prose but the documents DEFINING the completion protocol, the merge-consent convention and the H10 rule, so protecting `.claude/` while the file specifying what it enforces stayed docs-only-exempt was the same self-exemption one directory over; driving the real classify body with a lone `CLAUDE.md` change produced an exemption `success`. `README.md` is deliberately not listed. It also covers `.codex/` (#711), which mirrors `.claude/hooks/` byte for byte including the merge-consent hook — latent while that directory is untracked, live the moment it is tracked; the list stays ENUMERATIVE rather than derived, because a derived rule would have to be evaluated against the very file list being classified. Reading the CURRENT status for input (3) must tolerate `statuses: null`: `GET /commits/{sha}/status` serialises a nil slice as `null`, not `[]`, on a head with no statuses yet, and an `array`-only gate made `read_existing_verdict` `exit 1` and post nothing at all (#751, `ci.workflow-run-body-no-expressions`) — `null` is accepted only when `total_count` is 0, so a body that merely lost its array is still refused. Path predicates are evaluated by COUNTING with `grep -c`, never `\| grep -q` (SIGPIPE inversion) and never a here-string (temp-space failure) — see `ci.grep-q-pipefail-inversion`. | 2026-07-29 | [link](records/ci/exemption-provenance.md) |
|
||||
| `ci.format-gate-folder-mode` | The blocking `format` CI job (and matching pre-commit hook) runs `dotnet format whitespace . --folder --include <files>` instead of loading the full MSBuild/Roslyn solution, cutting the gate from ~480s to ~0.5s with unchanged whitespace/charset coverage. | 2026-07-19 | [link](records/ci/format-gate-folder-mode.md) |
|
||||
| `ci.functional-e2e-harness` | The `functional-e2e` CI job boots the PR's own code from source via `dotnet run` (`scripts/e2e-local.sh`) and runs deterministic assertions (`scripts/e2e-functional.sh`) as an advisory (non-blocking) job, not a `build` dependency or required check. Originally curl-only; since #445 the same job carries a second, headless-browser step for the contracts curl cannot express — see `ci.ui-e2e-harness`. | 2026-07-16 | [link](records/ci/functional-e2e-harness.md) |
|
||||
| `ci.gate-trigger-base-resolved` | The workflow that writes the branch-protection-required `review-verdict/h10` status triggers on `pull_request_target` with `branches: [main]`, never on plain `pull_request`. Gitea resolves a `pull_request` workflow DEFINITION from the PR's own head commit, so under that trigger a PR editing `.gitea/workflows/review-verdict.yml` ran its own rewritten copy and could post `h10=success` for itself; `pull_request_target` resolves the definition from the base instead. The `branches: [main]` filter is part of the rule, not a refinement of it: base resolution only relocates the rewrite from the head to the base, so without the filter a PR opened into an attacker-pushed base branch runs that branch's gate. `pull_request_target` is safe HERE only because this job never checks out or executes head-supplied code — it checks out `base.sha` and runs only that tree's scripts (`ci.shared-pr-file-enumeration`); reintroducing a head checkout under this trigger would be worse than the bug it fixed. This closes the rewrite route through THIS workflow and does NOT close the class: Gitea injects a write-capable `GITEA_TOKEN` into EVERY job, so any ref-resolved workflow — and a collaborator's own API token, since branch protection binds the context and not its issuer — can still forge `review-verdict/h10`. Tracked in #697; the exemption path has its own separate defects in #698. | 2026-07-28 | [link](records/ci/gate-trigger-base-resolved.md) |
|
||||
| `ci.gate-trigger-base-resolved` | The workflow that writes the branch-protection-required `review-verdict/h10` status triggers on `pull_request_target` with `branches: [main]`, never on plain `pull_request`. Gitea resolves a `pull_request` workflow DEFINITION from the PR's own head commit, so under that trigger a PR editing `.gitea/workflows/review-verdict.yml` ran its own rewritten copy and could post `h10=success` for itself; `pull_request_target` resolves the definition from the base instead. The `branches: [main]` filter is part of the rule, not a refinement of it: base resolution only relocates the rewrite from the head to the base, so without the filter a PR opened into an attacker-pushed base branch runs that branch's gate. `pull_request_target` is safe HERE only because this job never checks out or executes head-supplied code — it checks out `base.sha` and runs only that tree's scripts (`ci.shared-pr-file-enumeration`); reintroducing a head checkout under this trigger would be worse than the bug it fixed. This closes the rewrite route through THIS workflow and does NOT close the class: Gitea injects a write-capable `GITEA_TOKEN` into EVERY job, so any ref-resolved workflow — and a collaborator's own API token, since branch protection binds the context and not its issuer — can still forge `review-verdict/h10`. The credential half is now RESOLVED in `ci.actions-credential-scoping` (#697): CI's registry secret was the ADMIN account's basic auth and is now a PAT that cannot post a status, which removes the ADMIN escalation and that credential's route (a user credential's forgery carries a real `creator` and is inherited as a human verdict; an Actions job's carries `creator: null` and is re-derived — but do NOT read that asymmetry as protection: re-derivation fires only on the trigger's `types`, and posting a status is not one of them, so a POST timed after the last PR event simply stands). It does not remove EVERY route: `RENOVATE_TOKEN` is a `write:repository` bot PAT in the same secret store, reachable by any PR-added workflow. The injected token stays write-capable until Gitea >=1.26 with a Restricted default (server-management#714), and a collaborator's own token remains unfixable; the exemption path has its own separate defects in #698. | 2026-07-28 | [link](records/ci/gate-trigger-base-resolved.md) |
|
||||
| `ci.gitea-milestone-filter-noop` | Never filter issues with the server-side `?milestones=<name>` parameter — fetch all open issues once and filter LOCALLY on each issue's `.milestone.title`. | 2026-07-21 | [link](records/ci/gitea-milestone-filter-noop.md) |
|
||||
| `ci.grep-q-pipefail-inversion` | In any script running under `set -o pipefail`, a security or classification predicate of the form `producer \| grep -q…` is FORBIDDEN: `grep -q` exits at its first match, the producer then takes SIGPIPE and exits 141 once the data exceeds the pipe buffer (~64K), so `pipefail` reports the pipeline as FAILED even though grep MATCHED — inverting the predicate exactly when the input is large. A here-string (`grep -q… <<< "$data"`) is ALSO forbidden: bash materialises a large here-string via temporary storage, so it fails when temp space is full or unwritable, and inside an `if`/`!` that failure flips the predicate the same way. COUNT instead — `n=$(printf '%s\n' "$data" \| grep -cE "$re")` — because `grep -c` drains stdin (no early exit, no SIGPIPE) over an ordinary pipe (no temp file). Read grep's status honestly: exit 1 means a zero count and is a legitimate answer, anything >1 is a real error. Evaluate the counts ONCE at TOP LEVEL, never inline inside an `if`/`elif` condition: inside `$( )` an `exit` leaves only the subshell and `set -e` does not fire, so an error silently reads as "no match". Validate that each result is numeric and fail closed if not. This applies to both the enforced gate `.gitea/workflows/review-verdict.yml` and the advisory hook `.claude/hooks/pretooluse-merge-consent.sh`. | 2026-07-29 | [link](records/ci/grep-q-pipefail-inversion.md) |
|
||||
| `ci.infra-shaped-red-under-load` | When a job dies inside a setup/cache step before your code compiles, check the runner host's load before diagnosing the diff, and never file a CI bug off one sample under pressure. | 2026-07-21 | [link](records/ci/infra-shaped-red-under-load.md) |
|
||||
@@ -55,15 +56,18 @@ the link for rationale. Superseded/retired history lives in `archive/`. Regenera
|
||||
| `ci.monitor-armed-at-pr-open` | Arm a CI monitor on the PR head sha the moment the PR opens, polling the commit-status endpoint — not at the end of the work. | 2026-07-21 | [link](records/ci/monitor-armed-at-pr-open.md) |
|
||||
| `ci.no-host-health-gating` | Push when your work is validated — never SSH to bumblebee to sample load/RAM first, and never hand-schedule around other sessions' runs. | 2026-07-21 | [link](records/ci/no-host-health-gating.md) |
|
||||
| `ci.peak-anon-measurement` | The `test` job's headline memory figure is a sampled high-water mark of cgroup `anon`, produced by `scripts/ci-peak-anon.sh`; `memory.peak` and the end-of-job `anon`/`file` split are kept only as a cache-inflated reference. | 2026-07-19 | [link](records/ci/peak-anon-measurement.md) |
|
||||
| `ci.python-lint-ruff-config-committed` | The repo commits `ruff.toml`, and the `script-tests` job runs `ruff check` + `ruff format --check` under a PINNED ruff over an EXPLICIT population from `git ls-files`, never `ruff check .`. Never rely on `~/.config/ruff/ruff.toml`, and never add a lint rule to the config without making the tree clean against it in the same PR. | 2026-08-21 | [link](records/ci/python-lint-ruff-config-committed.md) |
|
||||
| `ci.required-job-step-execution-markers` | A step the runner declines to interpolate is DROPPED and the job still concludes `success` (`ci.workflow-run-body-no-expressions`). In `review-verdict.yml` that is fail-CLOSED — the required status is absent and the merge is blocked. In `docker-build.yml`'s `test` and `migrations` it is fail-OPEN: those are the other two required contexts on `main`, so the check reports green having done no work. So in those two jobs every `run:` step that is not `continue-on-error: true` calls `"$GITHUB_WORKSPACE/scripts/ci-step-ran.sh" mark <key>` as its FIRST act, and the job's LAST step calls `ci-step-ran.sh assert --always <keys> --gated <keys>`, which fails the job when an expected key was never recorded. PER STEP, not per job: a marker written by the first step only proves the job started, while the drop that costs something is `Test` or the migration replay. The guard carries NO `if:` — the default `success()` is the wanted condition, because a genuine failure in an early step legitimately skips every later one and an `always()` guard would announce a false "these steps never executed" on every ordinary red build; the invariant that makes the omission safe is that the guard is skipped only when an earlier step FAILED, which already fails the job, so guard-skipped implies job-red and every path to a green job runs the guard. Separately and independently, no `${{` OPENER may appear in any `run:` body of those two jobs OR of `build` — the drop mechanism requires the opener, so banning it makes the class unreachable rather than merely caught, and an UNCLOSED opener triggers the same rewrite as a well-formed pair. Pass values in through the step's `env:`, which is interpolated per value. The two halves have DIFFERENT scopes on purpose: markers cover the required pair, while the ban also covers `build`, whose `Smoke + IPTV E2E` step runs AFTER the image is pushed, so a drop there publishes a release candidate that was never booted and that `DeployStack jazz-media` then promotes. `functional-e2e` is delimiter-free but deliberately excluded (advisory by declaration), and `api-docs`/`format` keep one `github.base_ref` each and gate nothing that ships. The ban is enforced on the RELEASE PATH itself, not only in review (#767): a `scan` job runs the PyYAML-based ban test and `build` lists it in `needs:`, so a delimiter means `build` never runs and no image is published. A guard STEP inside `build` was tried first and is wrong — a step cannot protect the job it publishes from, and "my body has no opener so I cannot be dropped" is circular when only the PR-only test enforces that. The pytest in `script-tests` remains, but it is `on: pull_request` and not a required context, so it alone left the tag path unchecked. | 2026-08-10 | [link](records/ci/required-job-step-execution-markers.md) |
|
||||
| `ci.root-screenshot-guard` | The Husky `pre-commit` hook refuses a staged root-level `*.png` (belt-and-suspenders with the `.gitignore` rule); nested `*.png` real assets are unaffected. | 2026-07-12 | [link](records/ci/root-screenshot-guard.md) |
|
||||
| `ci.runner-placement` | No persistent Roslyn compiler server survives a CI build (`UseSharedCompilation=false` etc., runner env + Dockerfile `ENV`); every `services:` container gets its own explicit `--memory`/`--memory-swap`/`--cpus` cap (it does not inherit the job container's). | 2026-07-17 | [link](records/ci/runner-placement.md) |
|
||||
| `ci.script-tests-job` | The `scripts/tests/` pytest suite runs on every PR as a dedicated `script-tests` job in `pr-checks.yml` (`runs-on: small`, `setup-python` + `pip install pytest`, `PYTHONPATH=. python3 -m pytest scripts/tests -q`), unconditionally rather than behind a `scripts/**` path filter, and **never as a step inside `decisions-guard`** — a job whose reds a standing rule instructs sessions to ignore must never host a gate whose reds are real. Any new CI gate must be reachable by a failure that is unambiguously attributable to it. | 2026-07-26 | [link](records/ci/script-tests-job.md) |
|
||||
| `ci.script-tests-job` | The `scripts/tests/` pytest suite runs on every PR as a dedicated `script-tests` job in `pr-checks.yml` (`runs-on: small`, `setup-python` + `pip install pytest pyyaml`, `PYTHONPATH=. python3 -m pytest scripts/tests -q`; since #780 it also runs a pinned ruff over a `git ls-files` population first), unconditionally rather than behind a `scripts/**` path filter, and **never as a step inside `decisions-guard`** — a job whose reds a standing rule instructs sessions to ignore must never host a gate whose reds are real. Any new CI gate must be reachable by a failure that is unambiguously attributable to it. | 2026-07-26 | [link](records/ci/script-tests-job.md) |
|
||||
| `ci.shared-pr-file-enumeration` | A PR's complete set of changed file paths is computed by exactly one implementation, `scripts/pr-changed-files.sh`, called by both `.claude/hooks/pretooluse-merge-consent.sh` (advisory — a failure falls through to a human prompt) and `.gitea/workflows/review-verdict.yml` (enforced — a failure must fail closed, because a match here posts the branch-protection-required `review-verdict/h10` status with nobody in the loop). The script owns exhaustiveness (pagination, rename/path validation, head-sha binding, base-ref binding — see `ci.exemption-provenance` — and base-TIP binding, #707: the ref answers "did this PR RETARGET", the tip answers "did the base ADVANCE mid-enumeration", and only the second can see `/pulls/{n}/files` recomputing each offset-paged page against a moved base and dropping a path out of an already-consumed range; both ends of the window are bound, and an advance BEFORE the window is deliberately not an error, or ordinary churn on `main` would fail every open PR) and returns exit 0 only for a verified-complete list; it does NOT classify paths — each caller keeps its own docs-only allow-list, and the two allow-lists differ on purpose and stay separate. | 2026-07-26 | [link](records/ci/shared-pr-file-enumeration.md) |
|
||||
| `ci.small-lane-git-only` | `runs-on: small` is defined by what a job does (git-only), not its usual runtime; the two `docker build` jobs (docker-build.yml, ci-image.yml) move to `ubuntu-latest` because their worst-case memory, not median runtime, was pinning the small lane's per-slot cap. | 2026-07-20 | [link](records/ci/small-lane-git-only.md) |
|
||||
| `ci.ui-e2e-harness` | The UI-interactive E2E flows run as headless Playwright specs (`web/e2e/*.spec.ts`, driven by `scripts/e2e-ui.sh`) in a **second step of the existing advisory `functional-e2e` job**, never their own job; the browser is `chromium-headless-shell` **baked into the CI toolchain image** (`docker/ci/Dockerfile`, `PLAYWRIGHT_VERSION` kept equal to `web/package.json`'s EXACT `@playwright/test` pin), never installed per run; specs are `serial` with `retries: 0` and assert only contracts the curl harness structurally cannot reach. | 2026-07-25 | [link](records/ci/ui-e2e-harness.md) |
|
||||
| `ci.verdict-write-retarget-fence` | The `review-verdict/h10` job counts `change_target_branch` events on the PR's issue timeline at run start and again immediately before its POST, and writes NOTHING if the count moved. The COUNT is the key because the branch NAME is ABA-vulnerable — `main -> S -> main` reads `main` at both ends, which is how #698 route 1 obtained a forged exemption — while the event count is monotonic and cannot alias. Abstaining is a handoff, not a stall, and that is the property the design rests on: every retarget fires `edited`, which is in this workflow's `types:`, so the event that makes a run abstain has already queued a successor whose window opens after it; the induction terminates when retargeting stops and the last run writes the final answer. `updated_at` was REJECTED as the key because it also moves for comments and labels, which fire none of this workflow's `types:` — a run could abstain with no successor coming, which is a real stall. The count is trusted only when paging reached a validated EMPTY page; an untrusted count (unreadable page, non-array body, non-numeric length, page cap hit) blocks the exemption `success` ONLY and still lets `pending` through, because `pending` cannot turn an unreviewed head green while withholding it would strand ordinary PRs for no safety gain. SEPARATELY, and for the human-verdict race the fence does nothing about: after posting an exemption `success` the job re-reads `/statuses/{sha}` and, if a human `Review-verdict:` row appeared with an id ABOVE a high-water mark taken just before the POST, overwrites its own status with `pending` and logs an error. The repair is `pending`, NEVER a copy of the human's state, since re-posting their `failure` under the machine credential would attribute a human verdict to the job; its description is a SENTINEL that the classification refuses to grant an exemption over AND re-writes verbatim on every later run, so the block is a FIXED POINT rather than decaying — writing the generic `pending` description there instead erases the marker and the exemption simply returns one event later. The mark is captured BEFORE the last-moment re-read, not merely before the POST — a later mark leaves a multi-round-trip blind gap in which a verdict is neither seen by the re-read nor repaired afterwards. The id comparison is load-bearing: a mere presence test would fire forever on a base-mismatched verdict that `read_existing_verdict` deliberately declines to honour, deadlocking that PR's exemption permanently. Finally, a run whose last-moment re-read finds a sentinel it did not see at its FIRST read ABSTAINS instead of posting: that can only mean an overlapping run repaired a raced verdict mid-flight, and this run's `success` — frozen at classification time, with the human row below its own mark, so neither the fence nor the post-write check would catch it — would otherwise bury the rejection. That is the one path in this design that failed toward SUCCESS rather than `pending`. The post-write check counts TWO row shapes above the mark, not one — a human `Review-verdict:` row AND a machine sentinel — because with two overlapping runs the human row can sit BELOW the second run's mark while the first masks it and only then writes the sentinel, leaving the second to post its own `success` on top; counting the sentinel converges both runs on the fixed point instead. | 2026-08-03 | [link](records/ci/verdict-write-retarget-fence.md) |
|
||||
| `ci.verify-locally-ci-confirms` | Treat the local build/verify/review pass as the decision point and CI as confirmation — don't idle waiting on a run you have no reason to doubt. | 2026-07-21 | [link](records/ci/verify-locally-ci-confirms.md) |
|
||||
| `ci.web-test-per-test-timeouts` | Give heavy-render web tests an explicit per-test vitest timeout (e.g. 15s); never raise the global default to fix one slow test. | 2026-07-21 | [link](records/ci/web-test-per-test-timeouts.md) |
|
||||
| `ci.workflow-run-body-no-expressions` | A `run:` body is not shell when the runner reads it: the runner scans the whole scalar for the expression opener and, on finding one, rewrites the ENTIRE body into a single `format(...)` call. That rewrite is all-or-nothing, so a payload that does not evaluate fails the interpolation of the whole scalar — and the runner then DROPS THE STEP AND CONCLUDES THE JOB `success`. A shell comment is therefore NOT inert. In `.gitea/workflows/review-verdict.yml` no expression delimiter may appear in ANY `run:` body, in code or in prose, because a dropped step there is a dead merge gate rather than a failed build; pass values in through the step's `env:` block, which is interpolated per value so a bad payload cannot take the body with it, and describe an expression in prose by NAMING it (`a github.event.pull_request.number expression`) rather than quoting the delimiters. Repo-wide the rule is weaker and its reach must be stated precisely rather than generously: every expression payload in every workflow field must have a HEAD TOKEN naming a context or function the runner can resolve. That catches the defect above and a nonexistent context; it does NOT catch a syntactically invalid payload whose tokens are all known (`${{ github.ref == }}`), a renamed output (every token after the first is skipped), or an unclosed opener — those need an expression parser, and the guard is kept permissive on purpose because a red here blocks every merge through the combined status. In `review-verdict.yml` specifically, any step whose non-execution is consequential is paired with a start-marker guard that FAILS the job when the marker is absent, and that guard's own body must be expression-free — a guard the guarded mechanism can silently delete is worse than none. That pairing now also covers `docker-build.yml`'s `test` and `migrations` jobs, where a dropped step is fail-OPEN (the required check goes green having done no work) rather than fail-closed as it is here — see `ci.required-job-step-execution-markers`, which adds per-STEP markers there and extends this file's delimiter ban to those two jobs. It is still not a repo-wide property, but the remaining exceptions are narrower than this record originally said: `build` was brought into the ban too (its `Smoke + IPTV E2E` runs AFTER the image is pushed, so a drop there ships an unsmoked release candidate — its two payloads moved to `env:`, so the ban was free), leaving only `api-docs` and `format`, whose one `github.base_ref` each sits in a detect step that gates nothing that ships. | 2026-08-06 | [link](records/ci/workflow-run-body-no-expressions.md) |
|
||||
| `concurrency.diff-scalar-fanout` | The frozen Block optimistic-concurrency recipe (api-conventions §7a) fans out to Collection/Playout×2/MultiCollection/RerunCollection, keeping a guard-returned `PreconditionFailedError` out of any handler's generic `catch(Exception)`→422 mapping, and preserving each aggregate's existing `SaveChangesAsync() > 0` gate semantics under the new unconditional `Version++`. | 2026-07-11 | [link](records/concurrency/diff-scalar-fanout.md) |
|
||||
| `concurrency.etag-rotation-completion` | Every handler that mutates a versioned root's editor-visible config state must bump `Version` (rotating the ETag) with no per-aggregate carve-outs, short-circuiting on a genuine no-op before the bump so idempotent re-submits don't fire spurious rebuild fan-out; `SaveChangesForcingVersion` rebases the retry (stored + pending delta), never adopts the stored token verbatim. | 2026-07-12 | [link](records/concurrency/etag-rotation-completion.md) |
|
||||
| `concurrency.force-write-non-ifmatch` | Any handler that leaves a versioned root `Modified` or `Deleted` but takes no `If-Match` (deletes, item add/remove bumpers, scalar-config writers) must save through `ConcurrencyExtensions.SaveChangesForcingVersion` — force-write past a concurrent `Version` bump rather than throw an unhandled `DbUpdateConcurrencyException` (500). | 2026-07-12 | [link](records/concurrency/force-write-non-ifmatch.md) |
|
||||
@@ -77,6 +81,7 @@ the link for rationale. Superseded/retired history lives in `archive/`. Regenera
|
||||
| `docs.decision-one-file-per-record` | Each decision record is its own file at `docs/decisions/records/<area>/<topic>.md` (archived ones at `docs/decisions/archive/<area>/<topic>.md`) with YAML frontmatter; the filename IS the key, so one-active-record-per-key is a filesystem property rather than a validator check, and supersession is a `git mv`. | 2026-07-25 | [link](records/docs/decision-one-file-per-record.md) |
|
||||
| `docs.decision-optional-provenance` | Decision records gain two OPTIONAL fields — `stale-after: YYYY-MM-DD` on the metadata line and a `**Sources:**` line in the metadata block; the Open Knowledge Format (OKF) itself is NOT adopted as the record format. | 2026-07-25 | [link](records/docs/decision-optional-provenance.md) |
|
||||
| `docs.frontmatter-pyyaml-crosscheck` | `decisions_validate.py` runs `pyyaml_frontmatter_faults()` over every record-wing file: it loads the frontmatter with PyYAML and reports an ERROR when PyYAML rejects the document OR when any key's value differs from what the dependency-free `dl._read_frontmatter` read. PyYAML is the WRITER of these files (`migrate_decisions_split.render_record` emits them with `yaml.safe_dump`), so on any disagreement PyYAML is authoritative and the defect is in the FILE, not in either parser. The check is strictly additive: when PyYAML is not importable it is SKIPPED and `main()` says so with a `::notice::`, never silently — the read path stays dependency-free because `decisions-guard`, the Husky hooks and contributor machines install nothing. The comparison has exactly ONE implementation, called by both the validator and `test_frontmatter_reader_matches_pyyaml_on_every_real_record`, so the suite and the tool cannot drift on what "matches PyYAML" means. | 2026-08-04 | [link](records/docs/frontmatter-pyyaml-crosscheck.md) |
|
||||
| `docs.no-session-narrative` | Every durable artifact — an in-repo `docs/` page, a skill, a README, a code comment, an Obsidian vault page — records the END STATE. The path to that end state goes in the commit message, the Gitea issue, or the issue's `## Closing record`; it does not go in the artifact. Concretely: **a review finding is answered in the commit message, and only the corrected claim enters the doc.** Naming the destination is load-bearing — "do not write it in the doc" with no home loses the knowledge, and this repo has the inverse failure on record too (#542, where a pruned narrative turned out to be the only copy). THE TEST IS WHO BENEFITS: if only the author's timeline explains why a sentence is there, it is narrative and belongs in the commit; if a reader who never saw the session would act differently knowing it, it is a finding and stays. Session narrative reads as: first person or session chronology ("I initially thought", "an earlier draft counted", "my first attempt returned 0"), a correction of a belief the reader never held ("this was wrong, actually X" where only X matters), relative time ("earlier today", "currently investigating"), or a blow-by-blow diagnosis standing in place of the conclusion. THE CARVE-OUT, which must be stated or the rule gets over-applied — reader-facing history that must survive: a decision record's `supersedes`/`superseded-by`; a dated measurement or an explicitly stated snapshot boundary; a TESTED-AND-REJECTED negative result, kept so nobody re-proposes it on plausibility; the *why* behind a non-obvious choice; and a trap together with its consequence. `docs/decisions/records/**` and `docs/decisions/archive/**` are exempt WHOLESALE: a record narrating how a rule was got wrong is carrying the rationale it exists to carry. ENFORCEMENT IS ADVISORY ONLY — `scripts/check-doc-narrative.py`, run non-blocking from the `docs-reminder` job over ADDED lines. It is a string predicate over prose and may never become a blocking gate. | 2026-08-21 | [link](records/docs/no-session-narrative.md) |
|
||||
| `docs.record-wing-parse-guard` | `decisions_validate.py` asserts, per PATH, that every `*.md` under `docs/decisions/records/**` and `docs/decisions/archive/**` parses to exactly one record carrying a `key` — an ERROR, not a warning, since a file in the record wings that is not a record is a mistake by definition. A file sitting DIRECTLY in `archive/` is exempt only when it actually looks like a #610 stripped index — exactly one keyless record with a known generated heading — never merely by living there. The one other exemption, `archive/README.md`, is by exact RELATIVE PATH; nothing is ever exempt by BASENAME, since that would exempt the same filename in the active wing too. `_read_frontmatter` is deliberately NOT extended to accept YAML block scalars: every record value goes on ONE line, and the structural check is what makes that limitation loud instead of silent. | 2026-07-26 | [link](records/docs/record-wing-parse-guard.md) |
|
||||
| `docs.tracker-comment-retrofit` | When the knowledge exporter flags an over-cap tracker issue and excludes it from ingestion, triage its comments instead of assuming a retrofit is owed — and for each decision-shaped item check the **worked issue first**, because a tracker session comment is by construction a précis of the fuller closing record posted on the issue it narrates. Applied to #237 (111 comments) this yielded **zero** records, so server-management#642's "a fact found only in a #237 comment" retrieval row has no valid subject and its interim target (an already-migrated record) is permanent. | 2026-07-21 | [link](records/docs/tracker-comment-retrofit.md) |
|
||||
| `ffmpeg.external-logo-graphics-engine` | External-URL channel logos pass through to the graphics engine like any other watermark source; `WatermarkSelector` must never gate them on `File.Exists` (always false for a URL) and never route them through the ffmpeg-native overlay shortcut. | 2026-07-20 | [link](records/ffmpeg/external-logo-graphics-engine.md) |
|
||||
@@ -95,13 +100,15 @@ the link for rationale. Superseded/retired history lives in `archive/`. Regenera
|
||||
| `iptv.logo-drives-bug-preset` | One uploaded channel logo drives both the listing logo and the on-screen bug via a shared, seeded `ChannelLogo`-sourced watermark preset (`Channel Bug`), not new per-channel schema. | 2026-07-20 | [link](records/iptv/logo-drives-bug-preset.md) |
|
||||
| `locking.entitylocker-atomic-flags` | `EntityLocker` uses `Interlocked.CompareExchange`-guarded atomic flags plus a documented single-owner-release discipline (no owner tokens/leases); `Unlock*` on an already-unlocked slot returns `false` and logs a Warning rather than throwing. | 2026-07-11 | [link](records/locking/entitylocker-atomic-flags.md) |
|
||||
| `mcp.server-foundation` | `ErsatzTV.Mcp` is a fresh stdio JSON-RPC server wrapping frozen `/api/v1` with explicit narrow per-endpoint tools, read-only-by-default enforced at runtime (`ERSATZTV_ALLOW_WRITES`), machine-key auth, and opt-in `If-Match`. | 2026-07-20 | [link](records/mcp/server-foundation.md) |
|
||||
| `mcp.tool-schema-openapi-parity` | Every POST/PUT/PATCH tool in `ToolCatalog` declares exactly the request-body properties its endpoint accepts, each with a matching type, and EVERY tool (read and write) declares exactly its endpoint's query parameters, both asserted against the generated `ErsatzTV/wwwroot/openapi/v1.json` (linked into `ErsatzTV.Mcp.Tests`) by `Every_Write_Tool_Should_Declare_Exactly_Its_OpenApi_Request_Body_Fields` and `Every_Tool_Should_Declare_Exactly_Its_OpenApi_Query_Parameters`. A field the endpoint accepts but the tool omits is a DEFECT, not a deferral: on the full-replace tools (channel update, schedule update, custom-order) the omission is silently applied as a clear. The write tools are NOT uniformly full-replace — add-collection-items is additive, and several leave an omitted field unchanged — so each tool description states its own semantics. An omitted query parameter is UNREACHABLE, not merely undocumented, because `ToolArgumentValidator` rejects undeclared arguments. | 2026-08-06 | [link](records/mcp/tool-schema-openapi-parity.md) |
|
||||
| `media.lastscan-null-boundary` | A never-scanned `LastScan` surfaces as `null` at the API/MCP boundary, not the `0001-01-01` MinValue sentinel — enforced by an ongoing read-boundary coercion plus a one-time data migration cleanup. | 2026-07-18 | [link](records/media/lastscan-null-boundary.md) |
|
||||
| `media.remote-stream-probe` | `ValidatePlayoutItemPath` probes the Plex/Jellyfin/Emby remote-stream URL via `IRemoteStreamProber` before returning it; only a redirected 404 fails closed (`PlayoutItemNotAvailableFromMediaServer`), everything else fails open, and there is no toggle. | 2026-07-19 | [link](records/media/remote-stream-probe.md) |
|
||||
| `media.remote-stream-probe-externaljson` | External-JSON playout channels' `StreamRemotely` now probes the remote-stream URL through the same `IRemoteStreamProber` seam as the generated-playout path, closing the #473 scope gap for a channel kind with no DB `PlayoutItem` rows. | 2026-07-20 | [link](records/media/remote-stream-probe-externaljson.md) |
|
||||
| `media.source-mgmt-write-api` | Media-source management (local/Plex/Jellyfin/Emby) is a REST write API + SPA under `/app/libraries/*`, wrapping existing MediatR commands 1:1 with no new commands or DB migration; connection GETs never leak a stored `apiKey`, and each PUT-replace family's identity contract is documented per-family (not assumed uniform). | 2026-07-11 | [link](records/media/source-mgmt-write-api.md) |
|
||||
| `process.bom-format-detection-recipe` | Before any push touching `.cs`, detect BOMs with the `xxd` byte check and verify the format gate with `dotnet format --include` run under `bash -c`, never bare zsh. | 2026-07-21 | [link](records/process/bom-format-detection-recipe.md) |
|
||||
| `process.bom-format-detection-recipe` | Before any push touching `.cs`, detect BOMs with the `od -A n -t x1 -N 3` byte check and verify the format gate with `dotnet format --include` run under `bash -c`, never bare zsh. NOT `xxd`: it ships with vim and is absent on plain Linux hosts including this repo's CI runner, where the substitution yields empty, never matches, and the check reports all-clean — the same all-clean-detector failure this record was written about, in the detector it prescribed. | 2026-07-21 | [link](records/process/bom-format-detection-recipe.md) |
|
||||
| `process.branch-off-feature-branch` | To fix work on an unmerged feature branch, branch off that branch and land by fast-forward push — and after creating a worktree, drive the first Edit/Read from ITS absolute paths and `git status` it before building. | 2026-07-21 | [link](records/process/branch-off-feature-branch.md) |
|
||||
| `process.build-concurrency-limits` | Run at most 3–4 concurrent dotnet/npm builds on this Mac, gate launches on FREE RAM rather than CPU load, and never set `ETV_UPDATE_GOLDENS` / `ETV_UPDATE_PLAYOUT_GOLDENS`. | 2026-07-21 | [link](records/process/build-concurrency-limits.md) |
|
||||
| `process.check-and-use-pins-a-version` | Where a CHECK authorizes an ACTION over state that can change in between, the two are bound to ONE version of that state. Binding alone is not enough and is the half that keeps being skipped: a snapshot nothing re-validates is not pinned, it is a stale read wearing a version number. Three substrates, three mechanisms, and they are the SAME rule — in-process, a compare-exchange claim taken by the caller, never a `Volatile.Read` in one place and an `Interlocked` in another (`ffmpeg.work-ahead-slot-atomic`); over our own HTTP API, RFC 7232 `If-Match`/ETag, with the force-write path named explicitly rather than left implicit (`concurrency.ifmatch-rfc7232`, `concurrency.force-write-non-ifmatch`); against a remote service, a full commit sha, an image digest or a monotonic event count re-read immediately before the write. Prefer true compare-and-set where the server offers it. Where it does not — Gitea's commit-status API has no ETag, no If-Match and no expected-previous-state — the ceiling is READ-COMPARE-REFUSE: re-read the identifier immediately before the write and FAIL CLOSED on any movement, which narrows the window to one round trip and makes the loss observable instead of silent. A residual that cannot be closed is STATED in the code and carried in `docs/remote-state-inventory.md` as `UNSAFE-KNOWN` with the reason it is tolerable; "noticed" is not "accepted". Two identifier traps are load-bearing here: compare the FULL sha, never a 7-char prefix, and compare a base BRANCH REF rather than its tip sha, because the tip moves on every unrelated merge and comparing it deadlocks every open PR. Finally, and this is the failure #778 actually found: a mitigation that lives OUTSIDE the code relying on it — branch protection, a required status context, a server-side refusal — must be VERIFIED at the point of use, not asserted in a comment or in the reason string a human reads. A dated claim about configuration is not a check, and it is worse than no claim, because it talks the next reader out of looking. | 2026-08-16 | [link](records/process/check-and-use-pins-a-version.md) |
|
||||
| `process.codex-cheap-worker-launch` | For bounded tool-bearing selector/recon work, launch a Codex worker with `codex exec -m gpt-5.4-mini -c model_reasoning_effort=low -s read-only`; `spawn_agent` buys parallelism but no cost savings. | 2026-07-21 | [link](records/process/codex-cheap-worker-launch.md) |
|
||||
| `process.consistency-fix-new-code-scrutiny` | Review a "make X consistent with Y" change as new code, not as a mechanical copy — and for any timer or effect involved, ask explicitly "when does this fire?", including on mount. | 2026-07-21 | [link](records/process/consistency-fix-new-code-scrutiny.md) |
|
||||
| `process.enumerate-workaround-behaviors-before-deleting` | When an issue says "delete X", enumerate every behavior X provided before removing it — a workaround often serves a second purpose that outlives the first. | 2026-07-21 | [link](records/process/enumerate-workaround-behaviors-before-deleting.md) |
|
||||
@@ -120,8 +127,9 @@ the link for rationale. Superseded/retired history lives in `archive/`. Regenera
|
||||
| `process.subagent-drop-resume` | Treat a subagent connection drop as laptop sleep or transient network and re-resume via SendMessage — the work survives. | 2026-07-21 | [link](records/process/subagent-drop-resume.md) |
|
||||
| `release.api-contract-ci-gate` | A PR touching `ErsatzTV/Controllers/Api/**` or `ErsatzTV.Core/Api/**` must ship regenerated OpenAPI artifacts (`v1.json`, `v1.d.ts`, `endpoint-index.md`) in the same diff, enforced by a blocking `api-docs` CI job that regenerates-and-diffs against a fresh build. | 2026-07-12 | [link](records/release/api-contract-ci-gate.md) |
|
||||
| `release.done-when-merge-consent` | A PR may merge only when its linked issue's `## Done-when` checklist is fully ticked and the PR's CI is green, enforced by a PreToolUse hook on the Gitea merge tool (deny/allow/ask) plus a pre-push backstop for direct pushes to main. | 2026-07-12 | [link](records/release/done-when-merge-consent.md) |
|
||||
| `release.format-as-you-touch-rebase` | A blocking `format` CI job runs `dotnet format --verify-no-changes` scoped only to the PR's changed `.cs` files (never the legacy BOM backlog), and a PR branch must be kept current by rebasing on `origin/main` (never merging main in), enforced by `.husky/pre-push` → `prepush-rebase-check.sh`. | 2026-07-12 | [link](records/release/format-as-you-touch-rebase.md) |
|
||||
| `release.format-as-you-touch-rebase` | A blocking `format` CI job runs `dotnet format --verify-no-changes` scoped only to the PR's changed `.cs` files (never the legacy BOM backlog), and a PR branch must be kept current by rebasing on `origin/main` (never merging main in), enforced by `.husky/pre-push` → `prepush-rebase-check.sh`. H11 has ONE always-on carve-out, #719 — a push in which EVERY ref is under `refs/tags/` skips the freshness check, because a tag push cannot revert merged work, which is the failure mode H11 exists to prevent, and the release cut tags from a branch that is behind `origin/main` (observed on the v26.13.0 cut, #719). A push mixing branch and tag refs is still blocked, and so is a push with zero parsed ref lines (the exemption requires at least one, so empty stdin cannot vacuously disable H11). | 2026-07-12 | [link](records/release/format-as-you-touch-rebase.md) |
|
||||
| `release.live-e2e-required` | A PR that changes an API write-path handler must include a live-E2E pass (driving the real endpoint/screen and confirming the round-trip through a subsequent read), not only unit/characterization tests, and must state whether live-E2E ran or wasn't required. | 2026-07-12 | [link](records/release/live-e2e-required.md) |
|
||||
| `release.main-direct-push-disabled` | Branch protection on `main` carries `enable_push: false` AND `block_admin_merge_override: true`. Both halves are required and neither is sufficient. `enable_push: false` removes the direct-push path, leaving the PR merge path — the only path on which Gitea evaluates `status_check_contexts`, and therefore the only path on which `review-verdict/h10` is consulted at all. `block_admin_merge_override: true` then closes the force-merge bypass on that remaining path: with it false (the default), `CanBypassBranchProtection` returns true for a repo admin, so `POST /pulls/{n}/merge` with `force_merge: true` merges a PR whose `h10` is missing or red — one API call, no forgery, no PATCH. Do NOT "soften" the push half to a push WHITELIST: measured here, a whitelist naming `timothy` still admits the push, and `timothy` is the identity every agent session, PAT and injected `GITEA_TOKEN` already acts as, so the whitelist form closes nothing while reading in review as a control. Same reasoning is why the admin-override half is needed: an admin-shaped control that exempts the only admin exempts everybody. What remains open: a credential that can PATCH branch protection off can still undo either half — an accepted residual, not a closed route. Tag pushes are unaffected (`tag_protections` governs those separately), so the release cut still works. | 2026-08-05 | [link](records/release/main-direct-push-disabled.md) |
|
||||
| `release.merge-consent-autogrant` | When Done-when boxes are ticked, CI is green, and a fresh positive Review-verdict references head, the merge-consent hook emits `permissionDecision: allow` to actually suppress the redundant mechanical prompt — the derived state IS the consent, no separate conversational confirmation on that path. | 2026-07-12 | [link](records/release/merge-consent-autogrant.md) |
|
||||
| `release.migration-rehearsal-prodcopy` | Before promoting a migration-bearing release, rehearse the new image's migrations against a throwaway copy of the latest prod backup (`scripts/migration-smoke.sh`), gating PASS on the migrator's completion log line rather than HTTP readiness alone. | 2026-07-12 | [link](records/release/migration-rehearsal-prodcopy.md) |
|
||||
| `release.prepush-clean-worktree-guard` | A fail-open pre-push hook blocks a push when any file in the branch's diff vs `origin/main` also has uncommitted working-tree or index changes, since a stale-index commit (e.g. `git reset --soft` + `git add` over an edited-but-unstaged fix) can silently push, CI-test, and get reviewed a different tree than the one on disk. Scope is precise to pushed-diff files; escape hatch `ETV_ALLOW_DIRTY_PUSH=1`. | 2026-07-17 | [link](records/release/prepush-clean-worktree-guard.md) |
|
||||
@@ -160,6 +168,7 @@ the link for rationale. Superseded/retired history lives in `archive/`. Regenera
|
||||
| `security.iptv-browser-token` | Under a JWT-enabled deployment (`JWT:IssuerSigningKey` set), the browser SPA obtains a short-lived, globally-scoped `/iptv/*` access token from an authenticated `GET /api/v1/auth/iptv-token` and appends it as `?access_token=`; the endpoint answers 204 when JWT is disabled (nothing to mint). Lifetime defaults to 60 min, configurable via `JWT:BrowserTokenLifetimeMinutes`. | 2026-07-22 | [link](records/security/iptv-browser-token.md) |
|
||||
| `security.session-auth-dual-credential` | `ApiAuthorizationFilter` accepts a request when a valid `X-Api-Key` matches OR the principal is an authenticated session (cookie `ctv-session`, `HttpOnly`/`SameSite=Lax`); session-authenticated mutations require the presence-only `X-CSRF` header or are rejected 403. This narrows the OIDC-inert sub-claim of `security.blazor-removal-auth-posture` (#206) — the rest of that record's auth-surface enumeration still holds. | 2026-07-12 | [link](records/security/session-auth-dual-credential.md) |
|
||||
| `security.session-cutover-postify` | The browser SPA authenticates cookie-only (no more `X-Api-Key` from `web/`); the machine key is repurposed to external/MCP-only via `GET /api/auth/machine-key`; every side-effecting GET/HEAD under `/api` is converted to POST so the existing CSRF gate covers it (standing rule: never add a side-effecting GET/HEAD under `/api`). | 2026-07-12 | [link](records/security/session-cutover-postify.md) |
|
||||
| `session.local-code-intelligence` | C# and TypeScript find-all-references are available again; brief delegated agents to the `csharp-lsp` MCP tools (`csharp_references`, `csharp_diagnostics`, …) rather than the `LSP` tool, which no dispatched subagent has been observed to resolve (Claude Code 2.1.232, agent types `general-purpose` and `Explore`, 2026-08-14). Preconditions are machine-local — `env.DOTNET_ROOT` in `.claude/settings.local.json` and a root `node_modules/typescript` link — and checkable with `scripts/check-local-lsp.sh`. | 2026-08-14 | [link](records/session/local-code-intelligence.md) |
|
||||
| `session.shared-checkout-refresh` | Session end runs `scripts/refresh-shared-checkout.sh`, which fast-forwards `/Users/timothy/ersatztv` to `origin/main` (and reinstalls `web/node_modules` when the lockfile moved), refusing to touch anything unless that tree is on a clean, non-ahead `main`. | 2026-07-21 | [link](records/session/shared-checkout-refresh.md) |
|
||||
| `spa.add-to-layer` | All add-to-collection/playlist/schedule affordances share one component layer at `web/src/media/addTo/`; multi-select is an explicit screen-level toggle, and the per-card menu offers schedule only for the server-validated kinds. | 2026-07-10 | [link](records/spa/add-to-layer.md) |
|
||||
| `spa.app-shell-extraction` | `App.tsx` is only the composition root over `web/src/app/routes.tsx` (stable route-object identity), `app/AppShell.tsx` (shell chrome), and `app/ScreenContent.tsx` (exhaustive screen dispatch); primary actions are one explicit `PrimaryActionProvider` registration per screen, replacing the old global `ctv:primary-action` window event. | 2026-07-15 | [link](records/spa/app-shell-extraction.md) |
|
||||
@@ -185,10 +194,17 @@ the link for rationale. Superseded/retired history lives in `archive/`. Regenera
|
||||
| `spa.topbar-primary-action` | The TopBar's primary-action "+" button renders only when the active route declares a non-empty `primaryAction`, is wired (via a shared `usePrimaryAction` hook) only on single-unambiguous-create-flow list screens, and is dropped everywhere else rather than left as a dead/no-op button. | 2026-07-12 | [link](records/spa/topbar-primary-action.md) |
|
||||
| `spa.yaml-validator-textarea` | The YAML playout validator takes pasted YAML via a `<textarea>`, not a server-side file path, since the SPA has no filesystem access. | 2026-07-09 | [link](records/spa/yaml-validator-textarea.md) |
|
||||
| `startup.parallel-orientation` | A fresh session runs two concurrent tracks at startup — Orientation (`AGENTS.md`/`CLAUDE.md` → `docs/README.md` task-signal map → the active decisions catalog `docs/decisions/README.md`) and, only when no issue is named, Selection (`scripts/select-queue.sh N`, deterministic live-Gitea ranking). A named issue skips Selection entirely. ersatztv#237, the closed pickup tracker this replaces, is reduced to a single archival breadcrumb and MUST NOT be read for live state. | 2026-07-21 | [link](records/startup/parallel-orientation.md) |
|
||||
| `testing.deny-path-at-production-config-value` | Where behaviour is gated by a configuration value, an environment variable or a credential, the test matrix covers every value the surface will actually meet — the setting ABSENT, the setting at its PRODUCTION value, and each explicit opt-out — and it asserts the DENY branch, not only the allow branch. A fixture that OMITS the field tests the default and nothing else, so a fail-open reachable only through the configured value stays invisible however many tests are green (#756: thirty of them were). Two corollaries carry most of the weight. FIRST, a hand-written test double that is HANDED the resolved flag proves the CONSUMER reacts to it and says nothing about the line that DERIVES it; if no test constructs the real provider, a mistyped configuration key or a flipped default is unobservable to the whole suite. SECOND, the dangerous cell is whichever one production occupies, which is not always the explicit one: when the shipped default IS the permissive branch the absent case is the production case (#280's null `Api:WriteKey`), and when the default is fail-closed the configured value is the one nothing has exercised. Enumerate the cells before deciding which to test; do not infer the risky one from which is easier to write. This rule is NOT mechanically enforced and deliberately so — deciding whether a given test used the production value is a string predicate over test source, the class this repo has withdrawn twice. | 2026-08-21 | [link](records/testing/deny-path-at-production-config-value.md) |
|
||||
| `testing.e2e-cleanup-scope-by-pid` | An E2E harness or agent may only kill processes whose PIDs it captured at launch — capture the PID; whoever owns the lifecycle releases it from a `trap ... EXIT INT TERM`. Never `pkill -f "dotnet ErsatzTV.dll"` (or any pattern that can match a process this run did not start). A foreign listener is reported, not reaped. | 2026-07-25 | [link](records/testing/e2e-cleanup-scope-by-pid.md) |
|
||||
| `testing.e2e-local-fresh-config-dir` | Always point `scripts/e2e-local.sh` at a fresh config dir — leftover channels/schedules/DB rows bleed state between runs and corrupt assertions. (The *readiness-probe hang* this record was originally written about was fixed in #533; the fresh-dir rule stands on state-bleed grounds alone.) | 2026-07-21 | [link](records/testing/e2e-local-fresh-config-dir.md) |
|
||||
| `testing.enumerating-guard-identity-not-position` | A guard that cross-checks a hand-reviewed registry against call sites discovered across the whole repo must key each entry on properties INTRINSIC to the site — file, kind, and the value source text — and never on its absolute line or column. A registry keyed on position is a function of every other file in the repo, so a branch that never touches the guard can invalidate it; and because each PR is green against its own base, that failure is structurally invisible pre-merge and lands on `main` after review and after the merge gate. Dropping the position keeps every mutation the guard exists for — a NEW site, a REMOVED site and a CHANGED value each still fail, since each changes the identity multiset — and costs exactly ONE case, which must be stated rather than implied: a SAME-IDENTITY SUBSTITUTION within one file (delete a registered site, add a different unreviewed one with the same kind and value token, net-zero count) now passes. A REPORTED failure still prints the discovered line:column, because identity and diagnostics need not share a format. Comparison stays a MULTISET count rather than set membership, so two sites in one file sharing an identity must be discovered exactly that many times and a third occurrence still fails. A SCANNER test that asserts real AST positions against FIXED inline fixtures is the opposite case and keeps its line/column identity — it has no churn, because its input does not move. | 2026-07-27 | [link](records/testing/enumerating-guard-identity-not-position.md) |
|
||||
| `testing.fix-ships-a-witnessed-red-test` | A commit claiming to fix something may carry a `Proves: <pytest selector>` trailer; when it does, `scripts/prove-fix.sh` must show that selector GREEN with the fix and RED with the code side reverted, and CI enforces it per-PR. The trailer is opt-in — an unproven commit is allowed — but a claimed proof that does not hold fails the build. | 2026-08-16 | [link](records/testing/fix-ships-a-witnessed-red-test.md) |
|
||||
| `testing.full-replace-asserts-field-list` | Any path that writes a WHOLE entity or a WHOLE child collection — a PUT-replace handler, a hand-built request object, a test comparer standing in for one — derives its field list from the authoritative type and asserts SET EQUALITY against it, rather than enumerating the fields by hand. A hand-written list is correct on the day it is written and structurally unable to report the day it stops being: the field that drifts is the one nobody wrote a line for, so no amount of care in the existing lines can reach it. The failure is silent by construction — a full replace with a field omitted returns HTTP 200 and destroys that field's value (#754 drifted from a 28-property DTO by one and cleared it; the symptom arrived hours later as missing pixels). SECOND CLAUSE, separable from the first: where a replaced child row carries state keyed to its identity — progression, ordering, an enumerator position — the handler RECONCILES BY ID rather than delete-and-reinsert, because reinsertion silently resets state a client never asked to touch (#252: a schedule PUT reset fill-group progression; #500: a dedup fix became permanent data loss because the add filter and the remove filter used different keys, so the two halves must agree on the key). Delete-and-reinsert is acceptable ONLY where no such state exists, and that emptiness is a fact about today's schema that a later feature can silently invalidate — so record it where the handler is, dated, rather than leaving it to be re-derived. The canonical worked example is `ToolCatalogTests.Every_Write_Tool_Should_Declare_Exactly_Its_OpenApi_Request_Body_Fields`, which reads the accepted fields from the generated OpenAPI document and compares both directions. | 2026-08-21 | [link](records/testing/full-replace-asserts-field-list.md) |
|
||||
| `testing.guard-derives-population-from-source` | A guard that asserts a COMPLETENESS property enumerates its population from a machine-readable authoritative source — the enum, the generated OpenAPI document, the parsed workflow YAML, the provider list — and asserts SET EQUALITY in BOTH directions against it. It may not narrow that population with a filter, a `Where`, a `grep` or an early `continue` before the assertion, because a filter cannot see the member that is MISSING: the member whose absence is the defect is precisely the one the predicate excludes. A hand-written literal list of members is the same defect in slower motion — a filter frozen at authoring time, correct on the day it was written and unable to report the day it stopped being. Two boundaries bound the rule rather than weaken it. FIRST, filtering to select the SUBJECT of a PER-MEMBER property is legitimate and is not this defect: the excluded members satisfy the property vacuously, so the filtered walk and the whole walk assert the same thing (`ToolCatalogTests.Every_Query_Parameter_Should_Be_A_Declared_Property` filters to tools that declare query parameters, and a tool declaring none has nothing to check). The defect is filtering the population before a COMPLETENESS claim, which is what makes an absent member unrepresentable (#757 filtered on `QueryParameters is {Count: > 0}` and so could not see a tool that should have declared one and did not). SECOND, a population of VALUES always has an external authoritative source and this rule applies directly; a population of SITES IN CODE has no such list, needs find-all-references tooling, and is tracked separately in #777 — do not stretch a set-equality assertion over it. Distinguish the guard SCOPE (which subsystems it covers — a reviewed policy choice, legitimately hand-written) from the guard POPULATION (the members inside that scope — always derived). When the scope itself MIRRORS an authoritative source, the mirror needs its own equality check or a dated staleness marker, or the guard is complete within a scope that has silently gone stale. The canonical worked example in this repo is `ToolCatalogTests.Every_Tool_Should_Declare_Exactly_Its_OpenApi_Query_Parameters`; the canonical residual gap is `MARKED_JOBS` in `scripts/tests/test_ci_dropped_step_guard.py`. WHEN THE POPULATION IS FILES (#806), the authoritative source is the GIT INDEX and never a filesystem walk. A walk is not merely a weaker enumerator, it answers a question about the MACHINE rather than about the repo: it reports build output, generated shims and editor droppings, and it differs between CI and every checkout, so the same guard asserts a different population in each place. Derive with `git ls-files`, take direct children only unless a nested population is stated and wanted, and assert existence rather than filtering on it, because filtering is what makes a missing member unrepresentable. This is an instantiation and not a blanket rewrite: the question per guard remains whether it makes a COMPLETENESS claim over TRACKED files, and a walk that assembles a fixture or selects the SUBJECT of a per-member property stays a walk with its reason written down. | 2026-08-13 | [link](records/testing/guard-derives-population-from-source.md) |
|
||||
| `testing.guard-ships-with-mutation-proof` | A guard is not considered tested because a test involving it passes. It ships with a MUTATION PROOF: remove or disarm THAT GUARD'S CLAUSE ALONE, and a NAMED test must go red. ONE NAMED EXCEPTION, with its limits, because the rule degenerates without it: where the guard IS a test (a checker enforcing a repo invariant, with no separate script behind it), disarming it makes it ABSENT rather than red, so the proof is the contrapositive — INTRODUCE THE DEFECT THE GUARD EXISTS TO CATCH into an isolated copy of the guarded artifact, and the named test must go red. That is a mutation of the guarded SYSTEM rather than of the assertion, and it is admissible ONLY for checker-guards and ONLY when the mutation was executed and witnessed. It is NOT a licence to grade an ordinary script-guard MUTATION for having a bad-input test: feeding a script an input its clause rejects is BEHAVIOUR-ONLY, which is what three rows were regraded for. A file-level grade under this exception covers the clause its cited case actually mutates, not every assertion that later lands in the same file. Three things this excludes, each of which has already shipped here as a green suite over a dead check. FIRST, a behavioural test — one that feeds the guard a good input and a bad input and checks it passes and fails — proves the guard REACTS, never that it is LOAD-BEARING; #685 had two guards on one condition where deleting either left the whole suite green while every behavioural test passed. SECOND, mutating the WHOLE FILE does not count (#510): a whole-file revert cannot show that a test reaches a particular clause, so the mutation must target the clause. THIRD, the guard being WIRED is not the guard RUNNING — #631's suite was invoked by no CI job, #751's step was dropped by the runner and the job reported success in 6s against a normal 14-17s, and #719's new logic was never connected to stdin. Every guard that DERIVES A POPULATION also carries an ANTI-VACUITY assertion, because the characteristic failure of a completeness check is reporting that it proved everything while its population was empty; a guard with no population has nothing for such an assertion to be about, and stating it universally reads as coverage the unproven rows do not have. Mechanical enforcement is possible for the BOOKKEEPING and not for the JUDGEMENT, and the split is the decision: `docs/guard-inventory.md` lists every guard file with its Kind, its Proof class (`MUTATION`/`BEHAVIOUR-ONLY`/`NONE`) and a `file::function` ref, and `scripts/tests/test_guard_inventory.py` derives the guard population from the GIT INDEX and the call sites (#806), asserts SET EQUALITY against the rows, and resolves every claimed ref to a real `def`. So a new guard cannot ship unclassified and a renamed test cannot leave a row silently claiming coverage. Whether a row claiming `MUTATION` is telling the truth is no longer left to review: `testing.mutation-claims-are-executed` (#790) requires each such row to carry a DECLARED clause mutation that is applied to an isolated copy of the repository on every run, with the row's own named test required to go red. | 2026-08-13 | [link](records/testing/guard-ships-with-mutation-proof.md) |
|
||||
| `testing.hook-reports-its-own-execution` | Every script in `.claude/hooks/` sources `scripts/hook-fire-log.sh` and calls `etv_hook_fire_begin <its-own-name> <label> <capture\|stream>` as its FIRST act, before anything reads stdin. Two records are appended per invocation — a `fire` record on entry and an `exit` record carrying the exit status and the decision — to a session-scoped JSONL log. THE DECISION IS READ FROM WHAT THE HOOK ACTUALLY EMITTED, never declared by the hook author: Claude Code hooks (`capture` mode) always exit 0 and communicate by PRINTING JSON, so their stdout is diverted and replayed, and the recorded decision is parsed from those bytes; git hooks (`stream` mode) decide by EXIT CODE and their stdout is live progress text a human is watching, so it is not diverted and the decision is the status. That split is not a tuning knob — capturing a slow pre-push hook's output would hold it back until the end and read as a hang, and inferring a git hook's decision from absent JSON would put the report back into the guessing business this record exists to end. The population is DERIVED from `.claude/hooks/*.sh` by `scripts/tests/test_hook_fire_log.py`, so a new hook is uninstrumented-and-red rather than silently unobserved, and the report lists every hook that EXISTS rather than every hook that appears in the log — a report built from the log alone can only show hooks that fired, which makes the never-fired hook, the one finding worth having, invisible. THE INSTRUMENTATION MUST BE INVISIBLE TO THE HARNESS, and this is the load-bearing half: it sits in the stdin and stdout path of the most authoritative guards in the repo, so a differential test drives EVERY hook with and without it over a payload matrix and demands byte-equal stdout and equal exit status. It fails OPEN in exactly one direction — if the log cannot be written the hook behaves exactly as before — because observability that breaks a guard is worse than the blindness it replaces. Two mechanical traps are pinned by tests rather than left to care: stdout must be replayed from the FILE, since `out=$(cat f)` strips trailing newlines and delivers a guard's JSON one byte short with no parser anywhere to complain; and stdin must never be slurped when it is a TTY, because an interactive `git commit` hands its hooks a terminal and `cat` would block forever, hanging the commit the instrumentation was added to observe. | 2026-08-14 | [link](records/testing/hook-reports-its-own-execution.md) |
|
||||
| `testing.live-e2e-prepush-timing` | Run live-E2E via `scripts/e2e-local.sh` before pushing a write-path or UI change, and exercise download endpoints with curl, never a browser tab. | 2026-07-21 | [link](records/testing/live-e2e-prepush-timing.md) |
|
||||
| `testing.mutation-claims-are-executed` | A `MUTATION` row in `docs/guard-inventory.md` is not a statement that someone once witnessed a red. It carries a DECLARED clause mutation in `scripts/tests/mutation_manifest.py`, and `scripts/tests/test_mutation_harness.py` applies that mutation to an isolated copy of the repository on every run and requires the row's OWN named test to go red. The manifest and the MUTATION rows are compared for SET EQUALITY in both directions, so a row cannot claim the grade without a mutation and a mutation cannot outlive the grade it justifies. EXIT STATUS IS NOT THE VERDICT: each entry also declares the DIAGNOSTIC its red must carry, matched against pytest's exception output alone, because pytest reports a crashing test exactly as it reports a detecting one and a red for an unrelated reason is evidence about nothing. WHERE THE GUARD IS ITSELF A TEST, `target` may differ from `guard` and the exact-once check applies to the declared TARGET. Two shapes are admissible and the choice is not free. Where the guard's assertion IS the check — a completeness comparison against a Markdown inventory — the mutation goes into the guarded ARTIFACT, per `testing.guard-ships-with-mutation-proof`'s checker-guard exception, because mutating such a checker's own POPULATION demonstrates a false POSITIVE while proving nothing about the detection the row claims. Where the guard is a test module wrapping a separately mutable DETECTOR or helper, the clause may be in that detector, since disarming it is a real clause disarm and the module's own assertion is what notices. THE MUTATION IS DECLARED, NEVER INFERRED: a harness that guessed which clause of a 90-line hook is the guard would manufacture the confident-but-empty coverage this exists to prevent, which is why `testing.guard-ships-with-mutation-proof` rejected a generic runner. Where a proof test already names its clause in source, the manifest reuses THAT string, so a retarget in either place is caught by the other. COARSENESS IS RECORDED, NOT HIDDEN: each entry is graded `CLAUSE` or `DETECTOR`, and a `DETECTOR` entry — one whose detector accumulates faults from independent arms, so disarming any single arm leaves its proof test green — must CARRY the finer mutation that survived, which is re-run every time and required to keep surviving. Guards that are not graded `MUTATION` each carry a STATED reason in that same manifest, keyed on the guard and compared for SET EQUALITY against the inventory's `GUARD` rows in both directions — so a new guard cannot arrive without someone writing what a proof would need, and a reason cannot outlive the row it is about. Keying the reason on the row's GRADE instead is tautological (a new guard inherits one and nobody looks at it) and a pinned COUNT moves only on net change; both were tried and are rejected. The sandbox is a real git repository built from `git ls-files` with working-tree content, never a filesystem walk. | 2026-08-22 | [link](records/testing/mutation-claims-are-executed.md) |
|
||||
| `testing.playwright-mcp-download-and-recovery` | In Playwright-MCP E2E, fetch file-download endpoints with curl — never a browser tab or `window.open` — and if browser tools stall repeatedly, `pkill -f ms-playwright-mcp` and drive a fresh session. | 2026-07-21 | [link](records/testing/playwright-mcp-download-and-recovery.md) |
|
||||
| `testing.scripted-playout-golden-deferred` | The `PlayoutBuildGoldenTests` in-memory golden net covers Sequential (YAML) as of #381. Scripted's *end-to-end pipeline* is excluded — `ScriptedPlayoutBuilder` runs a user-authored external program that drives the engine over HTTP loopback, which the in-memory harness can't pin — so that full-pipeline (integration) harness is deferred to #563. But the scheduling *behavior* those scripts drive lives entirely in the in-process `SchedulingEngine` (the `ScriptedScheduleController` is a 1:1 pass-through to it), which IS directly unit/golden-testable; the earlier "Scripted is un-golden-able by construction" framing overstated the constraint by conflating transport with engine. #395 extracts that shared switch to `ContentEnumeratorBuilder` and adds a direct regression net (`ContentEnumeratorBuilderTests`) over it. | 2026-07-22 | [link](records/testing/scripted-playout-golden-deferred.md) |
|
||||
| `testing.troubleshoot-path-cannot-test-branding` | Verify logo/watermark/bug changes through a real channel playout — a green troubleshoot run proves nothing about branding. | 2026-07-21 | [link](records/testing/troubleshoot-path-cannot-test-branding.md) |
|
||||
|
||||
@@ -0,0 +1,89 @@
|
||||
---
|
||||
key: ci.actions-credential-scoping
|
||||
title: '2026-08-05 — CI''s registry credential is a scoped PAT, not the admin password, because Gitea cannot separate status-write from repo-write (#697)'
|
||||
status: active
|
||||
since: '2026-08-05'
|
||||
supersedes: none
|
||||
superseded-by: none
|
||||
rule: 'Any credential reachable from an Actions job is scoped to what that job needs. The container-registry secret `REGISTRY_PASSWORD` is a personal access token scoped `write:package` + `read:repository` — never an account PASSWORD. This matters because Gitea has NO `status` token scope: `POST /repos/{o}/{r}/statuses/{sha}` is gated by `reqRepoWriter(unit.TypeCode)`, so ANY credential that can write the repository can forge `review-verdict/h10`, the required context that is supposed to make merge-consent derived rather than assertable. Package-write IS a separate scope, so the registry credential can be made status-incapable at no cost: `scripts/ci-detect-already-validated.sh` only GETs. Do NOT add a `permissions:` key to constrain the injected `GITEA_TOKEN` on the assumption that it binds — below Gitea 1.26.0 it is silently a NO-OP, which is worse than absent because it reads in review as a constraint. That version precondition NO LONGER HOLDS: this instance was upgraded 1.25.4 -> 1.27.1 on 2026-08-05. What has NOT changed is that the consequence is unverified — whether `permissions:` is honored here, and what this instance''s default Actions token permission is, were both left UNPROBED (there is still no API surface: `/api/v1/settings/actions` 404s at 1.27.1). Probe before relying on it; do not read the upgrade alone as the constraint now working. Scoping is necessary and not sufficient: it bounds what a job may DO, never whether attacker YAML runs at all, so a self-referencing trigger needs its own filter (`ci-image.yml`, tracked in #744 — deliberately NOT bundled here, because editing that file re-points `ci-image-pin` at the editing commit and reddens a blocking job). This record closes ONE route. It does not close the class, and four later sections say exactly what survives — read them before citing this record as a mitigation.'
|
||||
signals: 'admin password in CI secrets, registry credential scope, ETV_STATUS_AUTH can write statuses, forge review-verdict/h10, head-resolved workflow holds credentials, Gitea token scopes, no status scope, write:package vs write:repository, permissions key no-op, GITEA_TOKEN default read/write, Restricted default token permissions, orphan secret, deploy key in secret store, toolchain image overwrite, prod floating tag write · paths: `.gitea/workflows/docker-build.yml`, `.gitea/workflows/ci-image.yml`, `.gitea/workflows/renovate.yml`, `scripts/ci-detect-already-validated.sh` · issues: #697, #672, #698, #742, #743, #420, server-management#714'
|
||||
mechanics: 'PAT `ci-registry-scoped-697`, scopes `write:package,read:repository`, stored as repo Actions secret `REGISTRY_PASSWORD`; `REGISTRY_USER` remains `timothy`. Verified 2026-08-05 on Gitea 1.25.4: registry push of a probe tag SUCCEEDED; `GET /commits/{sha}/status` 200; `POST /statuses/{sha}` REFUSED HTTP 403 `token does not have at least one of required scope(s), required=[write:repository], token scope=write:package,read:repository`. Probe artifacts deleted, confirmed 404. NOT measured with this token: the `container:` pull, the buildcache write and the base-image pull. Those rest on Gitea''s scope model (write implies read per category, read at tag `v1.25.4`) — INFERRED. Note WHICH run proves which: only the `container:` pull is exercised by a PR. `cache-to`/`cache-from` and the base-image pull are confined to the `build` job, which carries `if: github.event_name != ''pull_request''`, so they are first exercised on the post-merge push to `main` — AFTER the merge gate has passed. A wrong inference there reddens main, not the PR.'
|
||||
---
|
||||
|
||||
**What was wrong.** `REGISTRY_USER`/`REGISTRY_PASSWORD` were the **admin account's** basic auth, and
|
||||
`docker-build.yml` triggers on `pull_request` — head-resolved — so a PR's own code got instance-admin
|
||||
credentials. Basic auth carries no scope: the secret pushing an image administers every repo on the
|
||||
instance.
|
||||
|
||||
**Why the credential and not only the triggers.** Patching triggers enumerates *instances* of "a
|
||||
ref-resolved workflow obtains status-capable credentials", and adding a new workflow file is itself a
|
||||
route, so that enumeration never completes. But it is not either/or: `ci-image.yml`'s unfiltered
|
||||
`push:` is path-scoped to itself, so any branch push runs attacker YAML on a docker-capable runner
|
||||
with no PR. Scoping bounds what a job may DO; only a filter bounds whether it RUNS. That filter is
|
||||
**#744**, not this record: editing `ci-image.yml` re-points `ci-image-pin`'s `expected` at the editing
|
||||
commit and staleness-fails a **blocking** job. That is a toll, not a wall — the documented two-step
|
||||
(publish `:<short sha>`, then bump all five pins) clears it — but a rebase rewrites the sha and charges
|
||||
it again, so it lands alone (`land-toolchain-image-change-separately`).
|
||||
|
||||
**What the scoped token still reaches — not "just a registry credential".** `write:package` over owner
|
||||
`timothy` writes `ersatztv:prod` (the floating tag prod's `jazz-media` stack follows) and
|
||||
`ersatztv-ci:<sha>` (the toolchain image *executing* five `container:` jobs). A sha-named tag is not an
|
||||
immutable artifact (no container tag immutability in Gitea 1.25 — INFERRED), so overwriting the pinned
|
||||
tag is code execution inside CI, chaining back into the routes below. This is the deployment supply
|
||||
chain for prod and CI itself.
|
||||
|
||||
**Admin ownership is a real residual.** The PAT is minted under `timothy`, a site admin. The 403 proves
|
||||
the scope gate binds the *status* endpoint ahead of any admin bypass; it does NOT establish that for
|
||||
*package* endpoints, where Gitea resolves permission by owner and an admin passes object-level checks,
|
||||
so the token's package reach is plausibly wider than this repo. A non-admin bot account would close
|
||||
this, but is not free: packages live in a user namespace only its owner and admins can write. Both
|
||||
halves INFERRED, neither probed.
|
||||
|
||||
**Provenance, corrected.** `review-verdict.yml` leaves an existing `h10` alone only when it is
|
||||
positively identifiable as human — non-null `.creator.login` plus a `Review-verdict:` description
|
||||
(`release.verdict-status-check`). A user credential posts with a real creator and is INHERITED; an
|
||||
Actions job posts `creator: null` and is re-derived. **That asymmetry is not protection.** Re-derivation
|
||||
fires only on `opened|reopened|synchronize|ready_for_review|edited`, and posting a status is none of
|
||||
them, so a POST timed after the last event stands until the attacker merges. The gain here is that PR
|
||||
code can no longer escalate to instance admin — NOT that the durable forgery route is closed.
|
||||
|
||||
**The boundary is everything reachable from a job, not the secret store.** The store is a useful lower
|
||||
bound — auditing it rather than the workflow set is what found `RENOVATE_TOKEN` and
|
||||
`SERVERMGMT_DEPLOY_KEY` below, since any PR-added workflow can reference any secret. But
|
||||
`GITEA_TOKEN` is injected and never in the store; nor is the credential
|
||||
`actions/checkout` persists into `.git/config` (`docker-build.yml` omits `persist-credentials: false`);
|
||||
and jobs reach the runner's docker daemon.
|
||||
|
||||
**Measured vs inferred.** Measured here: the `v1.25.4` scope enum (`access_token_scope.go`) has no
|
||||
`status` entry; the `reqRepoWriter` gate (`routers/api/v1/api.go`); the probes in `mechanics`. Read from
|
||||
docs, NOT verified (2026-08-05): `permissions:` landed in 1.26.0 (Gitea PR #36173); no `app.ini` lever
|
||||
at any version; Gitea rejects GitHub's `statuses`/`checks` scopes.
|
||||
|
||||
**Version caveat — this record's measurements are pinned to 1.25.4, the instance is now 1.27.1.**
|
||||
The instance was upgraded mid-session on 2026-08-05 (#743). Everything above measured on 1.25.4 is
|
||||
therefore a *dated* claim, not a current one: the scope enum, the `reqRepoWriter` gate and the 403
|
||||
probe were all taken pre-upgrade and have NOT been re-run. They are recorded honestly as of their
|
||||
date and are the best evidence available, but do not cite them as current behaviour without
|
||||
re-probing. Re-verification of the 1.25.4-pinned claims across the CI docs is tracked separately.
|
||||
|
||||
**Surviving routes — this record is not a mitigation for any of them.** `RENOVATE_TOKEN` is a
|
||||
`write:repository` bot PAT in the same store, posting with a real creator, and cannot be scoped down
|
||||
because Renovate needs repo write (#742). The injected `GITEA_TOKEN` is write-capable in every job;
|
||||
only Gitea >=1.26 with the Actions default set to **Restricted** binds it (server-management#714) —
|
||||
the version half of that condition is now satisfied (1.27.1) but the *default* half is unverified, so
|
||||
treat this route as still open until probed. A
|
||||
collaborator's own token always can. `docker-build.yml` publishes `:prod` from a `v*` tag push and a tag
|
||||
may point at ANY commit — a prod image with no PR, review or status (tag protections are empty).
|
||||
**And none of it was necessary: direct pushes to `main` were server-side permitted, so the gate was
|
||||
bypassable with no forgery at all (#743).** That route is now closed — `main` carries
|
||||
`enable_push: false` (`release.main-direct-push-disabled`), which removes `main` as a destination for
|
||||
every write-only credential in this list, including the injected `GITEA_TOKEN` and `RENOVATE_TOKEN`.
|
||||
It does not remove them as *forgery* routes on the PR path, and it does not bind an admin credential,
|
||||
which can PATCH the protection off first. Correction to this record's earlier wording: a push
|
||||
*whitelist* would NOT have closed more of the class than the upgrade — measured 2026-08-05, a
|
||||
whitelist naming `timothy` still admitted the push, and every credential here acts as `timothy`.
|
||||
Treat this list as "at least these", never exhaustive. `SERVERMGMT_DEPLOY_KEY` remains in the
|
||||
store though its `bump-prod-compose` job went in `1b5efd7b9`, and its key on `timothy/server-management`
|
||||
is `read_only: false` — write access to the repo holding prod's GitOps stack definitions. Left in place
|
||||
by explicit decision 2026-08-05; recorded so it is accepted, not forgotten. Severity throughout: push
|
||||
access required, so a compromised contributor or subverted automated session, never an anonymous one.
|
||||
@@ -5,8 +5,8 @@ status: active
|
||||
since: '2026-07-21'
|
||||
supersedes: none
|
||||
superseded-by: none
|
||||
rule: Treat a `cancelled` conclusion as "no verdict" — never as pass or fail — and report FAILED and CANCELLED counts separately in any CI monitor.
|
||||
signals: 'conclusion cancelled · run-level vs job-level conclusion · pre-cancel genuine failure · CI monitor state != pending · phantom failure · paths: n/a · issues: #542'
|
||||
rule: 'Treat a `cancelled` conclusion as "no verdict" — never as pass or fail — and report FAILED and CANCELLED counts separately in any CI monitor. THE COMBINED COMMIT-STATUS ENDPOINT CANNOT EXPRESS THIS: `GET /repos/{o}/{r}/commits/{sha}/status` has states `success`/`failure`/`pending`/`error` and NO `cancelled`, so it reports a cancelled job as `failure`. Anything polling that endpoint — which is what a CI monitor naturally polls, because it is the per-sha view the merge gate reads — must resolve the job-level `conclusion` via `actions/runs/{id}/jobs` before reporting a red.'
|
||||
signals: 'conclusion cancelled · run-level vs job-level conclusion · pre-cancel genuine failure · CI monitor state != pending · phantom failure · commit-status endpoint has no cancelled state · combined status reports cancelled as failure · resolve job conclusion before reporting a red · paths: n/a · issues: #542, #790'
|
||||
mechanics: Gitea Actions run/job API; monitor logic, e.g. `fail=[j for j in jobs if j['conclusion']=='failure']; canc=[j for j in jobs if j['conclusion']=='cancelled']`.
|
||||
---
|
||||
|
||||
@@ -19,3 +19,15 @@ so never claim green on one.
|
||||
|
||||
A monitor that only asks "is state != pending" will report a cancelled run as a failure and send the
|
||||
next session debugging a phantom. Split the two counts explicitly.
|
||||
|
||||
**The endpoint most monitors poll cannot express the distinction at all.** `commits/{sha}/status` is
|
||||
the per-sha view — the one the merge gate reads and the natural thing to watch a PR head with — and
|
||||
its vocabulary is `success`/`failure`/`pending`/`error`. A cancelled job arrives there as `failure`.
|
||||
So "split the counts" is not implementable against that endpoint: the job-level `conclusion` has to
|
||||
be fetched from `actions/runs/{id}/jobs`, with the run id taken from the status entry's `target_url`.
|
||||
|
||||
Measured on ersatztv#790 (2026-08-22): three jobs reported `failure` on a head where everything that
|
||||
ran had passed. They had been auto-cancelled by the author's own next push. The cause is usually
|
||||
self-inflicted, which is the other half of the cost `ci.batch-pushes-no-cancel-route` and
|
||||
`process.local-gate-before-push` describe — an early push does not merely waste a runner slot, it
|
||||
manufactures reds that look like they belong to the diff.
|
||||
|
||||
@@ -5,7 +5,7 @@ status: active
|
||||
since: '2026-07-29'
|
||||
supersedes: none
|
||||
superseded-by: none
|
||||
rule: 'The three inputs the exemption decision rests on must each be bound to something the judged PR cannot mutate. (1) BASE — `scripts/pr-changed-files.sh` takes the expected base BRANCH as a REQUIRED 5th argument and re-reads it before and after paging, because `/pulls/{n}/files` diffs against the PR''s live base and retargeting moves the answer without moving the head sha; the workflow passes `github.event.pull_request.base.ref` from the `pull_request_target` payload, which a retarget cannot rewrite. (2) BOT EXEMPTION — an author match is necessary but never sufficient: `pull_request.user.login` is the PR''s immutable CREATOR while its head is not, so the exemption additionally requires EVERY changed path to be a dependency manifest (`Directory.Packages.props` or `.config/dotnet-tools.json`, and ONLY those — the npm manifests are excluded because `package.json` `scripts` are executed by CI). (3) INHERITED SUCCESS — the never-overwrite short-circuit fires only for a status POSITIVELY identified as a human verdict for THIS base, meaning a non-null `.creator.login` AND a `Review-verdict:` description AND, when that description records a base (`(base: …)`, `release.verdict-status-check`), a base matching the PR''s — tested by requiring the description to END with the exact literal `(base: <base>)` and to contain exactly ONE such marker, never by extracting a value (see below); a present-but-different base is rejected, an absent one is not, since verdicts predating that convention carry none; every other shape, including any unrecognised one, is re-derived rather than trusted. The bot and docs-only exemptions are evaluated as INDEPENDENT predicates and the decision made afterwards, never as an `elif` chain. `edited` is in the workflow''s `types:` so a retarget reclassifies — which gives DETECTION, not atomicity: status writes are not serialized, so a stale run can still post over a fresher one. That residual is now FENCED rather than merely tracked — the job refuses to write at all if the PR''s timeline retarget COUNT moved while it was classifying (`ci.verdict-write-retarget-fence`, #706) — leaving only the sub-round-trip window that no API without compare-and-set can close. The PROTECTED path list additionally covers `.codex/` (#711), which mirrors `.claude/hooks/` byte for byte including the merge-consent hook — latent while that directory is untracked, live the moment it is tracked; the list stays ENUMERATIVE rather than derived, because a derived rule would have to be evaluated against the very file list being classified. Path predicates are evaluated by COUNTING with `grep -c`, never `| grep -q` (SIGPIPE inversion) and never a here-string (temp-space failure) — see `ci.grep-q-pipefail-inversion`.'
|
||||
rule: 'The three inputs the exemption decision rests on must each be bound to something the judged PR cannot mutate. (1) BASE — `scripts/pr-changed-files.sh` takes the expected base BRANCH as a REQUIRED 5th argument and re-reads it before and after paging, because `/pulls/{n}/files` diffs against the PR''s live base and retargeting moves the answer without moving the head sha; the workflow passes `github.event.pull_request.base.ref` from the `pull_request_target` payload, which a retarget cannot rewrite. (2) BOT EXEMPTION — an author match is necessary but never sufficient: `pull_request.user.login` is the PR''s immutable CREATOR while its head is not, so the exemption additionally requires EVERY changed path to be a dependency manifest (`Directory.Packages.props` or `.config/dotnet-tools.json`, and ONLY those — the npm manifests are excluded because `package.json` `scripts` are executed by CI). (3) INHERITED SUCCESS — the never-overwrite short-circuit fires only for a status POSITIVELY identified as a human verdict for THIS base, meaning a non-null `.creator.login` AND a `Review-verdict:` description AND, when that description records a base (`(base: …)`, `release.verdict-status-check`), a base matching the PR''s — tested by requiring the description to END with the exact literal `(base: <base>)` and to contain exactly ONE such marker, never by extracting a value (see below); a present-but-different base is rejected, an absent one is not, since verdicts predating that convention carry none; every other shape, including any unrecognised one, is re-derived rather than trusted. The bot and docs-only exemptions are evaluated as INDEPENDENT predicates and the decision made afterwards, never as an `elif` chain. `edited` is in the workflow''s `types:` so a retarget reclassifies — which gives DETECTION, not atomicity: status writes are not serialized, so a stale run can still post over a fresher one. That residual is now FENCED rather than merely tracked — the job refuses to write at all if the PR''s timeline retarget COUNT moved while it was classifying (`ci.verdict-write-retarget-fence`, #706) — leaving only the sub-round-trip window that no API without compare-and-set can close. The PROTECTED path list additionally covers `CLAUDE.md` and `AGENTS.md` (#751) — they are not prose but the documents DEFINING the completion protocol, the merge-consent convention and the H10 rule, so protecting `.claude/` while the file specifying what it enforces stayed docs-only-exempt was the same self-exemption one directory over; driving the real classify body with a lone `CLAUDE.md` change produced an exemption `success`. `README.md` is deliberately not listed. It also covers `.codex/` (#711), which mirrors `.claude/hooks/` byte for byte including the merge-consent hook — latent while that directory is untracked, live the moment it is tracked; the list stays ENUMERATIVE rather than derived, because a derived rule would have to be evaluated against the very file list being classified. Reading the CURRENT status for input (3) must tolerate `statuses: null`: `GET /commits/{sha}/status` serialises a nil slice as `null`, not `[]`, on a head with no statuses yet, and an `array`-only gate made `read_existing_verdict` `exit 1` and post nothing at all (#751, `ci.workflow-run-body-no-expressions`) — `null` is accepted only when `total_count` is 0, so a body that merely lost its array is still refused. Path predicates are evaluated by COUNTING with `grep -c`, never `| grep -q` (SIGPIPE inversion) and never a here-string (temp-space failure) — see `ci.grep-q-pipefail-inversion`.'
|
||||
signals: 'forged review-verdict exemption, retarget race against the docs-only classifier, PR base changed mid-run, hijacked Renovate branch, bot exemption on a code change, machine-written success inherited as a verdict, status creator null vs user, never overwrite a human verdict, exemption chain skips docs-only for bots, why is my Renovate PR asking for a verdict, base ref binding on pr-changed-files.sh · paths: `.gitea/workflows/review-verdict.yml`, `scripts/pr-changed-files.sh`, `.claude/hooks/pretooluse-merge-consent.sh`, `scripts/tests/test_pr_changed_files.py` · issues: #698, #697, #672, #663, #649, #632'
|
||||
mechanics: '`scripts/pr-changed-files.sh <owner> <repo> <pr> <expected-head-sha> <expected-base-ref>` (5 args; a 4-arg call exits 2); workflow env `BASE_REF: ${{ github.event.pull_request.base.ref }}`; `BOT_MANIFESTS` anchored allow-list; short-circuit requires `.creator.login` non-null AND description matching `^Review-verdict:`; `types: [opened, reopened, synchronize, ready_for_review, edited]`'
|
||||
---
|
||||
@@ -79,9 +79,10 @@ test still fails: the guard degrades toward re-deriving, never toward trusting.
|
||||
|
||||
**What this does NOT close.** Anyone who can POST statuses directly can write both a creator and a
|
||||
`Review-verdict:` description and impersonate a verdict; branch protection binds the *context*, not its
|
||||
issuer. A provenance check, not an authentication one — that is `#697`, left open because its durable
|
||||
fix is credential scoping, partly server-management territory. Severity as `#672`: requires push
|
||||
access, so the threat model is a compromised contributor.
|
||||
issuer. A provenance check, not an authentication one — that was `#697`'s registry credential, fixed by
|
||||
scoping it off `write:repository` (`ci.actions-credential-scoping`); `GITEA_TOKEN`, `RENOVATE_TOKEN`,
|
||||
and a collaborator's own token still can. Severity as `#672`: requires push access, so the threat model
|
||||
is a compromised contributor.
|
||||
|
||||
**Verification honesty.** Route 1 was reproduced live; the "and now it fails" half cannot be shown from
|
||||
a PR, because `pull_request_target` resolves this definition from `main` — the self-test gap
|
||||
|
||||
@@ -5,7 +5,7 @@ status: active
|
||||
since: '2026-07-28'
|
||||
supersedes: none
|
||||
superseded-by: none
|
||||
rule: 'The workflow that writes the branch-protection-required `review-verdict/h10` status triggers on `pull_request_target` with `branches: [main]`, never on plain `pull_request`. Gitea resolves a `pull_request` workflow DEFINITION from the PR''s own head commit, so under that trigger a PR editing `.gitea/workflows/review-verdict.yml` ran its own rewritten copy and could post `h10=success` for itself; `pull_request_target` resolves the definition from the base instead. The `branches: [main]` filter is part of the rule, not a refinement of it: base resolution only relocates the rewrite from the head to the base, so without the filter a PR opened into an attacker-pushed base branch runs that branch''s gate. `pull_request_target` is safe HERE only because this job never checks out or executes head-supplied code — it checks out `base.sha` and runs only that tree''s scripts (`ci.shared-pr-file-enumeration`); reintroducing a head checkout under this trigger would be worse than the bug it fixed. This closes the rewrite route through THIS workflow and does NOT close the class: Gitea injects a write-capable `GITEA_TOKEN` into EVERY job, so any ref-resolved workflow — and a collaborator''s own API token, since branch protection binds the context and not its issuer — can still forge `review-verdict/h10`. Tracked in #697; the exemption path has its own separate defects in #698.'
|
||||
rule: 'The workflow that writes the branch-protection-required `review-verdict/h10` status triggers on `pull_request_target` with `branches: [main]`, never on plain `pull_request`. Gitea resolves a `pull_request` workflow DEFINITION from the PR''s own head commit, so under that trigger a PR editing `.gitea/workflows/review-verdict.yml` ran its own rewritten copy and could post `h10=success` for itself; `pull_request_target` resolves the definition from the base instead. The `branches: [main]` filter is part of the rule, not a refinement of it: base resolution only relocates the rewrite from the head to the base, so without the filter a PR opened into an attacker-pushed base branch runs that branch''s gate. `pull_request_target` is safe HERE only because this job never checks out or executes head-supplied code — it checks out `base.sha` and runs only that tree''s scripts (`ci.shared-pr-file-enumeration`); reintroducing a head checkout under this trigger would be worse than the bug it fixed. This closes the rewrite route through THIS workflow and does NOT close the class: Gitea injects a write-capable `GITEA_TOKEN` into EVERY job, so any ref-resolved workflow — and a collaborator''s own API token, since branch protection binds the context and not its issuer — can still forge `review-verdict/h10`. The credential half is now RESOLVED in `ci.actions-credential-scoping` (#697): CI''s registry secret was the ADMIN account''s basic auth and is now a PAT that cannot post a status, which removes the ADMIN escalation and that credential''s route (a user credential''s forgery carries a real `creator` and is inherited as a human verdict; an Actions job''s carries `creator: null` and is re-derived — but do NOT read that asymmetry as protection: re-derivation fires only on the trigger''s `types`, and posting a status is not one of them, so a POST timed after the last PR event simply stands). It does not remove EVERY route: `RENOVATE_TOKEN` is a `write:repository` bot PAT in the same secret store, reachable by any PR-added workflow. The injected token stays write-capable until Gitea >=1.26 with a Restricted default (server-management#714), and a collaborator''s own token remains unfixable; the exemption path has its own separate defects in #698.'
|
||||
signals: 'workflow definition resolved from head, PR rewrites the gate that judges it, self-approve a required status check, pull_request_target vs pull_request, gate trigger branches filter, attacker-supplied base branch, how to test a change to review-verdict.yml, workflow not exercised by its own PR, gate edit goes live only on merge, required_approvals 0 does not bind an author, forged commit status inherited by sha · paths: `.gitea/workflows/review-verdict.yml`, `scripts/tests/test_pr_changed_files.py` · issues: #672, #663, #649, #622'
|
||||
mechanics: '`on: pull_request_target: {branches: [main], types: [opened, reopened, synchronize, ready_for_review, edited]}` (`edited` added by `ci.exemption-provenance` so a retarget reclassifies); asserted by `test_the_workflow_trigger_is_pull_request_TARGET_scoped_to_main` in `scripts/tests/test_pr_changed_files.py`; the job''s own context is renamed to `... (pull_request_target)` and must stay OUT of branch protection''s required list'
|
||||
---
|
||||
@@ -54,7 +54,11 @@ inventory is not a short list: Gitea injects `GITEA_TOKEN` into **every** job, d
|
||||
read/**write**, so head-resolved, `push`-triggered and `workflow_dispatch` workflows alike are routes
|
||||
(1.24+ loads a dispatched definition from the selected branch). A collaborator's own API token is a
|
||||
route with no workflow at all — branch protection binds the *context*, not its issuer. Full inventory
|
||||
in `#697`; the exemption path's own defects are `#698`. No in-repository test can establish
|
||||
in `#697`, whose credential half is resolved in `ci.actions-credential-scoping` — the registry secret
|
||||
no longer carries status-write. That does NOT leave the workflow routes provenance-free: any
|
||||
PR-added workflow can reference `RENOVATE_TOKEN`, a `write:repository` bot PAT in the same store,
|
||||
whose status carries a real creator and IS inherited (`#742`). The exemption path's
|
||||
own defects are `#698`. No in-repository test can establish
|
||||
status-authority isolation: the sibling guard added here catches only plain-text naming of the
|
||||
context.
|
||||
|
||||
|
||||
@@ -0,0 +1,97 @@
|
||||
---
|
||||
key: ci.python-lint-ruff-config-committed
|
||||
title: 2026-08-21 — Python lint is a committed ruff.toml enforced in CI, not the operator machine's global config (#780)
|
||||
status: active
|
||||
since: '2026-08-21'
|
||||
supersedes: none
|
||||
superseded-by: none
|
||||
rule: The repo commits `ruff.toml`, and the `script-tests` job runs `ruff check` + `ruff format --check` under a PINNED ruff over an EXPLICIT population from `git ls-files`, never `ruff check .`. Never rely on `~/.config/ruff/ruff.toml`, and never add a lint rule to the config without making the tree clean against it in the same PR.
|
||||
signals: 'ruff · pyright · python lint · `ruff format --check` · lint passes on my machine but not yours · no repo lint config · S105 on a test stub credential · paths: `ruff.toml`, `.gitea/workflows/pr-checks.yml` · issues: #780, #773, #648, #512'
|
||||
mechanics: 'Config at repo root; `.gitea/workflows/pr-checks.yml` -> `script-tests` pins `ruff==0.12.11` via pip and runs both commands ahead of the jq preflight and pytest. Population is `git ls-files -z ''*.py'' ''*.pyi'' ''*.ipynb''` passed explicitly with `--no-force-exclude`, guarded by an empty-list arm; discovery-based invocation is defeated by an `exclude` in three config scopes, two of them per command. Bumping the pin is a deliberate PR because a new ruff release adds rules.'
|
||||
---
|
||||
|
||||
The global instructions tell every session to run `ruff check`, `ruff format --check` and `pyright`
|
||||
after touching Python. Before this, the repo enforced none of them and committed no config, so ruff
|
||||
fell back to whichever `~/.config/ruff/ruff.toml` the operator's machine happened to have — **a second
|
||||
machine lints this repo differently, or not at all.** That is the same shape as #643/#647/#648 (a
|
||||
shell gate whose behaviour was a function of an untested interpreter version) and #512 (a test that
|
||||
passed on a fast laptop and flaked on a starved CI VM): the verdict was a property of the environment
|
||||
rather than of the repo.
|
||||
|
||||
The committed config is the operator's global one apart from `per-file-ignores`, which is narrowed
|
||||
to `scripts/tests/**`. That is what the tree was de-facto written against, so adopting it cost a
|
||||
mechanical reformat rather than a rewrite: 74 findings against `706674272`, of which 57 were fixed in
|
||||
code (mostly by the format pass) and 17 carry a per-site `# noqa` with its reason inline. `RUF100` is
|
||||
selected so those suppressions stay honest — a `# noqa` that suppresses nothing is otherwise
|
||||
invisible, and three were live the moment the rule was switched on: one whose rule had stopped firing,
|
||||
one for a rule this config never enables, and one added mid-branch on a site the same branch had
|
||||
already fixed in code.
|
||||
|
||||
**One exemption is directory-wide, and it is the boring one.** `S101` for `scripts/tests/**`, because
|
||||
a test suite asserts. **`S105` is deliberately NOT directory-wide.** All eight of its hits among
|
||||
those 74 findings are stub credentials handed to the real hooks (`env["ETV_GITEA_TOKEN"] = "stub"`),
|
||||
with no true positive in the tree today (a ninth `# noqa: S105` predates this and sits on a
|
||||
commit-message marker in `decisions_validate.py`). A directory blanket would give up
|
||||
hardcoded-credential coverage over the largest Python surface in the repo, permanently, to suppress
|
||||
eight known lines — and this is the only Python lint the repo runs, so nothing else would catch a real
|
||||
token pasted into a fixture next year. Per-site `# noqa: S105` costs the same and keeps the rule live.
|
||||
|
||||
**The population comes from `git ls-files`, not from ruff's discovery, and that is the load-bearing
|
||||
part.** `ruff check .` reports on what it *discovers*, and an `exclude` defeats discovery in three
|
||||
different config scopes — including `[format]`, which is where an appended line lands by TOML rules (two of the
|
||||
three defeat each command). Measured with ruff 0.12.11 and `exclude = ["scripts/**"]`, against a
|
||||
tracked file holding an unused import, a hardcoded credential and a formatting error. The pattern
|
||||
matters: `exclude` is matched per FILE, so a bare `["scripts"]` works at the top level but matches
|
||||
nothing under `[lint]`/`[format]`. GREEN means the gate was silently off:
|
||||
|
||||
| `exclude` in | `ruff check .` | explicit `check` | `ruff format --check .` | explicit `format` |
|
||||
|---|---|---|---|---|
|
||||
| top level | GREEN | red | GREEN | red |
|
||||
| `[lint]` | GREEN | red | red | red |
|
||||
| `[format]` | red | red | GREEN | red |
|
||||
| top + `force-exclude` | GREEN | GREEN without `--no-force-exclude`, red with it | GREEN | same |
|
||||
|
||||
Only the top-level scope empties both discovery commands; `[lint]` empties `check`, `[format]` empties
|
||||
`format --check`, so in those two the job would still redden on the other step. `[format]` is where a
|
||||
line appended to `ruff.toml` lands, by TOML rules. The last row is the whole reason for the flag.
|
||||
|
||||
`include = []`, `extend-exclude` and a nested `scripts/ruff.toml` were tried too, and are equally
|
||||
inert against the explicit form. The empty-list arm is the anti-vacuity
|
||||
check — `ruff check .` over no files exits **0** with a stderr warning, so an emptied population is a
|
||||
green gate, not a red one. Enumerating from git also covers tracked-but-gitignored files, which
|
||||
discovery skips (`git add -f` under an ignored path is established practice here).
|
||||
|
||||
**The rule set is not covered, and that is a stated limit rather than an oversight.** `select = []`
|
||||
silences every selected rule, so the `ruff check` step goes green over any lint violation (a syntax
|
||||
error still reds) while still printing a reassuring file count. `ruff format --check` is unaffected, because formatting is not rule-selected. So the population
|
||||
arm makes an emptied *file* set loud, nothing makes an emptied *rule* set loud, and half the gate is
|
||||
killable by a config edit only a reviewer catches.
|
||||
|
||||
Both steps were witnessed red on the runner before merge, not argued to work — **on the body that
|
||||
shipped**: run 2173 job 9176 (`❌ Failure - Main Lint scripts (ruff check)` on an `F401`) and run 2170
|
||||
job 9163 (`❌ Failure - Main Lint scripts (ruff format --check)`), printing `Linting 34 tracked
|
||||
Python files` and `Format-checking 34 tracked Python files` — the population arm executing (34 = the
|
||||
33 tracked files plus the probe; the merged tree has 33). Each came from a temporary probe commit
|
||||
reverted before merge. Two probes are needed, not one: a check-dirty file stops the job
|
||||
before the format step ever runs. Earlier reds against the previous, discovery-based bodies were
|
||||
discarded rather than cited — a proof belongs to the code that ran, not to its predecessor.
|
||||
|
||||
**Lint runs early in the job, ahead of the jq preflight.** `Preflight jq version` is a hard `--expect`
|
||||
tripwire; a lint step behind it stops running for as long as the jq contract is broken, under a red
|
||||
that names jq. Ordering is the difference between a gate that is skipped and one that is not. The
|
||||
`git` half of `Preflight external tools` stays *ahead* of the lint steps, because they consume `git`:
|
||||
without it, a missing git arrives as an empty population and both steps report a population problem
|
||||
instead of the missing tool.
|
||||
|
||||
**`pyright` is deliberately NOT gated.** Its only findings here are `reportMissingImports` for
|
||||
`etv_client` in `scripts/scripted-schedules/entrypoint.py`, resolvable only inside that script's
|
||||
deploy environment. Gating it would put a node toolchain on the git-only `small` lane to find nothing.
|
||||
Revisit when this repo grows a typed Python surface — the reason is the cost/finding ratio today, not
|
||||
a judgement that type checking does not belong.
|
||||
|
||||
**The pin is the second half of the fix.** An unpinned `pip install ruff` re-introduces exactly the
|
||||
divergence the config closes, one layer up: the verdict becomes a function of *when* the job ran. Same
|
||||
argument as the `jq` pin in the same job (`ci.jq-version-contract`), and the same consequence — a bump
|
||||
is a PR someone reads. `pytest` and `pyyaml` in the same job stay unpinned, and the
|
||||
asymmetry is the point rather than an oversight: a pytest release does not add assertions to your
|
||||
suite, a ruff release adds rules to your lint.
|
||||
@@ -0,0 +1,116 @@
|
||||
---
|
||||
key: ci.required-job-step-execution-markers
|
||||
title: '2026-08-10 — every consequential `run:` step in docker-build.yml''s two REQUIRED jobs records that it executed, and a trailing guard fails the job when the set is incomplete (#756)'
|
||||
status: active
|
||||
since: '2026-08-10'
|
||||
supersedes: none
|
||||
superseded-by: none
|
||||
rule: 'A step the runner declines to interpolate is DROPPED and the job still concludes `success` (`ci.workflow-run-body-no-expressions`). In `review-verdict.yml` that is fail-CLOSED — the required status is absent and the merge is blocked. In `docker-build.yml`''s `test` and `migrations` it is fail-OPEN: those are the other two required contexts on `main`, so the check reports green having done no work. So in those two jobs every `run:` step that is not `continue-on-error: true` calls `"$GITHUB_WORKSPACE/scripts/ci-step-ran.sh" mark <key>` as its FIRST act, and the job''s LAST step calls `ci-step-ran.sh assert --always <keys> --gated <keys>`, which fails the job when an expected key was never recorded. PER STEP, not per job: a marker written by the first step only proves the job started, while the drop that costs something is `Test` or the migration replay. The guard carries NO `if:` — the default `success()` is the wanted condition, because a genuine failure in an early step legitimately skips every later one and an `always()` guard would announce a false "these steps never executed" on every ordinary red build; the invariant that makes the omission safe is that the guard is skipped only when an earlier step FAILED, which already fails the job, so guard-skipped implies job-red and every path to a green job runs the guard. Separately and independently, no `${{` OPENER may appear in any `run:` body of those two jobs OR of `build` — the drop mechanism requires the opener, so banning it makes the class unreachable rather than merely caught, and an UNCLOSED opener triggers the same rewrite as a well-formed pair. Pass values in through the step''s `env:`, which is interpolated per value. The two halves have DIFFERENT scopes on purpose: markers cover the required pair, while the ban also covers `build`, whose `Smoke + IPTV E2E` step runs AFTER the image is pushed, so a drop there publishes a release candidate that was never booted and that `DeployStack jazz-media` then promotes. `functional-e2e` is delimiter-free but deliberately excluded (advisory by declaration), and `api-docs`/`format` keep one `github.base_ref` each and gate nothing that ships. The ban is enforced on the RELEASE PATH itself, not only in review (#767): a `scan` job runs the PyYAML-based ban test and `build` lists it in `needs:`, so a delimiter means `build` never runs and no image is published. A guard STEP inside `build` was tried first and is wrong — a step cannot protect the job it publishes from, and "my body has no opener so I cannot be dropped" is circular when only the PR-only test enforces that. The pytest in `script-tests` remains, but it is `on: pull_request` and not a required context, so it alone left the tag path unchecked.'
|
||||
signals: 'required check green but no work done, step never ran but job green, Build & test green in seconds, EF migration integrity green without replaying, missing Run Main step marker, Unable to interpolate expression format(, dropped step docker-build, ci-step-ran.sh, marker file, expression delimiter in a required job · paths: `.gitea/workflows/docker-build.yml`, `scripts/ci-step-ran.sh`, `scripts/tests/test_ci_dropped_step_guard.py`, `scripts/tests/test_ci_release_path_scan_job.py` · issues: #756, #751, #684, #767'
|
||||
mechanics: '`scripts/ci-step-ran.sh` owns the marker path so it exists ONCE and the write and the read cannot diverge. It is keyed on `GITHUB_JOB`/`GITHUB_RUN_ID` — REQUIRED, refusing rather than falling back to a reusable name — plus `GITHUB_RUN_ATTEMPT`. All three REFUSE rather than falling back to a reusable name. The third was warn-and-default until its presence was measured: grepping a log for the variable NAME proves nothing, and inferring it from the absence of a stderr warning proves nothing either (stderr capture was itself unestablished), so `assert` was made to print `Marker identity: job=… run=… attempt=… (from the runner)` on STDOUT and the answer was read off run 1916 for both required jobs. That line is retained as standing evidence. Do NOT justify the keying with #751''s "RUNNER_TEMP is /tmp, not a private per-job dir": that was measured on a job with no `container:` and does not transfer — these jobs get a fresh container, which is the primary protection, and the keying is defence in depth. Held by `scripts/tests/test_ci_dropped_step_guard.py`: static (marker set derived from the workflow equals the guard''s expectations, bucket matches each step''s `if:`, guard is last / has no `if:` / is not advisory / has no delimiter) and behavioural (the guard''s real command line executed against markers written by the steps'' real marker lines, dropping each key in turn). The release-path `scan` job (#767) runs the existing PyYAML-based ban test rather than a second implementation, so there is no drift surface; `scripts/tests/test_ci_release_path_scan_job.py` holds the WIRING instead — that `build` needs it AND that `build`''s own `if:` carries no `always()`/`!cancelled()`/`failure()` (which would downgrade the `needs:` edge to mere ordering), that `scan` carries no job-level `if:` (one excluding the tag push restores the hole, one skipping the job skips `build` too) and is not advisory at STEP or JOB level, and that its own run bodies are delimiter-free. Its load-bearing test is an EXECUTION PROBE, not a shape assertion: it runs the scan step''s real `run:` body with the full env the runner would give it (workflow, job AND step `env:` tiers) against a copy of the repo whose `Smoke` body carries an injected delimiter, and requires a non-zero exit, with a clean-tree negative control. Shape assertions were tried and lost repeatedly — from `echo`ing the command to `PYTEST_ADDOPTS` one env tier up — so do NOT replace the probe with cheaper checks about the command. Two tiers cannot be reached from inside pytest at all and are handled differently: a step writing to `$GITHUB_ENV` is BANNED by test, and repo-root pytest configuration (`pytest.ini` `addopts`, `pytest_collection_modifyitems`) can deselect any test including the guards, so the positive control is a SHELL step — `scripts/ci-prove-ban-detects.sh` poisons the checked-out workflow in the REAL checkout, re-runs the ban test, and vouches ONLY for the ban test''s `build` parametrisation failing — any other outcome (exit 5 from a total deselect, exit 2 from a collection error, an unrelated test failing) is a REFUSAL, not a pass, because each weaker reading was a live bug in an earlier draft and the deselection disarm it exists to catch exits 5 rather than 1. That script is itself positively controlled — `test_the_PROOF_SCRIPT_itself_refuses_when_the_ban_is_deselected` and `test_the_PROOF_SCRIPT_refuses_when_the_WRONG_test_fails` cover the two refusal branches a disarm actually lands on, each verified by making that branch alone unreachable — since it was for a while the one guard exercised only on the happy path. The third branch (pytest passing outright) has no control and does not need one: neutering it falls through to the exit-code branch, which still refuses. A copy-based proof is not equivalent: it does not inherit the repo-root config a disarm would live in. Its steps carry markers and a trailing assert of their own, verified by the same drop-each-key-in-turn behavioural pattern. CARVE-OUT: the "fresh container is the primary protection, keying is defence in depth" reasoning above does NOT cover `scan` — it has no `container:` and runs on `small`, where RUNNER_TEMP is the shared host /tmp, so for that job the run-id/attempt keying is the ONLY protection. Residual: a single-job re-run that does not increment GITHUB_RUN_ATTEMPT would find the prior attempt''s marker file and the assert would pass even with the pytest step dropped. Identity on that lane was measured, not assumed — run 1929 printed `Marker identity: job=scan run=1929 attempt=1 (from the runner)`.'
|
||||
---
|
||||
|
||||
**Why per step, when #756 proposed per job.** A job-start marker answers "did this job begin", which
|
||||
was never in doubt. The fail-open it is supposed to close is a required context reporting success
|
||||
while the work inside it did not happen, and the steps that carry that work are `Test`, `Build` and
|
||||
the two migration replays — all of them well past step one. A guard positioned where it cannot see
|
||||
the case it was built for is the "guard that never executed" failure one level up, and this repo has
|
||||
now shipped that twice in the same file (#751's retarget fence, and #751's own guard).
|
||||
|
||||
**Why a script rather than an inline body, when #751 chose inline.** Two reasons and the second is
|
||||
the load-bearing one. The path literal exists once, so the write and the read cannot drift — #751
|
||||
carries it twice and spends real test effort proving the copies agree, because a divergence reddens
|
||||
every run and then gets deleted as broken. And a one-line `run: scripts/ci-step-ran.sh …` cannot
|
||||
contain an expression delimiter, so the mechanism being guarded against cannot drop the guard. #751's
|
||||
own record names that as the stronger construction and settled for inline only because its
|
||||
measurement showed it was not required there.
|
||||
|
||||
**Why a script is acceptable here and would NOT be in `review-verdict.yml`.** That workflow checks
|
||||
out the PR's BASE precisely so a PR cannot supply the code that judges it. `docker-build.yml` is
|
||||
head-resolved by design — a PR already supplies every test this job runs — so calling a script from
|
||||
the head adds no authority a PR did not already have. This is a correctness gate against silent
|
||||
no-ops, not a security gate against a hostile PR; that job belongs to `review-verdict/h10`. Do not
|
||||
carry this reasoning back into the gate workflow.
|
||||
|
||||
**The premise was re-measured on the BUILD lane, not inherited.** The whole guard rests on the runner
|
||||
still executing a LATER step after dropping an earlier one. #751 established that on the `small`
|
||||
lane; these two jobs run in a `container:` on `ubuntu-latest`, which is a different lane, so assuming
|
||||
it transfers would be the same shape of mistake the guard exists to catch. Measured by scratch PR
|
||||
#765 (Gitea 1.27.1, 2026-08-10), which reintroduced the exact #751 defect — an invalid expression
|
||||
payload inside a shell comment — in the `test` job's `revalidate` step. The step was dropped, the
|
||||
other eleven markers were still recorded — ten of them AFTER the drop, `detect` being the earlier
|
||||
eleventh — and the guard was the ONLY failing step
|
||||
in the job — so without it that run would have concluded `success` having skipped a step. The SAME
|
||||
run supplies the positive control on the same lane: its untouched `migrations` job marked all six
|
||||
steps, reported `All 6 expected step(s) executed`, and concluded `success`.
|
||||
|
||||
A second probe (PR #766, run 1913) settled the one path on which the `if:`-less guard could have been
|
||||
a silent no-op: a FAILING `continue-on-error` step. Had that flipped `success()`, the guard would be
|
||||
skipped on a still-green job. It does not — the advisory step failed, the guard ran anyway, reported
|
||||
`All 12 expected step(s) executed`, and the job stayed `success`. Full log extracts in
|
||||
docs/ci-cd.md.
|
||||
|
||||
**The two halves are deliberately different in kind, and neither is redundant.** The delimiter ban is
|
||||
static and absolute, and it makes the defect class UNREACHABLE in these jobs rather than merely
|
||||
detected — it is the cheaper and more general half, and it is enforceable today only because both
|
||||
jobs were already delimiter-free (measured 2026-08-10: `test` 0, `migrations` 0), and `build` was
|
||||
brought in by moving its two payloads to `env:` — leaving `api-docs` and `format` with one
|
||||
`github.base_ref` each, in detect steps that gate nothing that ships. The runtime markers catch a step
|
||||
that fails to run for any OTHER reason, including reasons not yet met. Keeping only the static half
|
||||
would be trusting that this is the only way a step can vanish, which is exactly the assumption #751
|
||||
falsified about shell comments.
|
||||
|
||||
**What this does not claim.** The guard proves a step STARTED, never that it did its work correctly
|
||||
— that is what the step's own exit status is for. It does not cover `uses:` steps, which are not
|
||||
`run:` bodies and cannot be dropped this way.
|
||||
|
||||
An earlier draft dismissed the non-required jobs as "a smaller cost (no required context lies)", and
|
||||
cold review showed that was false for the one that matters. `build`'s only delimiter-bearing body was
|
||||
`Smoke + IPTV E2E`, which runs AFTER `Build and push`: on a `v*` tag the candidate image is already
|
||||
published, and that step is the only thing that boots it. A drop there ships an unsmoked release
|
||||
candidate under a green tick, and prod promotion pulls exactly that image. It was also the cheap case
|
||||
— both payloads were plain values, so moving them into `env:` cost nothing and let `build` join the
|
||||
ban. The claim not to repeat is the draft's dichotomy ("give up interpolation or move into
|
||||
`scripts/`"); the `env:` escape hatch this record prescribes was the answer all along. What genuinely
|
||||
remains uncovered is `api-docs` and `format`, whose one `github.base_ref` each sits in a detect step
|
||||
that gates nothing that ships, and `functional-e2e`, which is advisory by declaration.
|
||||
|
||||
**The `build` ban is now fail-closed on the release path (ersatztv#767 — this was the open
|
||||
residual).** It used to be enforced only by `script-tests`, which is `on: pull_request` and is not a
|
||||
required context, so nothing re-checked it when a release was actually cut: a delimiter that reached
|
||||
`main` would still drop `Smoke` on the tag build and report green. A `scan` job now runs the
|
||||
PyYAML-based ban test and `build` lists it in `needs:`, so a delimiter means `build` never runs and
|
||||
no image is published.
|
||||
|
||||
**Two designs were tried, and the first one's failures are the reusable part.** The first put a
|
||||
bespoke stdlib scanner in `build` itself as an unconditional step before `Build and push`. Two
|
||||
independent reviews rejected it on two counts, both easy to re-invent:
|
||||
|
||||
- **A guard step cannot protect the job it lives in.** `build` publishes, so a guard step there is
|
||||
fail-OPEN if the runner drops it. The defence offered — "the guard's own body has no opener, so it
|
||||
cannot be dropped" — is circular, because the only thing enforcing that property was the same
|
||||
PR-only, non-required test being backstopped. A `needs:` edge is not circular: a red job skips its
|
||||
dependents by construction.
|
||||
- **A hand-written parser was strictly weaker than the check it backstopped.** It hand-parsed YAML to
|
||||
avoid provisioning PyYAML on `build`'s bare runner, and review found ~10 false NEGATIVES in one
|
||||
round (flow mappings, a quoted `"run":` key, aliases, multiline quoted scalars). For a security
|
||||
gate only false negatives matter, so this was worse than useless — it looked like enforcement. Do
|
||||
not re-attempt a bespoke scanner to save provisioning a dependency; run the real test.
|
||||
|
||||
**Why this needs no third marker bucket.** The deferral assumed the answer had to be markers on
|
||||
`build`, requiring a bucket that models `Smoke`'s publish-ref `if:`. It does not: the delimiter class
|
||||
is a *static* property of the workflow text, so a job that reads the text catches it without
|
||||
modelling any `if:`. The marker buckets are unchanged. Per-step markers on `build` remain a genuine
|
||||
smaller residual — they would catch a drop caused by something other than a delimiter.
|
||||
|
||||
**What this does not claim.** That no step can ever fail to run for another reason. The `scan` job's
|
||||
own steps carry markers and a trailing assert, which moves the terminal assumption rather than
|
||||
removing it: to fail open you must now drop the pytest step AND the assert step, not either alone.
|
||||
|
||||
**Measured, not assumed** — runs **1928** (poisoned) and **1929** (control), `workflow_dispatch` on
|
||||
scratch branches, 2026-08-11. The arrangement: a `workflow_dispatch` on a scratch branch whose `Smoke` body
|
||||
carries a deliberate delimiter must redden `scan` and leave `build` skipped, and the same dispatch
|
||||
without the poison must pass. Note that "no image was published" is NOT part of the evidence — on a
|
||||
scratch ref `Build and push` has `push: false` regardless, so that conjunct could not have come out
|
||||
the other way; the discriminating observation is `scan` red and `build` skipped. Do NOT repeat the
|
||||
cost estimate an earlier draft gave ("would require pushing a real `v*` tag").
|
||||
@@ -5,7 +5,7 @@ status: active
|
||||
since: '2026-07-26'
|
||||
supersedes: none
|
||||
superseded-by: none
|
||||
rule: 'The `scripts/tests/` pytest suite runs on every PR as a dedicated `script-tests` job in `pr-checks.yml` (`runs-on: small`, `setup-python` + `pip install pytest`, `PYTHONPATH=. python3 -m pytest scripts/tests -q`), unconditionally rather than behind a `scripts/**` path filter, and **never as a step inside `decisions-guard`** — a job whose reds a standing rule instructs sessions to ignore must never host a gate whose reds are real. Any new CI gate must be reachable by a failure that is unambiguously attributable to it.'
|
||||
rule: 'The `scripts/tests/` pytest suite runs on every PR as a dedicated `script-tests` job in `pr-checks.yml` (`runs-on: small`, `setup-python` + `pip install pytest pyyaml`, `PYTHONPATH=. python3 -m pytest scripts/tests -q`; since #780 it also runs a pinned ruff over a `git ls-files` population first), unconditionally rather than behind a `scripts/**` path filter, and **never as a step inside `decisions-guard`** — a job whose reds a standing rule instructs sessions to ignore must never host a gate whose reds are real. Any new CI gate must be reachable by a failure that is unambiguously attributable to it.'
|
||||
signals: 'scripts/tests never ran in CI, pytest not in any workflow, python test suite local-only, decorative test, decisions-guard runs the code not the tests, script-tests job, small lane pytest, negative control CI goes red · paths: `.gitea/workflows/pr-checks.yml`, `scripts/tests/`, `docs/ci-cd.md` · issues: #631, #610, #621, #622, #542'
|
||||
mechanics: '`.gitea/workflows/pr-checks.yml` -> `script-tests`; `docs/ci-cd.md` -> "`script-tests` job"'
|
||||
---
|
||||
@@ -29,7 +29,7 @@ it lives in, so **a job under a standing ignore-rule can host no real gate.**
|
||||
|
||||
This does not conflict with `ci.ui-e2e-harness` ("never their own job"). That record folds UI-E2E
|
||||
into `functional-e2e` because the specs need an app the job has *already booted* — sharing expensive
|
||||
setup. Here there is no shared setup to reuse (a checkout plus `pip install pytest pyyaml`), and the
|
||||
setup. Here there is no shared setup to reuse (a checkout plus `pip install pytest pyyaml` and a pinned ruff), and the
|
||||
sibling job carries an ignore-rule. Same question, opposite answers, for stated reasons.
|
||||
|
||||
**Unconditional, not path-filtered.** The suite's real input set spans more than `scripts/`:
|
||||
@@ -104,3 +104,9 @@ from a proxy ("fewer than we asked for", "jq didn't complain").
|
||||
It is not yet a *required* status check — `main` requires only `Build & test (.NET)`,
|
||||
`EF migration integrity` and `review-verdict/h10`. It reddens the run; promoting it to required is a
|
||||
branch-protection change left deliberately separate.
|
||||
|
||||
Since #780 the job also lints Python before pytest, so its display name is
|
||||
`Script lint and tests (ruff + pytest)`. It does **not** invoke `ruff check .` — the invocation and
|
||||
the reasons for its exact shape are `ci.python-lint-ruff-config-committed`. Why the lint lives here
|
||||
rather than in a job of its own: it needs the same `setup-python`, it costs seconds, and a second job would double the dispatch overhead
|
||||
this file exists to keep small.
|
||||
|
||||
@@ -32,5 +32,5 @@ mechanics: sum-of-caps rule (#406/#604); second jazz runner at `--cpu-shares=128
|
||||
works only because it lands after load clears, so a capacity problem read as a bug in the guard. A
|
||||
job that fails with zero log output is evidence about the runner, not about the job.
|
||||
- **#574's skip-task queueing does not return** by moving `build` back to `ubuntu-latest`:
|
||||
`needs: [test, migrations]` means it cannot be dispatched until the jobs it would have queued behind
|
||||
`needs: [test, migrations, scan]` means it cannot be dispatched until the jobs it would have queued behind
|
||||
have already finished.
|
||||
|
||||
@@ -7,7 +7,7 @@ supersedes: none
|
||||
superseded-by: none
|
||||
rule: 'The `review-verdict/h10` job counts `change_target_branch` events on the PR''s issue timeline at run start and again immediately before its POST, and writes NOTHING if the count moved. The COUNT is the key because the branch NAME is ABA-vulnerable — `main -> S -> main` reads `main` at both ends, which is how #698 route 1 obtained a forged exemption — while the event count is monotonic and cannot alias. Abstaining is a handoff, not a stall, and that is the property the design rests on: every retarget fires `edited`, which is in this workflow''s `types:`, so the event that makes a run abstain has already queued a successor whose window opens after it; the induction terminates when retargeting stops and the last run writes the final answer. `updated_at` was REJECTED as the key because it also moves for comments and labels, which fire none of this workflow''s `types:` — a run could abstain with no successor coming, which is a real stall. The count is trusted only when paging reached a validated EMPTY page; an untrusted count (unreadable page, non-array body, non-numeric length, page cap hit) blocks the exemption `success` ONLY and still lets `pending` through, because `pending` cannot turn an unreviewed head green while withholding it would strand ordinary PRs for no safety gain. SEPARATELY, and for the human-verdict race the fence does nothing about: after posting an exemption `success` the job re-reads `/statuses/{sha}` and, if a human `Review-verdict:` row appeared with an id ABOVE a high-water mark taken just before the POST, overwrites its own status with `pending` and logs an error. The repair is `pending`, NEVER a copy of the human''s state, since re-posting their `failure` under the machine credential would attribute a human verdict to the job; its description is a SENTINEL that the classification refuses to grant an exemption over AND re-writes verbatim on every later run, so the block is a FIXED POINT rather than decaying — writing the generic `pending` description there instead erases the marker and the exemption simply returns one event later. The mark is captured BEFORE the last-moment re-read, not merely before the POST — a later mark leaves a multi-round-trip blind gap in which a verdict is neither seen by the re-read nor repaired afterwards. The id comparison is load-bearing: a mere presence test would fire forever on a base-mismatched verdict that `read_existing_verdict` deliberately declines to honour, deadlocking that PR''s exemption permanently. Finally, a run whose last-moment re-read finds a sentinel it did not see at its FIRST read ABSTAINS instead of posting: that can only mean an overlapping run repaired a raced verdict mid-flight, and this run''s `success` — frozen at classification time, with the human row below its own mark, so neither the fence nor the post-write check would catch it — would otherwise bury the rejection. That is the one path in this design that failed toward SUCCESS rather than `pending`. The post-write check counts TWO row shapes above the mark, not one — a human `Review-verdict:` row AND a machine sentinel — because with two overlapping runs the human row can sit BELOW the second run''s mark while the first masks it and only then writes the sentinel, leaving the second to post its own `success` on top; counting the sentinel converges both runs on the fixed point instead.'
|
||||
signals: 'stale review-verdict run overwrites a fresher one, retarget ABA against the docs-only classifier, concurrency group does not serialize pull_request_target, gitea auto-cancel push vs pull_request_target, forged exemption restored after reclassification, human BLOCKED silently turned green, post-write status verification, change_target_branch timeline count, why does my PR post no verdict status after a retarget · paths: `.gitea/workflows/review-verdict.yml`, `scripts/tests/test_pr_changed_files.py` · issues: #706, #698, #672, #663, #622'
|
||||
mechanics: '`count_retargets()` pages `GET /repos/{repo}/issues/{pr}/timeline?limit=50&page=N` (cap 20) setting `rt_count`/`rt_ok`, trusted only on a validated empty page; `retargets_before`/`retargets_before_ok` captured before enumeration, re-counted immediately before the POST; `max_id_before` from `GET /repos/{repo}/statuses/{sha}` (a BARE ARRAY, unlike the combined `/commits/{sha}/status` object); repair POST is `pending`; tests `test_a_RETARGET_DURING_the_run_posts_NOTHING`, `test_a_PR_retargeted_BEFORE_the_run_but_QUIET_during_it_is_STILL_exempt`, `test_an_UNTRUSTED_retarget_count_withholds_the_EXEMPTION`, `test_an_UNTRUSTED_retarget_count_STILL_LETS_PENDING_THROUGH`, `test_a_human_verdict_landing_AFTER_the_POST_is_repaired_to_pending`, `test_a_PRE_EXISTING_human_row_does_NOT_trigger_a_repair`'
|
||||
mechanics: '`count_retargets()` pages `GET /repos/{repo}/issues/{pr}/timeline?limit=50&page=N` (cap 20) setting `rt_count`/`rt_ok`, trusted only on a validated empty page, which is a page of EITHER `null` (what this endpoint really returns past the end) or `[]` — an `array`-only type gate read the real terminator as unreadable and withheld every exemption (#751); `retargets_before`/`retargets_before_ok` captured before enumeration, re-counted immediately before the POST; `max_id_before` from `GET /repos/{repo}/statuses/{sha}` (a BARE ARRAY, unlike the combined `/commits/{sha}/status` object); repair POST is `pending`; tests `test_a_RETARGET_DURING_the_run_posts_NOTHING`, `test_a_PR_retargeted_BEFORE_the_run_but_QUIET_during_it_is_STILL_exempt`, `test_an_UNTRUSTED_retarget_count_withholds_the_EXEMPTION`, `test_an_UNTRUSTED_retarget_count_STILL_LETS_PENDING_THROUGH`, `test_a_human_verdict_landing_AFTER_the_POST_is_repaired_to_pending`, `test_a_PRE_EXISTING_human_row_does_NOT_trigger_a_repair`'
|
||||
---
|
||||
|
||||
`ci.exemption-provenance` closed three routes into the exemption path and left one residual it named:
|
||||
@@ -70,3 +70,31 @@ marker, so the exemption returned two events later instead of one. Only a re-pos
|
||||
4. **A timeline over the 20-page cap can never be exempted** — `rt_ok` stays `no` on every run, so only
|
||||
a human verdict clears it and comment-flooding becomes a fail-closed denial of exemption.
|
||||
Negligible at 1000 events; the log says so rather than promising a later run will fix it.
|
||||
|
||||
**CORRECTION, 2026-08-06 (ersatztv#751).** Residual 4 above described as a narrow edge case what was
|
||||
in fact the universal behaviour: `rt_ok` stayed `no` on **every** pull request, not only over-cap ones,
|
||||
so the fence withheld **every** exemption `success` from the day it shipped. A page past the end of
|
||||
this endpoint is the JSON value `null`, not `[]` (measured at Gitea 1.27.1 on PR #752; the same
|
||||
instance returns `[]` for an empty `/issues/{n}/comments`, so it is not consistent between endpoints).
|
||||
`count_retargets` gated on `type == "array"` and therefore read the real terminator as unreadable,
|
||||
never reaching the validated empty page it required. Renovate and docs-only PRs got no status at all.
|
||||
|
||||
Two reasons it read as deliberate rather than broken, both worth carrying forward:
|
||||
|
||||
- **It never ran.** This fence shipped in 8f6d4f443 — the same commit whose prose comment stopped the
|
||||
classify step from executing at all (`ci.workflow-run-body-no-expressions`). Merging a guard and
|
||||
first executing it are different events, and only the second tells you anything.
|
||||
- **The double asserted the wrong shape while claiming to be measured.** The stub's comment read "Real
|
||||
shapes, measured on this instance and deliberately mirrored" and it printed `[]` past the end. So the
|
||||
`array`-only gate was never exercised by the suite either. Correcting the double and restoring the
|
||||
old gate reddens most of the fence suite — 18 tests when first measured, 21 once three more
|
||||
fence-dependent tests existed. The invariant, not the number, is that every one of them had been
|
||||
green for the wrong reason. A fidelity claim in a test double is an assertion, and it decays like any
|
||||
other.
|
||||
|
||||
The type is now read as a value (`case` over `jq -r 'type'`) rather than through `jq -e`, whose
|
||||
exit-status semantics already bit this workflow at jq 1.6 (`ci.jq-version-contract`), and both `null`
|
||||
and `[]` terminate the walk. `test_the_fence_TRUSTS_the_count_and_POSTS_when_the_timeline_terminates`
|
||||
is parameterised over both shapes and asserts the POSTED STATUS rather than the log line — on the real
|
||||
probe run the log said `Decision: state=success` and the job still posted nothing, so the decision and
|
||||
the write are separate events and only the write is what a merge reads.
|
||||
|
||||
@@ -0,0 +1,105 @@
|
||||
---
|
||||
key: ci.workflow-run-body-no-expressions
|
||||
title: '2026-08-06 — an expression delimiter anywhere in a `run:` body, INCLUDING in a comment, silently drops the step and reports the job green (#751)'
|
||||
status: active
|
||||
since: '2026-08-06'
|
||||
supersedes: none
|
||||
superseded-by: none
|
||||
rule: 'A `run:` body is not shell when the runner reads it: the runner scans the whole scalar for the expression opener and, on finding one, rewrites the ENTIRE body into a single `format(...)` call. That rewrite is all-or-nothing, so a payload that does not evaluate fails the interpolation of the whole scalar — and the runner then DROPS THE STEP AND CONCLUDES THE JOB `success`. A shell comment is therefore NOT inert. In `.gitea/workflows/review-verdict.yml` no expression delimiter may appear in ANY `run:` body, in code or in prose, because a dropped step there is a dead merge gate rather than a failed build; pass values in through the step''s `env:` block, which is interpolated per value so a bad payload cannot take the body with it, and describe an expression in prose by NAMING it (`a github.event.pull_request.number expression`) rather than quoting the delimiters. Repo-wide the rule is weaker and its reach must be stated precisely rather than generously: every expression payload in every workflow field must have a HEAD TOKEN naming a context or function the runner can resolve. That catches the defect above and a nonexistent context; it does NOT catch a syntactically invalid payload whose tokens are all known (`${{ github.ref == }}`), a renamed output (every token after the first is skipped), or an unclosed opener — those need an expression parser, and the guard is kept permissive on purpose because a red here blocks every merge through the combined status. In `review-verdict.yml` specifically, any step whose non-execution is consequential is paired with a start-marker guard that FAILS the job when the marker is absent, and that guard''s own body must be expression-free — a guard the guarded mechanism can silently delete is worse than none. That pairing now also covers `docker-build.yml`''s `test` and `migrations` jobs, where a dropped step is fail-OPEN (the required check goes green having done no work) rather than fail-closed as it is here — see `ci.required-job-step-execution-markers`, which adds per-STEP markers there and extends this file''s delimiter ban to those two jobs. It is still not a repo-wide property, but the remaining exceptions are narrower than this record originally said: `build` was brought into the ban too (its `Smoke + IPTV E2E` runs AFTER the image is pushed, so a drop there ships an unsmoked release candidate — its two payloads moved to `env:`, so the ban was free), leaving only `api-docs` and `format`, whose one `github.base_ref` each sits in a detect step that gates nothing that ships.'
|
||||
signals: 'Unable to interpolate expression format(, step never ran but job green, missing Run Main step marker, review-verdict/h10 absent after a green run, docs-only PR unmergeable, Renovate PR unmergeable, exemption stopped working, expression in a shell comment, workflow comment changed behaviour · paths: `.gitea/workflows/review-verdict.yml`, `scripts/tests/test_pr_changed_files.py` · issues: #751, #706, #748'
|
||||
mechanics: '`RAN_MARKER` written at the top of the classify step and asserted by the `Assert the classifier actually executed` step (`if: always()`, expression-free body, `exit 1` on a missing marker); static guards `test_the_verdict_workflow_has_NO_expression_delimiter_in_any_run_body` (raw scalar, absolute, gate file only) and `test_every_workflow_expression_names_a_REAL_context_or_function` (repo-wide, allow-list of contexts/functions) plus `test_a_dropped_classify_step_FAILS_the_job_instead_of_going_green` (pins marker path agreement and guard ordering across ONE yaml parse)'
|
||||
---
|
||||
|
||||
**How it happened, which is the part that generalises.** The #706 note explaining why a concurrency
|
||||
group does not work in `review-verdict.yml` quoted a `concurrency:` snippet containing a PR-number
|
||||
expression *as an illustration*, inside a shell comment. `pr number` is not a valid expression. From
|
||||
8f6d4f443 (2026-08-03) to 2026-08-06 the classify step therefore never ran, `review-verdict/h10` was
|
||||
posted by nothing but a human hand, and both exemption classes silently stopped working — while every
|
||||
run reported success. The prose documenting a fix disabled the fix.
|
||||
|
||||
**Why nothing caught it.** Every pre-existing workflow-shape test in
|
||||
`scripts/tests/test_pr_changed_files.py` reads `_code_lines()`, which strips comment lines. That is
|
||||
correct for what it was for — its own docstring notes that prose legitimately discusses
|
||||
`pulls/N/files`, and a raw scan would redden the repo over a piece of writing — but it encodes the
|
||||
assumption this bug falsifies: that a comment in a workflow cannot change behaviour. Inside a `run:`
|
||||
scalar it can. The strict test added here reads the RAW scalar for exactly that reason and must never
|
||||
adopt `_code_lines`.
|
||||
|
||||
**The silent green is the defect; the delimiter was only the trigger.** An absent required status
|
||||
reads as "not reviewed yet" on an ordinary PR, which is indistinguishable from the correct pending
|
||||
state — so a normal PR looked normal while the gate was dead. The visible cost landed on the two
|
||||
classes with no human in the loop: PR #739 (docs-only) merged 2026-08-05 with ZERO commit statuses on
|
||||
its head, and got in only because admin force-merge was still enabled. #743 removed that escape the
|
||||
next day, so by the time this was found the workaround that had been absorbing the bug was gone and
|
||||
the next docs-only or Renovate-manifest PR would have been permanently stuck. The two Renovate PRs in
|
||||
the window escaped by timing alone, merging minutes before the bad commit landed.
|
||||
|
||||
**Scope of the strict rule, and why it is not repo-wide.** As of 2026-08-06, `docker-build.yml`,
|
||||
`ci-image.yml` and `pr-checks.yml` interpolated into `run:` bodies legitimately (7 occurrences then;
|
||||
#756 removed `build`'s two, leaving 5 today — see below). A repo-wide ban would be
|
||||
false and would be deleted the first time it got in someone's way. `review-verdict.yml` earns the
|
||||
absolute rule on two counts: it writes the branch-protection-required status, and its `run:` bodies
|
||||
are ~700 lines of dense prose — the only place the delimiter has ever appeared by accident.
|
||||
|
||||
The first of those two counts turned out to apply elsewhere as well, and #756 acted on it: the
|
||||
absolute ban now also covers `docker-build.yml`'s `test` and `migrations` jobs, which write the other
|
||||
two required contexts and were delimiter-free already, so the rule cost nothing to impose there. The
|
||||
remaining 2 occurrences inside `docker-build.yml` — `api-docs` and `format`, one `github.base_ref`
|
||||
each — sit in jobs that gate nothing that ships (5 repo-wide, counting `ci-image.yml` and the two
|
||||
`pr-checks.yml` gates). `build` is banned too, and NOT because it is required (it is not): its
|
||||
`Smoke + IPTV E2E` step runs after the image is pushed, so a drop there publishes a release candidate
|
||||
that was never booted. Read this paragraph as scoping the rule to steps whose non-execution is
|
||||
CONSEQUENTIAL — required contexts and the release path — rather than to this one file.
|
||||
|
||||
**The probe found a SECOND, independent reason the gate posted nothing**, and it is why fixing the
|
||||
interpolation alone would not have restored the exemptions: a page past the end of
|
||||
`/issues/{n}/timeline` is JSON `null`, not `[]`, so the retarget fence never trusted its count for ANY
|
||||
PR and withheld every exemption `success`. Corrected in `ci.verdict-write-retarget-fence`, whose stated
|
||||
residual had described that universal behaviour as a narrow over-cap edge case. Both defects shipped in
|
||||
the same commit, which is the general lesson: a guard that has never executed has told you nothing, and
|
||||
merging it is not executing it.
|
||||
|
||||
**A THIRD instance of the same server behaviour was found by cold review of this fix**, and it is
|
||||
the reason to distrust "I fixed the two I could see". `GET /commits/{sha}/status` also returns
|
||||
`statuses: null` — not `[]` — for a head with no statuses yet (measured on PR #739's head 5fa672e2:
|
||||
`{"state":"pending","total_count":0,"statuses":null}`). `read_existing_verdict` gated on
|
||||
`.statuses | type == "array"` and took its `exit 1` path, posting nothing: fail-closed, but the same
|
||||
user-visible outcome again. Its double printed `{"statuses": []}` at all three no-verdict sites, so
|
||||
that branch was unreachable in the suite; correcting the double and restoring the old gate turns 40+
|
||||
tests red. `scripts/pr-changed-files.sh` was swept too and is unaffected — `pulls/{n}/files` returns
|
||||
`[]`. The generalisable rule is that a nil Go slice serialises to `null`, so EVERY list-shaped field
|
||||
on this API is suspect, and a per-endpoint measurement is the only way to know.
|
||||
|
||||
**Restoring the exemptions restores a hole that had been dead**, and this is worth saying rather than
|
||||
presenting the change as pure repair. `DOCS_ONLY` matched `CLAUDE.md` and `AGENTS.md`, the documents
|
||||
that define the completion protocol and the H10 rule itself — so those were auto-exemptible while
|
||||
`.claude/` was protected, which is the same self-exemption the workflow header rules out, one
|
||||
directory over. Reachable only because exemptions work again, hence fixed here (both added to
|
||||
`PROTECTED`; see `ci.exemption-provenance`). For the same reason, #706's known residual — the
|
||||
sub-round-trip ABA window, "narrowed and observable, not closed" — comes back with the working fence:
|
||||
while `rt_ok` was never `yes`, route 1 was closed by accident.
|
||||
|
||||
**Three guards were proposed or written for the same hole and the first two were no-ops** — the hole
|
||||
being that concluding "no verdict exists" is what licenses posting over one. `total_count` is per-PAGE
|
||||
here (`?limit=1` on a 6-context head gives `len=1, total_count=1`), so length-vs-total is equal by
|
||||
construction; and "refuse on a full page at `limit=100`" was DEAD CODE, because the instance caps
|
||||
`limit` at `MAX_RESPONSE_ITEMS`, measured at 50 — a cap this repo already documented in three places
|
||||
before the guard was written against 100. The working version asks the server: read page 2 when the row
|
||||
is absent from page 1, and refuse if it carries anything. Cap-independent, so no reconfiguration
|
||||
re-breaks it. Second, `jq -r` renders the number `0` and the string `"0"` identically, so the zero
|
||||
check requires the JSON type as well. Neither was a live failure — both are the difference between a
|
||||
guard that holds because the input happens to be well-formed and one that holds because it checks.
|
||||
|
||||
**The tests written to close a review finding then needed closing themselves**, which is the honest
|
||||
shape of work on this file. The behavioural guard test first extracted the two marker lines by text and
|
||||
ran them alone — which passes even if the write is moved into a function nobody calls. It now executes
|
||||
the classify body's real PREFIX down to the write, reproducing the production control flow instead of a
|
||||
reconstruction of it. The anti-vacuity check first hand-counted `run:` keys with a regex, which
|
||||
false-redded legal spellings (`- run: |`, a single-line `run: echo ok`) and could count a `run: |`
|
||||
inside a heredoc; hand-parsing YAML to validate a YAML parse is the wrong shape, so it now asserts on
|
||||
content — the walk reached at least three bodies and one over 5000 characters.
|
||||
|
||||
**Verified by mutation, not by a green suite.** All six mutations produce a red and the restored tree
|
||||
is green: reintroducing the exact defect (caught by both the strict and the general test), deleting
|
||||
the guard step, deleting only the marker write, weakening `if: always()`, turning the guard's
|
||||
`exit 1` into `exit 0`, and putting a delimiter in the guard's own body.
|
||||
@@ -0,0 +1,56 @@
|
||||
---
|
||||
key: docs.no-session-narrative
|
||||
title: '2026-08-21 — a doc records the end state; the path to it goes in the commit message, not the artifact (#784)'
|
||||
status: active
|
||||
since: '2026-08-21'
|
||||
supersedes: none
|
||||
superseded-by: none
|
||||
rule: 'Every durable artifact — an in-repo `docs/` page, a skill, a README, a code comment, an Obsidian vault page — records the END STATE. The path to that end state goes in the commit message, the Gitea issue, or the issue''s `## Closing record`; it does not go in the artifact. Concretely: **a review finding is answered in the commit message, and only the corrected claim enters the doc.** Naming the destination is load-bearing — "do not write it in the doc" with no home loses the knowledge, and this repo has the inverse failure on record too (#542, where a pruned narrative turned out to be the only copy). THE TEST IS WHO BENEFITS: if only the author''s timeline explains why a sentence is there, it is narrative and belongs in the commit; if a reader who never saw the session would act differently knowing it, it is a finding and stays. Session narrative reads as: first person or session chronology ("I initially thought", "an earlier draft counted", "my first attempt returned 0"), a correction of a belief the reader never held ("this was wrong, actually X" where only X matters), relative time ("earlier today", "currently investigating"), or a blow-by-blow diagnosis standing in place of the conclusion. THE CARVE-OUT, which must be stated or the rule gets over-applied — reader-facing history that must survive: a decision record''s `supersedes`/`superseded-by`; a dated measurement or an explicitly stated snapshot boundary; a TESTED-AND-REJECTED negative result, kept so nobody re-proposes it on plausibility; the *why* behind a non-obvious choice; and a trap together with its consequence. `docs/decisions/records/**` and `docs/decisions/archive/**` are exempt WHOLESALE: a record narrating how a rule was got wrong is carrying the rationale it exists to carry. ENFORCEMENT IS ADVISORY ONLY — `scripts/check-doc-narrative.py`, run non-blocking from the `docs-reminder` job over ADDED lines. It is a string predicate over prose and may never become a blocking gate.'
|
||||
signals: 'no session narrative in docs · the reader never saw the earlier draft · who-benefits test · answer a review finding in the commit message not the doc · end state versus path to it · earlier draft · first version of this table · my first attempt · relative time in a doc · carve-out for dated measurement and snapshot boundary · tested-and-rejected negative result · decision records are exempt on purpose · advisory not blocking · string predicate over prose · paths: `scripts/check-doc-narrative.py`, `.gitea/workflows/pr-checks.yml`, `docs/handoffs/chicorytv-issue-queue.md`, `docs/defect-shapes-773.md` · issues: #784, #773, #767, #743, #542'
|
||||
mechanics: 'The RULE is about every durable artifact; the DETECTOR''s population is narrower and is stated here so a row in it never reads as coverage it does not have. `scripts/check-doc-narrative.py` scans `docs/**/*.md` minus `docs/decisions/**` (exempt wholesale, in both modes), plus root-level `*.md`. Skills under `.claude/`, `web/`, and every other nested markdown file outside `docs/` are OUT of its scope and stay a human judgement. It warns and exits 0 on every path — bad argument, unresolvable ref, unreadable file, unhandled exception — asserted per argument shape in `scripts/tests/test_check_doc_narrative.py`, not only in prose, EXCEPT the unhandled-exception arm, which is a bare `except` no test exercises and is recorded as unproven rather than implied. The invariant is held at the JOB level too (`continue-on-error` on both steps): a script that returns 0 does not keep a job green if a setup action fails. Every knob DEMONSTRATED to break the parse has a row in `FORMAT_KNOBS` in the test file; the user and system config files are removed from the picture entirely so an unnamed one cannot reach it. Three separate review rounds each found ONE more knob turning a real hit into a clean-looking `scanned 0 file(s)`, so the third fix removed the surface rather than naming a fourth knob — a completeness claim over the knob space would be exactly the enumeration that failed three times. COVERAGE BOUNDARY, written once so it is not rediscovered one clause at a time, and naming the clauses rather than a category because a category is where the last mis-sort hid. MUTATION-PROVEN: the parse path (hunk state, the `\` marker, the `diff --git` reset, `splitlines`, the population count, `re.IGNORECASE`, the SCANNED-NOTHING return) and five pins — `core.quotePath=false`, `--find-renames`, `--dst-prefix`, `--no-ext-diff`, `--no-color`. UNPROVEN and defensive, recorded as such rather than implied to be covered: `-U0` (the context arm handles any `-U`, so removing it reddens nothing), the `GIT_CONFIG_GLOBAL`/`SYSTEM`/`NOSYSTEM` overrides (which exist for the UNNAMED knob and therefore cannot be witnessed — the script says so itself), `--src-prefix`, the `+++ /dev/null` deletion arm, the malformed-`@@` arm, the stderr relay, `git()`''s `OSError` return, `run_all`''s failure sentinel, `open(errors=)` and the top-level `except`. Both lists ENUMERATE; neither is a universal over the file, and a clause on neither list has simply not been measured. `--diff` scans only lines ADDED in the PR (with rename detection, so a `git mv` does not re-flag a file''s pre-existing content) and reports `SCANNED NOTHING` rather than a clean-looking line when it cannot resolve the base. `--all` sweeps the tracked corpus from `git ls-files`, never a filesystem walk (#778). Two consequences of added-lines-only are deliberate and stated so they are not mistaken for coverage: a file MOVED into the population (`web/x.md` to `docs/x.md`) is never scanned by any PR, and neither is anything already in the corpus. `--all` is the only thing that sees either, which is why the sweep is a task a person runs rather than a job.'
|
||||
---
|
||||
|
||||
The rule already existed, correctly stated, and scoped to exactly one file. The kickoff handoff said
|
||||
a paragraph of narrative there is a tax paid by every future session, *because that file is pasted
|
||||
into every session*. That reason is file-specific. The general reason is broader and applies to
|
||||
every doc in the repo: **a reader coming to a doc cold never saw the earlier draft**, so "we
|
||||
previously got this wrong" carries nothing they can act on.
|
||||
|
||||
This is the recurring shape `#773` measures — a rule established where it was first noticed and
|
||||
never extended to its class. It is the same shape as #743, where a control defended the merge path
|
||||
while the push path stayed open, and as #767's record puts it: *"'the ban is enforced' and 'the ban
|
||||
is enforced **where it matters**' were never separated."* The generalisation is the fix; the six
|
||||
stripped instances in `docs/defect-shapes-773.md` were the symptom.
|
||||
|
||||
**Why it leaks — a mechanism, not a discipline problem.** A review finding creates pressure to
|
||||
answer *in the artifact*. The correction and the justification-for-the-correction get written in the
|
||||
same keystroke, and the artifact is the file already open, so the justification lands there too.
|
||||
Every one of the six instances came out of a review round. That is why the rule names a destination
|
||||
rather than only a prohibition: the commit message is written at the same moment, is permanent, and
|
||||
is where provenance is actually looked for.
|
||||
|
||||
**Why the carve-out is half the rule.** Over-stripping is the more common failure. A doc cut to bare
|
||||
facts reads as arbitrary and the next reader "fixes" it back — which is how a rejected approach gets
|
||||
re-proposed and a trap gets re-sprung. `docs/defect-shapes-773.md` carries both a dated measurement
|
||||
and a struck-through `shellcheck` row recording a *tested* negative result; both are reader-facing
|
||||
history and both must survive. The who-benefits test is what separates them from narrative, and it
|
||||
is stated as a question rather than a word list because a word list is exactly what the mechanised
|
||||
half of this cannot be trusted to be.
|
||||
|
||||
**Why enforcement is advisory and stays that way.** A narrative detector is a string predicate over
|
||||
prose. `docs/defect-shapes-773.md` §4 and `testing.guard-derives-population-from-source` both argue
|
||||
that a weak string-matching detector is the symptom-keyed mistake, and the withdrawn
|
||||
`test_review_verdict_vocabulary_parity.py` — six cold-review rounds, then deleted — is the empirical
|
||||
case: every round's fix was locally correct and the sequence never converged. As a blocking gate
|
||||
this is a bad bet. As a non-blocking nudge it is nearly free, and the repo already runs that exact
|
||||
pattern in `docs-reminder` for the parity doc. So the detector warns; it never fails a run. Budget
|
||||
for it being wrong sometimes, and make that acceptable by not letting it block.
|
||||
|
||||
**It is Python, not shell, and that is the same argument one level down.** A detector over diff output has to decide what each line IS, and deciding that from its prefix alone — without hunk state — is a string predicate too. Four distinct defects fell out of one bash implementation of it: the no-trailing-newline marker counted as content, an added line whose own text began `++ ` eaten by the `+++ ` header arm, `core.quotePath` hiding non-ASCII paths, and a final unterminated line dropped by `read`. Those are four sites of one mistake, so the mechanism was replaced rather than the sites patched one at a time. Anything that parses a diff here should parse it with hunk state or not at all.
|
||||
|
||||
**If this ever becomes a hook, it belongs at USER scope, not in this repo.** The rule is not
|
||||
ersatztv-specific — it is true of every repo — and an ersatztv-only hook would enforce it exactly
|
||||
where it was first noticed and nowhere else, which is the shape this record exists to close.
|
||||
Mechanically that means `~/.claude/hooks/` plus the user `settings.json`. Worth stating because
|
||||
`~/.claude/hooks/` holds only cosmetic hooks today (terminal title, statusline), so there is no
|
||||
precedent there for a content rule and someone would have to decide that deliberately.
|
||||
@@ -0,0 +1,104 @@
|
||||
---
|
||||
key: mcp.tool-schema-openapi-parity
|
||||
title: '2026-08-06 — every MCP tool declares exactly its endpoint''s OpenAPI request-body fields and query parameters, asserted in CI (#754, #757)'
|
||||
status: active
|
||||
since: '2026-08-06'
|
||||
supersedes: none
|
||||
superseded-by: none
|
||||
rule: 'Every POST/PUT/PATCH tool in `ToolCatalog` declares exactly the request-body properties its endpoint accepts, each with a matching type, and EVERY tool (read and write) declares exactly its endpoint''s query parameters, both asserted against the generated `ErsatzTV/wwwroot/openapi/v1.json` (linked into `ErsatzTV.Mcp.Tests`) by `Every_Write_Tool_Should_Declare_Exactly_Its_OpenApi_Request_Body_Fields` and `Every_Tool_Should_Declare_Exactly_Its_OpenApi_Query_Parameters`. A field the endpoint accepts but the tool omits is a DEFECT, not a deferral: on the full-replace tools (channel update, schedule update, custom-order) the omission is silently applied as a clear. The write tools are NOT uniformly full-replace — add-collection-items is additive, and several leave an omitted field unchanged — so each tool description states its own semantics. An omitted query parameter is UNREACHABLE, not merely undocumented, because `ToolArgumentValidator` rejects undeclared arguments.'
|
||||
signals: 'MCP tool schema drift, full-replace write, silently dropped field, graphicsElementIds, padToNearestMinute, additionalProperties false · paths: `ErsatzTV.Mcp/ToolCatalog.cs`, `ErsatzTV.Mcp.Tests/ToolCatalogTests.cs`, `docs/mcp.md` · issues: #754, #757, #58, #616'
|
||||
mechanics: '`ErsatzTV.Mcp.Tests/ToolCatalogTests.cs`; `ErsatzTV.Mcp.Tests.csproj` links `openapi/v1.json`'
|
||||
---
|
||||
|
||||
`ToolCatalog.ChannelFields()` declared 27 of `UpdateChannelRequest`'s 28 properties. The missing one
|
||||
was `graphicsElementIds`, which attaches channel-level graphics elements including the built-in On
|
||||
Now/Next overlay (`graphics.channel-level-attachment`).
|
||||
|
||||
The cost was not "one field you cannot set". `PUT /api/v1/channels/{id}` is a **full replace**, and
|
||||
the tool's own description instructs the caller to *"send the full desired state"* — which the schema
|
||||
could not express. An agent that faithfully GET-edit-PUT a channel detached every attached graphics
|
||||
element, with a `200` and no error. Nothing surfaced until the overlay stopped rendering at the next
|
||||
transition, hours later. That is the `optional-parameter-on-shared-primitive-is-opt-out` shape: the
|
||||
omission is invisible at the call site and only observable as missing pixels.
|
||||
|
||||
Fixing the one field would have left the mechanism intact, and the mechanism had already produced a
|
||||
second instance: `ScheduleFlags()` omitted `padToNearestMinute`, which both `CreateScheduleRequest`
|
||||
and `UpdateScheduleRequest` carry and `UpdateProgramScheduleHandler` writes unconditionally — so
|
||||
`ersatztv_update_schedule` silently cleared a configured pad the same way. Nothing tied a tool's
|
||||
declared arguments to the contract it wraps, so the next added DTO property would have drifted too.
|
||||
|
||||
So the guard is the decision, and it is asserted against the **generated OpenAPI document** rather
|
||||
than the DTO types: `v1.json` is the actual wire contract, it is already regenerated by
|
||||
`scripts/update-openapi.sh` as part of the API checklist, and asserting against it keeps
|
||||
`ErsatzTV.Mcp.Tests` free of a project reference to the whole ASP.NET host. The test derives each
|
||||
tool's body set exactly as `ErsatzTvApiClient` does — declared arguments minus path parameters, minus
|
||||
query parameters, minus the reserved `ifMatch` header — so the guard cannot disagree with the routing
|
||||
it guards.
|
||||
|
||||
Three anti-vacuity properties are deliberate, per the repo's standing "a test that filters on the
|
||||
property it asserts cannot see what is missing" rule:
|
||||
|
||||
- The **covered write-tool set is pinned by name**, not merely filtered. A tool that stops being a
|
||||
write verb, or a new one that is added, changes this list rather than silently leaving the loop.
|
||||
- A **missing or unrecognised spec is a failure**, never an empty comparison: an absent `v1.json`
|
||||
fails with the path it looked in, and a request body that is not a plain `$ref` (an `allOf`,
|
||||
`oneOf`, or inline schema), or a property whose type is a union this guard has not been taught,
|
||||
fails asking to be taught the shape instead of comparing against `{}`.
|
||||
- **Names are compared with types**, not alone. A name-only guard is the same defect one level down:
|
||||
the tool would advertise `string` for an `int?`, the agent would send `"30"`, and the API would
|
||||
reject it — green test, broken tool. The generator's `["null", T]` nullable form and its `$ref`
|
||||
(enum → `string`, model → `object`) are normalized onto the catalog's vocabulary, arrays down to
|
||||
their element type.
|
||||
|
||||
All were verified by mutation rather than assumed: dropping `graphicsElementIds`, dropping
|
||||
`padToNearestMinute`, retyping either field, drifting an array's element type, and removing the
|
||||
copied spec each turn the suite red, and each failure names the field or path at fault.
|
||||
|
||||
**Query parameters are guarded the same way, across every tool (#757).** A second test compares each
|
||||
tool's routed `QueryParameters` against the spec's `parameters[in=query]` for its path and verb, reads
|
||||
included — the drift that existed when this was written was entirely on reads. An omitted parameter
|
||||
there is worse than an undeclared body field: `additionalProperties:false` means the caller cannot
|
||||
pass it *at all*, so the capability is unreachable rather than merely undocumented (`ersatztv_list_playouts`
|
||||
had lost its channel-name `query` filter and `ersatztv_get_playout_items` its `showFiller`; #616 was
|
||||
the same shape with paging). That test **accumulates** its mismatches and asserts once, so a run
|
||||
reports the whole drift set — failing on the first would invite fixing one tool at a time, which is
|
||||
how the twin in this very issue stayed hidden.
|
||||
|
||||
It also **composes with** the older `Every_Query_Parameter_Should_Be_A_Declared_Property`, and the pair
|
||||
is the clearest illustration in this repo of why "a test that filters on the property it asserts cannot
|
||||
see what is missing" is a rule. That older test filters `Where(t => t.QueryParameters is { Count: > 0 })`
|
||||
— so a tool that lost its query parameters entirely escaped it, which is exactly how `list_playouts` and
|
||||
`get_playout_items` hid. The new test has no filter and reports them as *unreachable*; the old one then
|
||||
checks that a routed parameter is also a declared argument. Neither subsumes the other, and the inner
|
||||
duplicate of the old check was deliberately removed from the new test rather than kept as a second copy.
|
||||
|
||||
**Scope, stated so it is not mistaken for more.** Request bodies are compared for POST/PUT/PATCH only.
|
||||
DELETE is uncovered because `ErsatzTvApiClient` builds a body for POST/PUT/PATCH only, so a body
|
||||
argument on a DELETE tool would be silently dropped; no tool has one today. Header arguments (`ifMatch`)
|
||||
and per-parameter *descriptions* are not compared either — `api.paging-zero-based` is pinned by its own
|
||||
test.
|
||||
|
||||
The type comparison is **lossy by design, at the catalog's ceiling**: the catalog's vocabulary is
|
||||
`{string, integer, number, boolean, object, array<T>}`, so every object component collapses to `object`
|
||||
and every enum to `string`. Swapping one model or enum for another is therefore invisible here
|
||||
(verified by repointing `logo` at a structurally unrelated model — the suite stays green), as is
|
||||
`format` (`int32` vs `int64`). That is the right ceiling rather than a gap to close: comparing deeper
|
||||
than the catalog can express would assert a distinction no tool schema carries, and an opaque object
|
||||
like `logo` is copied through from a GET verbatim, so nested drift cannot cause the silent-clear this
|
||||
record exists to prevent. `integer` vs `number` IS distinguished. The `>1` non-null type-union
|
||||
assertion is a fail-loud guard for a shape this generator does not currently emit, so it is deliberate
|
||||
but **unexercised**.
|
||||
|
||||
The guard is also a **two-job conjunction**, not self-contained: it compares against a checked-in
|
||||
`v1.json`, so it is only as fresh as the regeneration. What keeps it honest is the `api-docs` CI job,
|
||||
whose `^ErsatzTV/Controllers/Api/` path filter covers the directory every request DTO lives in — a
|
||||
new DTO property cannot leave `v1.json` stale without that job going red. That holds for a DTO's OWN
|
||||
properties and no further: a NESTED model such as `ArtworkContentTypeModel` lives in
|
||||
`ErsatzTV.Application/Artworks/`, outside that filter, so changing it can leave `v1.json` stale without
|
||||
the job firing. Pre-existing, and harmless to this guard only because nested shape is not compared.
|
||||
|
||||
`graphicsElementIds` is declared on the **update tool only**, not in the shared `ChannelFields()`:
|
||||
`CreateChannelRequest` has no such property, and the tool schemas are `additionalProperties:false`,
|
||||
so sharing it would make every create call send an unknown property. `padToNearestMinute` is on both
|
||||
schedule requests, so it does belong in the shared `ScheduleFlags()`. The parity test is what makes
|
||||
that per-field placement checkable rather than a matter of care.
|
||||
@@ -1,11 +1,11 @@
|
||||
---
|
||||
key: process.bom-format-detection-recipe
|
||||
title: '2026-07-21 — BOM/format pre-push detection: use the `xxd` loop, and run `dotnet format --include` under `bash -c` (#542)'
|
||||
title: '2026-07-21 — BOM/format pre-push detection: use the `od` byte check, and run `dotnet format --include` under `bash -c` (#542, detector corrected #797)'
|
||||
status: active
|
||||
since: '2026-07-21'
|
||||
supersedes: none
|
||||
superseded-by: none
|
||||
rule: Before any push touching `.cs`, detect BOMs with the `xxd` byte check and verify the format gate with `dotnet format --include` run under `bash -c`, never bare zsh.
|
||||
rule: 'Before any push touching `.cs`, detect BOMs with the `od -A n -t x1 -N 3` byte check and verify the format gate with `dotnet format --include` run under `bash -c`, never bare zsh. NOT `xxd`: it ships with vim and is absent on plain Linux hosts including this repo''s CI runner, where the substitution yields empty, never matches, and the check reports all-clean — the same all-clean-detector failure this record was written about, in the detector it prescribed.'
|
||||
signals: 'UTF-8 BOM · `efbbbf` · `dotnet format --verify-no-changes` · `--include` · `mapfile` · zsh vs bash · #311 format gate · detector verification · paths: `ErsatzTV.sln`, `.editorconfig` · issues: #542, #311, #70, PR #402, PR #405'
|
||||
mechanics: Local pre-push shell; CI's format job recipe (`ci.format-gate-folder-mode`). The BOM *policy* is `release.format-as-you-touch-rebase`; this record is the mechanics only.
|
||||
---
|
||||
@@ -13,11 +13,25 @@ mechanics: Local pre-push shell; CI's format job recipe (`ci.format-gate-folder-
|
||||
Detection loop, verbatim:
|
||||
|
||||
```bash
|
||||
for f in $(git diff --name-only origin/main...HEAD -- '*.cs'); do head -c3 "$f" | xxd -p | grep -q "^efbbbf" && echo "BOM: $f"; done
|
||||
for f in $(git diff --name-only origin/main...HEAD -- '*.cs'); do [ "$(od -A n -t x1 -N 3 < "$f" | tr -d ' \n')" = efbbbf ] && echo "BOM: $f"; done
|
||||
```
|
||||
|
||||
**Verify your detector.** An `od -An -c | grep '357 273 277'` check reported all-clean while 19 files
|
||||
were actually dirty. A detector that can only say "ok" is worse than none — use the `xxd` form or read
|
||||
**The prescribed detector was itself an all-clean detector, corrected 2026-08-14 (#797).** This
|
||||
record was written because a detector that can only say "ok" is worse than none. It then prescribed
|
||||
`xxd -p`, and **`xxd` ships with vim and is absent on plain Linux hosts, including this repo's CI
|
||||
runner** — where the substitution yields the empty string, never matches `efbbbf`, and the loop
|
||||
prints nothing for a tree full of BOMs. The rule failed its own test, in the recipe it recommends,
|
||||
for three weeks.
|
||||
|
||||
Read the original warning precisely, because it is still right and it is not about `od`: the form
|
||||
that reported all-clean over 19 dirty files was `od -An -c | grep '357 273 277'` — octal character
|
||||
output, whose spacing and escaping vary. The replacement is `od -A n -t x1 -N 3`, which emits
|
||||
hexadecimal bytes. That is a different invocation of the same tool, and it was measured across 17
|
||||
inputs (empty, sub-3-byte, exactly the BOM, BOM+NUL, binary, UTF-16 BOM, unreadable, 20 MB, awkward
|
||||
filenames) on BSD `od`, GNU `od`, and inside the actual CI runner image: identical in every cell.
|
||||
`tr -d ' \n'` is load-bearing — BSD pads to a fixed column width and GNU does not.
|
||||
|
||||
**Verify your detector** — use the form above, or read
|
||||
the bytes directly, and distrust a clean result you didn't prove can go dirty.
|
||||
|
||||
**`dotnet format --include` DOES work here** — an earlier note claiming it silently no-ops was WRONG.
|
||||
|
||||
@@ -0,0 +1,138 @@
|
||||
---
|
||||
key: process.check-and-use-pins-a-version
|
||||
title: '2026-08-16 — a check and the action it authorizes are bound to one version, or the gap is stated and fenced (#778)'
|
||||
status: active
|
||||
since: '2026-08-16'
|
||||
supersedes: none
|
||||
superseded-by: none
|
||||
rule: 'Where a CHECK authorizes an ACTION over state that can change in between, the two are bound to ONE version of that state. Binding alone is not enough and is the half that keeps being skipped: a snapshot nothing re-validates is not pinned, it is a stale read wearing a version number. Three substrates, three mechanisms, and they are the SAME rule — in-process, a compare-exchange claim taken by the caller, never a `Volatile.Read` in one place and an `Interlocked` in another (`ffmpeg.work-ahead-slot-atomic`); over our own HTTP API, RFC 7232 `If-Match`/ETag, with the force-write path named explicitly rather than left implicit (`concurrency.ifmatch-rfc7232`, `concurrency.force-write-non-ifmatch`); against a remote service, a full commit sha, an image digest or a monotonic event count re-read immediately before the write. Prefer true compare-and-set where the server offers it. Where it does not — Gitea''s commit-status API has no ETag, no If-Match and no expected-previous-state — the ceiling is READ-COMPARE-REFUSE: re-read the identifier immediately before the write and FAIL CLOSED on any movement, which narrows the window to one round trip and makes the loss observable instead of silent. A residual that cannot be closed is STATED in the code and carried in `docs/remote-state-inventory.md` as `UNSAFE-KNOWN` with the reason it is tolerable; "noticed" is not "accepted". Two identifier traps are load-bearing here: compare the FULL sha, never a 7-char prefix, and compare a base BRANCH REF rather than its tip sha, because the tip moves on every unrelated merge and comparing it deadlocks every open PR. Finally, and this is the failure #778 actually found: a mitigation that lives OUTSIDE the code relying on it — branch protection, a required status context, a server-side refusal — must be VERIFIED at the point of use, not asserted in a comment or in the reason string a human reads. A dated claim about configuration is not a check, and it is worse than no claim, because it talks the next reader out of looking.'
|
||||
signals: 'check-and-use race · TOCTOU over remote state · pin a version or compare-and-set · read-compare-refuse · fail closed on movement · snapshot that stops being true · full sha never a 7-char prefix · base ref not base tip sha · monotonic event count not a branch name · ABA · required status check verified not asserted · the mitigation lives outside the code that relies on it · `UNSAFE-KNOWN` with a stated reason · paths: `docs/remote-state-inventory.md`, `scripts/tests/test_remote_state_inventory.py`, `.claude/hooks/pretooluse-merge-consent.sh`, `scripts/post-review-verdict.sh`, `scripts/pr-changed-files.sh` · issues: #778, #773, #707, #706, #632, #622, #536'
|
||||
mechanics: 'The detector is not a linter — there is no way to spot "this code should have pinned a sha". It is `testing.guard-derives-population-from-source` applied to an enumerated inventory: `docs/remote-state-inventory.md` classifies every in-scope executable `PINNED`/`CAS`/`UNSAFE-KNOWN`/`N/A`, and `scripts/tests/test_remote_state_inventory.py` derives the population from `git ls-files` — the index, never the filesystem, which reports untracked build output and differs per machine — and asserts set equality both ways, so a new script that talks to a remote service cannot ship unclassified. Nothing checks that a `PINNED` claim is true; that stays with review.'
|
||||
---
|
||||
|
||||
`docs/defect-shapes-773.md` §3 names this as **Family D**, the one class #773's taxonomy had no
|
||||
bucket for at all. Three raters proposed it unprompted; a fourth, working blind, proposed it again
|
||||
under its own name. Five records: #536, #622, #632, #706, #707.
|
||||
|
||||
**The unifying property, and why the name matters.** A check and the action it authorizes are
|
||||
separated in time over state that can change in between, with nothing pinning a version. #622's
|
||||
own record states it exactly: *"The gate was never bypassed — it was satisfied against a snapshot
|
||||
that stops being true."* That sentence is the whole class. Nothing is mis-scoped and no predicate
|
||||
is wrong; the answer was simply computed about a different world than the one the action lands in.
|
||||
|
||||
**The repo had already solved this twice without noticing it was one problem.** #536 was fixed with
|
||||
a compare-exchange claim. The whole `/api/v1` write surface was given RFC 7232 `If-Match` a year
|
||||
earlier. Both are this rule; neither pointed at the other, and the tooling — the third substrate,
|
||||
where the state is somebody else's server — got the fix a third time from scratch at #706 and #707.
|
||||
That is the same one-record-per-instance growth `defect-shapes-773.md` §4 criticises in this
|
||||
repo's own knowledge base, and it is the reason this record is written at the class level.
|
||||
|
||||
**Binding is the easy half; re-validation is the half that gets skipped.** A sha captured into a
|
||||
variable and then used in a URL feels pinned and is not. `scripts/post-review-verdict.sh` is the
|
||||
worked example of doing it properly: it re-reads the PR immediately before the status POST,
|
||||
compares both `.head.sha` and `.base.ref`, and `die`s without writing anything on either mismatch.
|
||||
The comparison, not the capture, is what makes it safe.
|
||||
|
||||
**Where no compare-and-set exists, say so instead of implying one.** Gitea's status API offers no
|
||||
conditional write, so `review-verdict.yml` cannot make its read and its POST one operation. It
|
||||
narrows the window twice — a monotonic `change_target_branch` event-count fence, and a
|
||||
high-water-mark re-read that repairs a `success` posted over a human verdict back to `pending` —
|
||||
and then states the remaining gap in the file. The count is used rather than the branch NAME
|
||||
because a name is ABA-vulnerable: `main -> S -> main` reads `main` at both ends, which is how #698
|
||||
route 1 obtained a forged exemption. An honest residual is a design output. A file claiming
|
||||
atomicity it does not have is the thing that stops getting re-examined.
|
||||
|
||||
**The failure this issue actually found, which none of the five records predicted.** The
|
||||
merge-consent hook's scheduled-auto-merge path is safe only because `review-verdict/h10` is a
|
||||
REQUIRED status check on `main` — a commit status belongs to one sha, so a commit pushed after
|
||||
scheduling cannot inherit the verdict and Gitea refuses the merge. That is #622's fix and it
|
||||
works. But it is *branch-protection configuration*. It lives outside this repo, nothing in the
|
||||
repo compared the two, and the hook asserted it in a comment **and in the grant reason a human
|
||||
reads**:
|
||||
*"because the verdict status is bound to this sha, a commit pushed before Gitea merges will clear it
|
||||
and block the merge."* Switch that context off and every word of that sentence becomes false while
|
||||
the hook keeps printing it and keeps auto-granting.
|
||||
|
||||
So the class has a second face: not only "the state moved between the check and the action", but
|
||||
"the thing that made the action safe was never observed at all". The hook now reads the repo's FULL
|
||||
rule list, `GET /repos/{owner}/{repo}/branch_protections`, and classifies it — present proceeds,
|
||||
unreadable **asks** (a transient failure, or a credential without the repo-admin scope that endpoint
|
||||
needs, is not evidence of safety), nothing-can-govern **denies**, and anything it cannot decide
|
||||
**asks**. Denying on absence is the point: that is #622's hole reopened, and its defining property
|
||||
is that it is silent from the merge caller's side.
|
||||
|
||||
**Reading the LIST rather than the rule named after the base is the load-bearing choice**, and the
|
||||
first version got it wrong in the way this record is about. `GET …/branch_protections/{name}` is an
|
||||
exact database lookup that performs no matching and knows nothing about precedence, so a 200 from it
|
||||
establishes only that a rule with that NAME lists the context — never that the context is required
|
||||
ON that branch. Gitea resolves the governing rule by Priority first and plain-name-ness second, so a
|
||||
glob rule can outrank an exactly-named one. Using the by-name endpoint first and the list only on a
|
||||
404 therefore guarded the 404 path while the 200 path — the one that actually fires, since the rule
|
||||
IS named `main` — granted without the check at all: hardened code that was dead, beside live code
|
||||
that was not. It was fixed by DELETING the by-name path, not by documenting the gap, so there is one
|
||||
fetch, one classifier and one argument to keep true. Absence is now established by the classifier
|
||||
over a list that WAS read, never by an HTTP status, because a 404 from the list endpoint means the
|
||||
repo was not found rather than that the branch is unprotected.
|
||||
|
||||
**And that check is a preflight, not a pin — say so, because the temptation is to bank it.** The
|
||||
first draft of the inventory graded that path `PINNED`; cold review pointed out that the hook's
|
||||
own comment concedes the read pins nothing, and it was right. Branch protection has no version,
|
||||
ETag or conditional read, so an admin can still weaken it after the hook looks. What the check
|
||||
buys is drift DETECTION and the removal of an unobserved assumption, which is the honest ceiling
|
||||
for that API. The residual is BOUNDED, not closed, and the bound is a trust assumption worth
|
||||
naming: everything on that path assumes repo-admin branch-protection config is not hostile. Saying
|
||||
it was "closed one layer down" by the very protection an admin may have removed was circular — the
|
||||
same sentence appeared in the inventory and was rewritten there first, which is how a stale twin
|
||||
survives a fix round. A row claiming otherwise would be exactly the overclaim this record warns
|
||||
about, and it is recorded here because the record's own deliverable made it on the first pass.
|
||||
|
||||
This is the same gap `testing.guard-derives-population-from-source` already flagged one directory
|
||||
over — `MARKED_JOBS` in `test_ci_dropped_step_guard.py` is a hand-written mirror of those same
|
||||
required contexts, annotated with a date. A dated comment is a claim about the past. Two
|
||||
independent guards now depend on that configuration; one of them checks it.
|
||||
|
||||
**The deliverable's own population was wrong three times, and that is the most transferable part of
|
||||
this record.** The inventory is the detector, so its population *is* the guard. Round one filtered
|
||||
the scope on an outbound-network token list, which omitted `git fetch` — this repo's commonest
|
||||
remote read — so a hook that fetches `origin/main` and derives a push decision was invisible.
|
||||
Round two dropped the filter but used non-recursive `glob`, so four nested files stayed out, one
|
||||
of them calling a live ErsatzTV API and acting on the reply. Round three used `rglob`, which is
|
||||
recursive and therefore also enumerated `.husky/_/` — untracked, gitignored shims that `npm ci`
|
||||
generates — leaving the guard **red on every developer checkout and green in CI**, which never
|
||||
runs `npm ci`. A guard that fails everywhere except where it runs teaches its readers to ignore
|
||||
it, which is worse than no guard at all.
|
||||
|
||||
Every round shipped with an argument for why the traversal was sufficient, and every argument was
|
||||
wrong the same way. The fix that finally held was not a better traversal: it was **deriving the
|
||||
population from `git ls-files`**. The filesystem is not an authoritative source — it reports build
|
||||
output, editor droppings and whatever else is on disk, and it differs per machine. The index is
|
||||
authoritative, versioned, identical for CI and every checkout, and excludes untracked generated
|
||||
files by construction rather than by an exclusion list somebody has to maintain. So the
|
||||
generalisation is the one `testing.guard-derives-population-from-source` already states, applied
|
||||
one level up: when a guard enumerates a population, ask **what is the authoritative list of these
|
||||
things** — and if the answer is "whatever the walk finds", the guard is not finished, however
|
||||
carefully the walk is written.
|
||||
|
||||
**Grade down before you argue.** Three cold-review rounds demoted row after row — the scheduled-merge
|
||||
path, both merge-consent head/base reads, the file enumerator, every registry-tag row — and in
|
||||
each case the row asserted a property the code did not have while the code beneath it was fine.
|
||||
Only **three** rows survive as `PINNED`, out of roughly seventy. That is the honest finding about
|
||||
this class in a tooling codebase: almost nothing that talks to a remote service is genuinely
|
||||
pinned, most of it is bounded by an argument, and the argument is what has to be written down.
|
||||
|
||||
The denominator is deliberately approximate, and that is a finding rather than laziness. Written
|
||||
as an exact "N of M" it went stale **three times in three rounds** — twice because a demotion
|
||||
landed after the count, once because splitting one row into two moved M inside the very commit
|
||||
that cited it. A figure that changes whenever the artifact it describes is edited is a second copy
|
||||
of that artifact, and this repo already knows what to do about a hand-maintained mirror: give it
|
||||
an equality check or stop maintaining it. `docs/guard-inventory.md` took the first route because
|
||||
its counts are the point; a rationale record takes the second, because the load-bearing claim here
|
||||
is "almost nothing is pinned", not any particular integer. A row that overstates is worse than a
|
||||
row that admits a gap, because this repo's own record is that a guard described as sound stops
|
||||
being re-examined.
|
||||
|
||||
**What this record does not claim.** The inventory grades *files*, so it cannot see an existing file
|
||||
growing a second unpinned read — the sites-in-code limit tracked in #777. And nothing verifies
|
||||
that a row marked `PINNED` is telling the truth. Both residues are stated in
|
||||
`docs/remote-state-inventory.md` rather than papered over, because a guard described as sound
|
||||
stops being re-examined, which is the failure mode this whole family is made of.
|
||||
@@ -5,8 +5,8 @@ status: active
|
||||
since: '2026-07-12'
|
||||
supersedes: none
|
||||
superseded-by: none
|
||||
rule: A blocking `format` CI job runs `dotnet format --verify-no-changes` scoped only to the PR's changed `.cs` files (never the legacy BOM backlog), and a PR branch must be kept current by rebasing on `origin/main` (never merging main in), enforced by `.husky/pre-push` → `prepush-rebase-check.sh`.
|
||||
signals: 'format-as-you-touch, rebase not merge, BOM backlog · paths: `.husky/pre-push`, `.claude/hooks/prepush-rebase-check.sh` · issues: #311 (H11), #309, #310, #269, #312'
|
||||
rule: 'A blocking `format` CI job runs `dotnet format --verify-no-changes` scoped only to the PR''s changed `.cs` files (never the legacy BOM backlog), and a PR branch must be kept current by rebasing on `origin/main` (never merging main in), enforced by `.husky/pre-push` → `prepush-rebase-check.sh`. H11 has ONE always-on carve-out, #719 — a push in which EVERY ref is under `refs/tags/` skips the freshness check, because a tag push cannot revert merged work, which is the failure mode H11 exists to prevent, and the release cut tags from a branch that is behind `origin/main` (observed on the v26.13.0 cut, #719). A push mixing branch and tag refs is still blocked, and so is a push with zero parsed ref lines (the exemption requires at least one, so empty stdin cannot vacuously disable H11).'
|
||||
signals: 'format-as-you-touch, rebase not merge, BOM backlog, tag-only push exemption, H11 blocks release cut, refs/tags pre-push, vacuous-truth guard · paths: `.husky/pre-push`, `.claude/hooks/prepush-rebase-check.sh`, `scripts/tests/test_prepush_rebase_check_tag_exemption.py` · issues: #311 (H11), #719, #309, #310, #269, #312'
|
||||
mechanics: '`docs/contributing.md` §7; `.claude/hooks/prepush-rebase-check.sh`; `npm run check:api`'
|
||||
---
|
||||
|
||||
@@ -36,5 +36,22 @@ git hook has no "ask"); deliberate escape `ETV_SKIP_REBASE_CHECK=1`. This supers
|
||||
guidance to "merge main into your PR branch." (After a rebase that conflicts in *generated* artifacts —
|
||||
v1.json/v1.d.ts/endpoint-index — regenerate, don't hand-resolve; `npm run check:api` guards.)
|
||||
|
||||
**2a. The tag-only carve-out (#719).** H11 fired on the release cut: tagging a commit on `main` from
|
||||
a branch that is behind `origin/main` tripped the freshness check, and the rebase advice it printed
|
||||
did not even apply — no branch was being pushed. Observed while cutting `v26.13.0` (#719); note
|
||||
`docs/ci-cd.md` → "Cutting a release" documents the tag step itself, not the release-notes-PR flow
|
||||
that leaves the branch behind, so the frequency is attested by #719 rather than by that doc. The hook now reads git's pre-push ref lines (`<local ref> <local sha>
|
||||
<remote ref> <remote sha>`) and exits 0 when every parsed line's *remote* ref is under `refs/tags/`.
|
||||
Two details are load-bearing and easy to regress:
|
||||
- `.husky/pre-push` consumes stdin into `$_prepush_refs` before any guard runs, so it must **forward**
|
||||
those lines (`printf '%s\n' "$_prepush_refs" | …`). Without that the check receives EOF and the
|
||||
exemption is dead code that silently never fires. The unit tests drive the hook directly and would
|
||||
still pass, so this wiring is not covered by them.
|
||||
- The exemption requires **at least one** parsed ref line. "All refs are tags" is vacuously true for
|
||||
zero lines, which would disable H11 for every push; with no lines the hook falls through to the
|
||||
normal freshness check. `scripts/tests/test_prepush_rebase_check_tag_exemption.py` pins both the
|
||||
negative control (branch push from a behind branch still blocked), the mixed branch+tag case, and
|
||||
the two zero-line cases.
|
||||
|
||||
Rationale, as with the whole hook program: make the process rule a derivation/hook, not prose to
|
||||
remember (#303 methodology review). Tracked: #311; sibling #312 (H12 issue-qualification audit).
|
||||
|
||||
@@ -0,0 +1,79 @@
|
||||
---
|
||||
key: release.main-direct-push-disabled
|
||||
title: '2026-08-05 — `main` refuses direct pushes (`enable_push: false`), because a push whitelist would have been a no-op here (#743)'
|
||||
status: active
|
||||
since: '2026-08-05'
|
||||
supersedes: none
|
||||
superseded-by: none
|
||||
rule: 'Branch protection on `main` carries `enable_push: false` AND `block_admin_merge_override: true`. Both halves are required and neither is sufficient. `enable_push: false` removes the direct-push path, leaving the PR merge path — the only path on which Gitea evaluates `status_check_contexts`, and therefore the only path on which `review-verdict/h10` is consulted at all. `block_admin_merge_override: true` then closes the force-merge bypass on that remaining path: with it false (the default), `CanBypassBranchProtection` returns true for a repo admin, so `POST /pulls/{n}/merge` with `force_merge: true` merges a PR whose `h10` is missing or red — one API call, no forgery, no PATCH. Do NOT "soften" the push half to a push WHITELIST: measured here, a whitelist naming `timothy` still admits the push, and `timothy` is the identity every agent session, PAT and injected `GITEA_TOKEN` already acts as, so the whitelist form closes nothing while reading in review as a control. Same reasoning is why the admin-override half is needed: an admin-shaped control that exempts the only admin exempts everybody. What remains open: a credential that can PATCH branch protection off can still undo either half — an accepted residual, not a closed route. Tag pushes are unaffected (`tag_protections` governs those separately), so the release cut still works.'
|
||||
signals: 'direct push to main, push whitelist, enable_push false, branch protection bypass, review-verdict/h10 bypassable without forging, merge consent derived not asserted, pre-receive hook declined, Not allowed to push to protected branch, protected branch, tag_protections, release tag push, GITEA_TOKEN repo write, RENOVATE_TOKEN, site admin bypass, PR-only flow · paths: `docs/ci-cd.md` · issues: #743, #697, #698, #622, #672, #706, #742, server-management#714'
|
||||
mechanics: 'Gitea 1.27.1. `PATCH /api/v1/repos/timothy/ersatztv/branch_protections/main` with `{"enable_push": false, "block_admin_merge_override": true}`; whitelist fields left off (`enable_push_whitelist: false`, empty arrays), `enable_force_push: false`, `enable_merge_whitelist: false`, `required_approvals: 0`. MEASURED 2026-08-05 against a throwaway `probe-743-*` rule rather than against `main`: with `enable_push: false` a push by `timothy` (site admin) was REFUSED — `pre-receive hook declined`, `Not allowed to push to protected branch`; after PATCHing the same rule to `enable_push: true` + `enable_push_whitelist: true` + `push_whitelist_usernames: ["timothy"]` the identical push SUCCEEDED. Separately probed on a second throwaway rule: a contents-API write (`PUT /repos/{o}/{r}/contents/{path}` with `branch` set to the protected branch) was REFUSED HTTP 403 `user cannot commit to repo [user: timothy]` — so the web-editor/API file-write surface does not bypass it either. Then on `main` itself: `git push origin HEAD:main` REFUSED, and a tag-only push SUCCEEDED from the same worktree. `GET .../tag_protections` returns `[]`; repo is `fork: false`, `mirror: false`. NOT measured, source-attested only (Gitea 1.27 `CanBypassBranchProtection`, `services/pull/check.go`, `routers/private/hook_pre_receive.go`): that `block_admin_merge_override: false` would have let an admin `force_merge` past the required contexts — the field was set to true rather than probed, since probing it means merging an unreviewed PR. All probe artifacts (two rules, two branches, one tag) deleted and confirmed gone; `origin/main` head unchanged at `08e95f9ec` throughout.'
|
||||
---
|
||||
|
||||
**Why a whitelist was the wrong shape.** #743 proposed "a push whitelist on `main` (or disable direct
|
||||
push entirely)" as if the two were interchangeable. They are not, and which one is right depends on a
|
||||
fact about *this* instance: the only accounts with repository write are `timothy` (a site admin) and
|
||||
`renovate`. Every credential in the threat model — an agent session, a collaborator PAT, the
|
||||
`GITEA_TOKEN` Gitea injects into every Actions job — authenticates as one of those two, and
|
||||
overwhelmingly as `timothy`. A whitelist admitting `timothy` therefore admits precisely the identity
|
||||
the control is supposed to constrain. It would have ticked the issue's box while changing nothing.
|
||||
This was measured, not reasoned: the same push was refused under `enable_push: false` and accepted
|
||||
under a whitelist naming `timothy`.
|
||||
|
||||
**Disabling push alone was NOT enough, and the reason is the same argument twice.** The first draft of
|
||||
this record disabled direct push and concluded that `review-verdict/h10` was therefore load-bearing.
|
||||
An independent review caught that this repeated on the merge path exactly the mistake it had just
|
||||
diagnosed on the push path. The push argument was: a whitelist naming `timothy` fails because
|
||||
`timothy` is the identity every credential already holds. The merge path had the identical shape —
|
||||
`block_admin_merge_override` defaulted to `false`, so `CanBypassBranchProtection` returned true for a
|
||||
repo admin and `POST /pulls/{n}/merge` with `force_merge: true` merged straight past a missing or red
|
||||
`h10`. One API call, cheaper than the push route it replaced. **An admin-shaped control that exempts
|
||||
the only admin exempts everybody.** Both fields are now set; treat them as one control, and never
|
||||
cite `enable_push: false` alone as the reason the gate holds.
|
||||
|
||||
**What this actually closes, and what it does not.** It closes the *write-only* credential routes,
|
||||
which is most of #743's own "who can do it" list: the injected `GITEA_TOKEN` (repo write, not admin),
|
||||
`RENOVATE_TOKEN`, and any non-admin collaborator PAT. Those can no longer reach `main` at all, by any
|
||||
path that skips the gate.
|
||||
|
||||
It does **not** close the admin route. `timothy` is a site admin, so a credential holding that
|
||||
identity can `PATCH` either field off, act, and restore it — the exact sequence used to *prove* the
|
||||
push semantics above. Closing that requires agent sessions to run as a scoped non-admin credential,
|
||||
which is a different change with its own cost (packages live in a user namespace; see the "Admin
|
||||
ownership is a real residual" section of `ci.actions-credential-scoping`). Recorded as an accepted
|
||||
residual rather than fixed here, so it is not mistaken for covered. The severity bound from #697 and
|
||||
#743 is unchanged throughout: push access is required, so this is a compromised contributor or a
|
||||
subverted automated session, never an anonymous attacker.
|
||||
|
||||
**Which write surfaces were enumerated.** `git push` (measured, refused), the contents API and by
|
||||
extension the web editor / upload path (measured on a probe branch, refused HTTP 403 — they share the
|
||||
`CanUserPush` predicate, which has no admin special-case and no `unprotected_file_patterns` carve-out
|
||||
since that field is empty), apply-patch / revert / cherry-pick (source-attested, same predicate),
|
||||
force push (`enable_force_push: false`), default-branch deletion (separately refused), and fork-sync /
|
||||
mirror (not applicable: `fork: false`, `mirror: false`). Merge remains the one intended path.
|
||||
|
||||
**Why the release cut does not deadlock.** #743 flagged that the tag path had to keep working, and
|
||||
#719 documents H11 blocking a tag-only push on every release cut. Branch protection is scoped to
|
||||
`refs/heads/main`; tags are governed by an entirely separate mechanism, and `tag_protections` on this
|
||||
repo is empty, so tag pushes are unrestricted by anything except ordinary write permission. Demonstrated
|
||||
rather than assumed: from one worktree, the branch push to `main` was refused and a tag push succeeded.
|
||||
Do not conflate the two mechanisms — disabling branch push says nothing about tags, and a future
|
||||
tag-protection rule would not inherit from this one.
|
||||
|
||||
**The `docker-build.yml` `persist-credentials` question (#743's fourth box), decided and deferred.**
|
||||
Its six `actions/checkout` steps omit `persist-credentials: false`, so a head-resolved job keeps a
|
||||
write-capable credential in `.git/config`. It *should* be set — but not blind, and not in this PR,
|
||||
because two steps run `git fetch --no-tags --depth=100 origin "$base_ref" || true` and feed the result
|
||||
into the changed-file skip logic. That `|| true` means a credential regression does not fail the job;
|
||||
it silently yields an empty changed-file set, and the skip logic then reads "nothing changed". The repo
|
||||
is public, so anonymous fetch is *expected* to cover it — expected is not measured, and the failure
|
||||
mode is silent, which is the shape that has burned this repo before. The correct order is: drop the
|
||||
`|| true` masking so a fetch failure is loud, then set `persist-credentials: false` and confirm both
|
||||
jobs still compute a non-empty changed set on a PR that genuinely changes files.
|
||||
|
||||
**Why this is not redundant with the Husky pre-push hooks.** `.husky/pre-push` guards (H6 done-when,
|
||||
H11 rebase, H13 clean worktree) are client-side and deliberately fail-open — a git hook cannot prompt.
|
||||
They are not installed in CI, not present in a fresh clone until `husky` runs, and `--no-verify`
|
||||
bypasses them, which the worktree workflow uses routinely. They are good friction against mistakes and
|
||||
were never a control against a credential. This record is the server-side half; the hooks remain useful
|
||||
and unchanged.
|
||||
@@ -182,8 +182,24 @@ described as one:
|
||||
|
||||
**Only this file's instance is closed, not the class.** Any head-resolved workflow holding
|
||||
credentials that can POST a commit status can still forge `review-verdict/h10`;
|
||||
`docker-build.yml` demonstrably can, and must stay head-resolved because it builds the PR's own
|
||||
code. Tracked in #697. So the "careless change rather than a hostile one" posture below still
|
||||
`docker-build.yml` demonstrably could, and must stay head-resolved because it builds the PR's own
|
||||
code — so #697 scoped its credential instead (`ci.actions-credential-scoping`), leaving AT LEAST
|
||||
these: the injected `GITEA_TOKEN` (posts with `creator: null`), `RENOVATE_TOKEN` (a
|
||||
`write:repository` bot PAT in the same secret store, so it posts with a real creator and IS
|
||||
inherited, #742), a collaborator's own token, and the `v*` tag push — which matters less for
|
||||
forging this status than for what else it does: `docker-build.yml` publishes `:prod` from a tagged
|
||||
ref, and a tag may point at any commit, so it ships a prod image with no PR, review or status.
|
||||
None of which used to be even required — direct pushes to `main` were server-side permitted, so
|
||||
the gate could be skipped without forging anything (#743). **That route is now closed**
|
||||
(`release.main-direct-push-disabled`): `main` carries `enable_push: false` *and*
|
||||
`block_admin_merge_override: true`, so every change reaches `main` through the PR merge path,
|
||||
which is the only path on which these required contexts are evaluated. What survives is the
|
||||
forgery list above — those routes post a status rather than skip it, so they are still real —
|
||||
**plus one skip route that is not forgery at all**: a credential that can `PATCH` branch
|
||||
protection can turn either field off, act, and restore it. `timothy` is a site admin, so every
|
||||
session holds that capability; it is an accepted residual, recorded in
|
||||
`release.main-direct-push-disabled` and `ci.actions-credential-scoping`, not a closed route. So
|
||||
the "careless change rather than a hostile one" posture below still
|
||||
describes the repo accurately — it is simply no longer *this* workflow that is the weakest link.
|
||||
An untrusted-contributor repo would still need the classification moved somewhere no PR can
|
||||
reach (server-side policy), not merely a base-pinned definition.
|
||||
|
||||
@@ -0,0 +1,55 @@
|
||||
---
|
||||
key: session.local-code-intelligence
|
||||
title: '2026-08-14 — Brief subagents at the csharp-lsp MCP tools; no subagent has been observed to reach the LSP tool (#777)'
|
||||
status: active
|
||||
since: '2026-08-14'
|
||||
supersedes: none
|
||||
superseded-by: none
|
||||
rule: 'C# and TypeScript find-all-references are available again; brief delegated agents to the `csharp-lsp` MCP tools (`csharp_references`, `csharp_diagnostics`, …) rather than the `LSP` tool, which no dispatched subagent has been observed to resolve (Claude Code 2.1.232, agent types `general-purpose` and `Explore`, 2026-08-14). Preconditions are machine-local — `env.DOTNET_ROOT` in `.claude/settings.local.json` and a root `node_modules/typescript` link — and checkable with `scripts/check-local-lsp.sh`.'
|
||||
signals: 'find all references, findReferences, csharp-ls, typescript-language-server, LSP tool, csharp-lsp MCP, libhostfxr, MSBuildLocator, DOTNET_ROOT, subagent tool availability · paths: `docs/local-lsp-tooling.md`, `scripts/check-local-lsp.sh`, `.claude/settings.local.json`, `.mcp.json` · issues: #777, #773, #403, #671'
|
||||
mechanics: '`docs/local-lsp-tooling.md` — surfaces, configuration, traps, verification'
|
||||
---
|
||||
|
||||
**The capability that would address the residue of Family A was configured, enabled, and dead.** The
|
||||
population that `testing.guard-derives-population-from-source` cannot reach is the one whose members
|
||||
are *sites in code* rather than values — #403 (5 of 6 dispatch sites) and #671 (4 of 10 media types).
|
||||
Find-all-references answers exactly that, and across 811 session transcripts the `LSP` tool was
|
||||
invoked **zero** times, against 23,661 `Bash` calls over the same corpus. Both C# surfaces failed to
|
||||
start; nothing reported it, because a language server that cannot initialize is indistinguishable
|
||||
from one nobody asked.
|
||||
|
||||
**Two surfaces, and only one of them has been observed to cross the subagent boundary.** A dispatched
|
||||
subagent's `ToolSearch` returns "No matching deferred tools found" for `select:LSP` while the same
|
||||
query resolves in the main session — measured on Claude Code 2.1.232, agent types `general-purpose`
|
||||
and `Explore`, 2026-08-14, plus the independent §5.1 observation. Three observations on one client
|
||||
version: design around it, but it is measured behaviour, not a guaranteed contract. The MCP side is
|
||||
positively evidenced — 326 MCP calls from inside subagent turns across four servers in the transcript
|
||||
corpus — though not yet for `csharp-lsp` specifically, which only became startable on 2026-08-14. So
|
||||
the standing note telling *workflow agents* to use C# code intelligence was unsatisfiable as written
|
||||
against the `LSP` tool, and is satisfiable once pointed at the MCP tools. Naming the surface is the
|
||||
load-bearing part: an agent that cannot resolve the tool substitutes Grep and does not say so.
|
||||
|
||||
**Both failures were a config naming a path the machine does not have.** `MSBuildLocator` needs a
|
||||
dotnet root owning `host/fxr`, which a Homebrew `bin/dotnet` does not have (`libexec` does); the MCP
|
||||
entry named a dotnet install that no longer existed, while `~/.codex/config.toml`'s copy of the same
|
||||
server had been migrated. Neither returned a wrong answer — each refused to start, which is the
|
||||
benign half of environment divergence and the reason it survived so long.
|
||||
|
||||
**`.mcp.json` and `.mcp/` are gitignored, so the working configuration is not recoverable from the
|
||||
repo.** That is what let one client's copy drift from the other's with nothing to compare against.
|
||||
`docs/local-lsp-tooling.md` carries the entry verbatim so a second machine can reconstruct it, and
|
||||
`scripts/check-local-lsp.sh` turns every precondition into a check. The script is deliberately wired
|
||||
to **no** CI job: every dependency it tests is a developer-machine install, so a red on a runner
|
||||
would carry no information.
|
||||
|
||||
**Rejected: making find-all-references a mandatory step before multi-site fixes.** Not for
|
||||
availability reasons — C# is *expected* to reach delegated agents through the MCP server, so a
|
||||
mandate would likely be enforceable there (an inference from the MCP boundary generally, not a
|
||||
measurement of `csharp-lsp` from a subagent); only the TypeScript route is main-session-bound. The reason is readiness, and it bites each server
|
||||
separately: `csharp-ls` answers `[]` while `ErsatzTV.sln` loads (minutes), and
|
||||
`typescript-language-server` answers with the declaration alone while its own project graph loads —
|
||||
20 references reported as 1, with nothing marking it incomplete. Mandating the step without a way to prove the server had settled would license treating
|
||||
that answer as a population — the precise failure the step exists to prevent, now carrying the
|
||||
authority of a rule. It is documented as available and recommended for the multi-site case, with the
|
||||
re-issue-and-confirm discipline attached, and the zero-invocation baseline above is what a future
|
||||
measurement should be compared against.
|
||||
@@ -0,0 +1,87 @@
|
||||
---
|
||||
key: testing.deny-path-at-production-config-value
|
||||
title: '2026-08-21 — A config-gated guard is tested on its DENY branch at the value production actually runs (#779)'
|
||||
status: active
|
||||
since: '2026-08-21'
|
||||
supersedes: none
|
||||
superseded-by: none
|
||||
rule: 'Where behaviour is gated by a configuration value, an environment variable or a credential, the test matrix covers every value the surface will actually meet — the setting ABSENT, the setting at its PRODUCTION value, and each explicit opt-out — and it asserts the DENY branch, not only the allow branch. A fixture that OMITS the field tests the default and nothing else, so a fail-open reachable only through the configured value stays invisible however many tests are green (#756: thirty of them were). Two corollaries carry most of the weight. FIRST, a hand-written test double that is HANDED the resolved flag proves the CONSUMER reacts to it and says nothing about the line that DERIVES it; if no test constructs the real provider, a mistyped configuration key or a flipped default is unobservable to the whole suite. SECOND, the dangerous cell is whichever one production occupies, which is not always the explicit one: when the shipped default IS the permissive branch the absent case is the production case (#280''s null `Api:WriteKey`), and when the default is fail-closed the configured value is the one nothing has exercised. Enumerate the cells before deciding which to test; do not infer the risky one from which is easier to write. This rule is NOT mechanically enforced and deliberately so — deciding whether a given test used the production value is a string predicate over test source, the class this repo has withdrawn twice.'
|
||||
signals: 'deny path at the production value · fixture omits the field tests only the default · fail-open by default · parametrise the whole config matrix · absent versus configured versus opt-out · a fake that is handed the flag never runs the line that derives it · nobody constructs the real provider · `Api:WriteKey` · `Api:RequireKeyForReads` · `ERSATZTV_ALLOW_WRITES` · `memory_pressure` absent · `xxd` absent on the runner · paths: `ErsatzTV/Services/ApiKeyProvider.cs`, `ErsatzTV.Tests/Services/ApiKeyProviderTests.cs`, `ErsatzTV.Tests/Filters/ApiAuthorizationFilterTests.cs`, `docs/guard-inventory.md` · issues: #779, #773, #756, #768, #751, #647, #282, #280'
|
||||
mechanics: 'Detector F of `docs/defect-shapes-773.md` §4. No CI check implements it; the population of config-gated guard FILES is already derived and machine-checked by `scripts/tests/test_guard_inventory.py`, and this rule is the judgement layered on top of that population.'
|
||||
---
|
||||
|
||||
**The rule exists because the green suite is the symptom, not the reassurance.** #756's fixture
|
||||
omitted a field. The unset behaviour was correct, thirty tests said so, and a fail-open reachable
|
||||
only through the *production* value sat behind them untouched. Nobody skipped a test; they tested
|
||||
the cell that was easy to construct and read the result as coverage of the setting.
|
||||
|
||||
**The three cells, because naming them is most of the fix.** A gated surface has a value that is
|
||||
absent, a value production sets, and one or more explicit opt-outs. Which cell is dangerous is a
|
||||
property of the surface, not a constant, and this is where the reasoning usually goes wrong:
|
||||
|
||||
- **Default permissive.** #280 — API writes fail *open* when `Api:WriteKey` is null or empty, and
|
||||
null/empty was the shipped default. Here the absent cell *is* production, and a test that
|
||||
configures a key to exercise the guard has stepped off the dangerous cell to do it.
|
||||
- **Default fail-closed.** `Api:RequireKeyForReads` defaults to `true`. Now the absent cell is safe
|
||||
and the configured cells are the unexercised ones.
|
||||
|
||||
**The variant this repo was actually carrying, found by #779's audit and worth more than the
|
||||
principle.** Every assertion about the read-gating posture ran through a hand-written
|
||||
`FakeApiKeyProvider` that is *handed* the boolean — `ApiAuthorizationFilterTests`,
|
||||
`ApiKeyEndpointRequiresKeyTests`. Those tests are good ones: they cover the deny branch, at
|
||||
`requireKeyForReads: true`, which is the production value. And they could not see a defect in
|
||||
`ApiKeyProvider` at all, because **no test in the repository constructed it.** The single line
|
||||
deriving the posture from configuration — `configuration.GetValue(RequireKeyForReadsConfigurationKey,
|
||||
true)` — had never been executed by a test. Mistype the key, flip the default, and every one of
|
||||
those green deny-path assertions stays green while the shipped build serves anonymous reads.
|
||||
|
||||
That is the shape to internalise: *the fake was accurate, the fixture was at the production value,
|
||||
and the coverage was still absent* — because the double stood exactly where the untested code was.
|
||||
Ask not only "which value did the test use" but "which code did the value flow through".
|
||||
|
||||
**Fixed here**, by constructing the real provider across the matrix in `ApiKeyProviderTests`:
|
||||
absent, `true`/`True`/`TRUE`, `false`/`False`, and — the cell the first draft of this record forgot
|
||||
while claiming "the whole matrix" — a present-but-non-boolean value. `ConfigurationBinder` returns
|
||||
the default ONLY for a null section value, so `""`, `1` or `yes` reach `BooleanConverter` and throw
|
||||
`InvalidOperationException` at construction (measured). That is fail-CLOSED: an operator who writes
|
||||
`Api__RequireKeyForReads=` with nothing after it in a compose file gets a refusing app, not silent
|
||||
anonymous reads. It is pinned so a later switch to a lenient `TryParse` cannot turn a typo into a
|
||||
posture change. `Api:WriteKey` is set in those cases only so
|
||||
`ResolveKey` returns before touching the live config volume; that is a test-isolation detail and
|
||||
deliberately not their subject.
|
||||
|
||||
**What the mutation proof here does and does not show, because the honest version is weaker than
|
||||
the headline.** Flipping the shipped default `true` -> `false` reddens
|
||||
`Read_Gating_Is_Required_When_The_Setting_Is_Absent` and only that test. That establishes the
|
||||
*absent* cell is load-bearing. It also shows the three explicitly-configured `true`/`True`/`TRUE`
|
||||
cases — the cells this rule is named for — are NOT independently load-bearing against that mutation:
|
||||
a mistyped configuration key is caught, but by the `false` cases, not the `true` ones. A mutation class that would
|
||||
redden `"True"`/`"TRUE"` alone is a hand-rolled case-sensitive parse replacing `GetValue`. They are kept as behaviour coverage of the real binder rather than deleted, and the
|
||||
distinction is written down instead of being smoothed over: a matrix that is *complete* is not
|
||||
thereby a matrix in which every cell *discriminates*.
|
||||
|
||||
**Why no check enforces this.** Deciding whether a test exercised a production value means reading
|
||||
test source and judging intent — a string-matching predicate over code, which
|
||||
`docs/defect-shapes-773.md` §4 argues against and which this repo has withdrawn twice after
|
||||
round-churn (#629, #774). What *is* mechanical already exists: `test_guard_inventory.py` derives the
|
||||
guard-file population and asserts set equality, so this rule has a maintained list to be applied to.
|
||||
The judgement stays with review, and saying so is the honest position rather than shipping a
|
||||
keyword matcher that would manufacture the confident-but-empty coverage the rule is about.
|
||||
|
||||
**Environment counts as configuration.** The permissive branch is often reached by a tool being
|
||||
absent rather than a setting being wrong, and it looks identical from inside. `pretooluse-bom-guard.sh`
|
||||
detected BOMs with `xxd`, which ships with vim and was **absent on the Linux CI runner**, so the
|
||||
comparison never matched and every BOM was allowed in silence while the hook fired on every commit
|
||||
(`docs/guard-inventory.md`). `pretooluse-agent-ram.sh` has the same construction today —
|
||||
`memory_pressure` is macOS-only and its absence yields `exit 0`, allowing unbounded fan-out — and its
|
||||
deny branch is exercised by no test. It is left to #785, which owns mutation proofs for the unproven
|
||||
guards; splitting one guard's proof across two issues is how a row ends up claiming coverage twice
|
||||
and holding none.
|
||||
|
||||
**Audit residue, so the gaps are tracked rather than implied closed.** Of the config-gated guards
|
||||
enumerated from `git ls-files` over `.claude/hooks/`, `.husky/`, `scripts/` and `.gitea/workflows/`:
|
||||
the C# auth surface is now covered end to end; `docker-build.yml`'s dropped-step fail-open is covered
|
||||
by `test_ci_dropped_step_guard.py`; and the untested permissive branches that remain are all hook
|
||||
`exit 0` escape hatches (`ETV_ALLOW_DIRTY_PUSH`, `ETV_SKIP_REBASE_CHECK`, absent credentials in
|
||||
`prepush-donewhen.sh`, absent `python3` in `decisions-guard.sh`) — each already a row in
|
||||
`docs/guard-inventory.md` with proof `NONE`, and so already inside #785's scope.
|
||||
@@ -0,0 +1,68 @@
|
||||
---
|
||||
key: testing.fix-ships-a-witnessed-red-test
|
||||
title: '2026-08-16 — A fix''s test is witnessed RED before the fix, or it pins nothing (#794)'
|
||||
status: active
|
||||
since: '2026-08-16'
|
||||
supersedes: none
|
||||
superseded-by: none
|
||||
rule: 'A commit claiming to fix something may carry a `Proves: <pytest selector>` trailer; when it does, `scripts/prove-fix.sh` must show that selector GREEN with the fix and RED with the code side reverted, and CI enforces it per-PR. The trailer is opt-in — an unproven commit is allowed — but a claimed proof that does not hold fails the build.'
|
||||
signals: 'witnessed red, test does not discriminate, prove the fix, Proves trailer, revert the code keep the test, mutation proof for fixes, red-green · paths: `scripts/prove-fix.sh`, `scripts/tests/test_prove_fix.py`, `.gitea/workflows/pr-checks.yml` · issues: #794, #776, #793, #796, #775'
|
||||
mechanics: '`scripts/prove-fix.sh [--repo DIR] <commit> [selector]`; the `prove-fix` job in `.gitea/workflows/pr-checks.yml`'
|
||||
---
|
||||
|
||||
**`testing.guard-ships-with-mutation-proof` generalised from guards to fixes.** That record says a
|
||||
guard is not tested because a test involving it passes. The same argument applies to every bug fix,
|
||||
and nothing enforced it. Two six-round issues in one week had the same recurring mechanism, and it
|
||||
was not any individual bug: **the fix's test was written to confirm the fix, not to discriminate
|
||||
against its absence.** #776 shipped a test asserting stdout after a signal that never read
|
||||
`returncode` — the one thing its fix changed. #793 produced five false greens in its own verification
|
||||
code, every one a check that could not go red (#796).
|
||||
|
||||
**The check is a red-green pair, not a red.** `prove-fix.sh` runs the selector twice: once at the
|
||||
commit (control, must be GREEN) and once with the commit's non-test files reverted (must be RED,
|
||||
and RED means pytest exit 1 exactly — see below). The control is what makes the second run mean anything — a test that is already failing
|
||||
proves nothing by failing again, and without the control a broken selector sails through as PROVEN.
|
||||
That hole existed in the first version of the script and its own test found it.
|
||||
|
||||
**Opt-in by trailer, enforced when present.** Requiring `Proves:` on every commit would block docs,
|
||||
CI and refactor commits that have no code side to revert, and a gate that blocks ordinary work gets
|
||||
switched off — which is how a check ends up running nowhere (#631). So the trailer is the author's
|
||||
*claim*, and CI checks claims. The job says so out loud when a PR carries none: its green asserts
|
||||
nothing about that PR, and must not be read as fix coverage.
|
||||
|
||||
**Refusing beats guessing.** With no trailer and no argument the script exits rather than inferring
|
||||
the selector from the touched test files. That heuristic silently does nothing when a fix edits an
|
||||
existing test — the exact case worth catching — and a prover that quietly proves nothing is worse
|
||||
than one that refuses.
|
||||
|
||||
**Only pytest exit 1 counts as red, and that is the whole safety argument.** "Non-zero" is not "the
|
||||
test failed": 2 is an interrupted collection, 3 an internal error, 4 a usage error, 5 nothing
|
||||
collected, and a killed run gives 143. Cold review measured a SIGTERM being read as red in an
|
||||
early DRAFT — cancellation masquerading as evidence, produced by the tool meant to prevent it. The
|
||||
first *committed* version (`587edbecc`) exits 5 on a signal by a different route, so do not expect
|
||||
to reproduce PROVEN there; the reproducible false PROVEN on that sha is the marker case —
|
||||
`( cd X && pytest ); rc=$?` returning 1 because `cd` failed, witnessed by
|
||||
`test_a_harness_failure_is_NOT_reported_as_PROVEN`. `--continue-on-collection-errors` is passed so a
|
||||
genuine collection failure (what happens when the fix ADDED the module the test imports) is reported
|
||||
as a test error and exits 1, rather than being lost in the ambiguous 2/3 band. Measured: pass 0,
|
||||
fail 1, collection-error 2 → 1 with the flag, SIGTERM 143.
|
||||
|
||||
**Each phase gets a fresh worktree and its own TMPDIR.** Sharing one lets state written during the
|
||||
control run decide the second run — a test that creates a marker and fails when it exists would "go
|
||||
red" with the fix still in place, a false proof manufactured entirely by the harness.
|
||||
|
||||
**A merge commit is refused, not silently resolved.** It has several parents, so "the code before
|
||||
this change" is ambiguous; evaluating one of them quietly would be the overclaim this record is
|
||||
about. Put the trailer on the commit carrying the fix.
|
||||
|
||||
**What this does NOT establish, stated because a prover that overclaims is the defect it exists to
|
||||
catch.** It shows a test *can* go red, not that it asserts the *right* property. #776's
|
||||
`test_output_survives_a_SIGTERM` would have passed this check while still missing `returncode`. That
|
||||
judgement stays with review. It is Python-only: C# and `web/` fixes need their own runners and no
|
||||
coverage is claimed for them.
|
||||
|
||||
**The sibling rule is deliberately NOT mechanised.** #776's worst artifacts were two wrong
|
||||
measurements written into a decision record, both from quoting a review summary without re-running
|
||||
it. The rule — *a number in a durable artifact carries the command that produced it, or is not
|
||||
written* — stays a convention. Detecting "this number lacks provenance" is a string predicate over
|
||||
prose, which `docs/defect-shapes-773.md` §4 argues against and this repo has withdrawn twice.
|
||||
@@ -0,0 +1,138 @@
|
||||
---
|
||||
key: testing.full-replace-asserts-field-list
|
||||
title: '2026-08-21 — A full-replace path asserts its COMPLETE field list against the DTO, and reconciles by id where child state exists (#779)'
|
||||
status: active
|
||||
since: '2026-08-21'
|
||||
supersedes: none
|
||||
superseded-by: none
|
||||
rule: 'Any path that writes a WHOLE entity or a WHOLE child collection — a PUT-replace handler, a hand-built request object, a test comparer standing in for one — derives its field list from the authoritative type and asserts SET EQUALITY against it, rather than enumerating the fields by hand. A hand-written list is correct on the day it is written and structurally unable to report the day it stops being: the field that drifts is the one nobody wrote a line for, so no amount of care in the existing lines can reach it. The failure is silent by construction — a full replace with a field omitted returns HTTP 200 and destroys that field''s value (#754 drifted from a 28-property DTO by one and cleared it; the symptom arrived hours later as missing pixels). SECOND CLAUSE, separable from the first: where a replaced child row carries state keyed to its identity — progression, ordering, an enumerator position — the handler RECONCILES BY ID rather than delete-and-reinsert, because reinsertion silently resets state a client never asked to touch (#252: a schedule PUT reset fill-group progression; #500: a dedup fix became permanent data loss because the add filter and the remove filter used different keys, so the two halves must agree on the key). Delete-and-reinsert is acceptable ONLY where no such state exists, and that emptiness is a fact about today''s schema that a later feature can silently invalidate — so record it where the handler is, dated, rather than leaving it to be re-derived. The canonical worked example is `ToolCatalogTests.Every_Write_Tool_Should_Declare_Exactly_Its_OpenApi_Request_Body_Fields`, which reads the accepted fields from the generated OpenAPI document and compares both directions.'
|
||||
signals: 'full replace asserts its field list · hand-maintained mirror drifts by one field · reconcile by id not delete and reinsert · 200 and the field is gone · add filter and remove filter must share a key · fill-group progression reset by a PUT · derive the comparer from the DTO · a lossless round-trip test that is itself a hand-copied list · anti-vacuity PIN not a floor on a reflective walk · stale exemption must still name a real property · a complete comparer over a hand-written fixture · universal negative in a record is a trap · paths: `ErsatzTV.Tests/Application/ProgramSchedules/ScheduleItemResponseRoundTripTests.cs`, `ErsatzTV.Mcp.Tests/ToolCatalogTests.cs`, `ErsatzTV.Application/ProgramSchedules/Commands/ReplaceProgramScheduleItemsHandler.cs` · issues: #779, #773, #757, #754, #500, #252'
|
||||
mechanics: 'Detector G of `docs/defect-shapes-773.md` §4. Enforced per-site by a reflective comparison over the DTO plus a written-down count pin, not by a repo-wide check — see the record body for why a global one is not proposed. The exemption set is empty today; the machinery guarding exemptions remains for the first one that earns its place.'
|
||||
---
|
||||
|
||||
**This is `testing.guard-derives-population-from-source` applied to a write path, and it is worth its
|
||||
own record because the population here is not obviously a population.** A guard asserting
|
||||
completeness over an enum or a tool catalog visibly has members to enumerate. A PUT handler and its
|
||||
test look like ordinary code with a lot of fields in them — and the fields *are* the population.
|
||||
#754 is what that looks like when nobody notices: a hand-maintained wrapper one field short of a
|
||||
28-property DTO, returning 200, clearing the value, and surfacing hours later as missing pixels.
|
||||
|
||||
**The repo already had the right answer in one place.** `ToolCatalogTests`'s write-tool test reads
|
||||
the accepted request-body fields out of the generated OpenAPI document, walks the tool catalog
|
||||
unfiltered, and compares both directions — an accepted field a tool never declares and a declared
|
||||
field the API does not accept are opposite defects and are reported as such. That test is the fix
|
||||
for #754/#757 and it is the shape to copy, not the prose above it.
|
||||
|
||||
**Where #779's audit found the mechanism still live: in the test whose job was to catch it.**
|
||||
`ScheduleItemResponseRoundTripTests` is the release gate for the flat schedule-item DTO — a
|
||||
GET → map → PUT → GET fixed point asserting the round trip is lossless. Its comparison was
|
||||
`AssertSemanticallyEqual`, a hand-written run of `b.X.ShouldBe(a.X)` lines. On audit it covered every
|
||||
property but `Id` — so it was *complete*, and had no way to say so: add one more field to
|
||||
`ScheduleItemResponseModel` and it is compared by nobody, the round trip drops it, and the test named
|
||||
for losslessness stays green. The gate against the drift was itself the drift, one altitude up.
|
||||
|
||||
It now derives the property set by reflection over the DTO, and the exemption set is **empty** —
|
||||
which is the most useful thing this exercise produced. The first version exempted `Id`, reasoning
|
||||
that the PUT replaces the item set so B's rows are new rows with new ids. That is the opposite of
|
||||
what the endpoint does: the request forwards each `Id`, so the handler takes its id-based reconcile
|
||||
path and updates rows in place. The exemption was discarding an assertion on a rationale that
|
||||
contradicted the code — the failure this record is about, committed inside the fix for it.
|
||||
|
||||
Two further details. Every exempted name is asserted to still *exist* on the record, because a stale
|
||||
exemption exempts nothing while reading in review as a considered decision. And the anti-vacuity
|
||||
check is a **pin** (`ShouldBe(55)`), not a `>=` floor: a floor lets properties vanish silently, which
|
||||
is the one-sided version of the both-directions rule this record invokes.
|
||||
(`testing.guard-derives-population-from-source` endorses a floor on its canonical example; this is
|
||||
strictly stronger, not a contradiction of it.) Comparing against the
|
||||
reflected count minus exemptions would be tautological — both sides come from the same reflection —
|
||||
so the number is written down and must be bumped deliberately. It was obtained by raising the pin and
|
||||
reading the failure, not counted off the source.
|
||||
|
||||
**The `Id` exemption, and the paragraph that replaced it three times.** Two mutations of the
|
||||
round-trip fixture, both executed:
|
||||
|
||||
| Mutation of `ToReplaceCommand` | Result |
|
||||
|---|---|
|
||||
| `null` for **every** id | GREEN — `requestIds.Count == 0` takes the positional fallback, which also reuses each row |
|
||||
| `null` for **index 0 only** | RED, `Id differs` — the id-based branch deletes the unreferenced row and inserts the id-less item as new |
|
||||
|
||||
So comparing `Id` discriminates, and the exemption was discarding a real assertion. It is still not a
|
||||
substitute for `ReplaceProgramScheduleItemsReconcileTests`
|
||||
(`Reorder_ById_Should_Move_State_With_The_Logical_Item_Not_The_Slot`,
|
||||
`Insert_ById_In_Middle_Should_Keep_Existing_Ids_And_State`), which pass real ids and pin that state
|
||||
moves with the logical item rather than the slot.
|
||||
|
||||
**The transferable part is not the `Id` detail.** This paragraph was written four times. Every draft
|
||||
paired a correct observation with a confident causal story, and three of those stories were
|
||||
contradicted by the code — including one that generalised a single measured case into a universal
|
||||
("no id-less mutation can redden it") that a *partial* payload falsifies. The rule that survives:
|
||||
**state the measurement and the code path you actually read; do not generalise from one executed case
|
||||
to a class of cases, and do not explain a mechanism you did not measure.** A decision record is
|
||||
exactly where such a story does the most damage, because it reads as checked.
|
||||
|
||||
|
||||
**A completeness guarantee on the COMPARER is not one on the FIXTURE.** The seed builder feeding this
|
||||
round trip is still a hand-written list. A new nullable field is now reflected and compared — as
|
||||
`null` against `null` — until someone also seeds it, so the round trip can still drop it while this
|
||||
test stays green. Detector G is satisfied for the comparison, not for the fixture; that residue is
|
||||
real and is named here rather than left implied.
|
||||
|
||||
**On the second clause, the audit's finding was a negative one — and the first version of it was too
|
||||
strong, which is exactly why negatives are dangerous to record.** Four handlers — playlist items,
|
||||
block items, deco-template items, template items — `RemoveRange` their children and rebuild, the
|
||||
shape #252 was fixed for. The claim written here first was that no persisted state is keyed to those
|
||||
item ids. That is false, and the counterexamples live in the same subsystem the record cites
|
||||
(verified 2026-08-21):
|
||||
|
||||
- `PlayoutItem.SchedulingContext` is a persisted column holding a serialized
|
||||
`BlockSchedulingContext(BlockId, **BlockItemId**, Enumerator, Seed, Index)`.
|
||||
`ReplaceBlockItemsHandler` deletes and reinserts, so the stored `BlockItemId` no longer refers to
|
||||
the row it was written for. `ProcessSchedulingContextHandler` looks it up: it either resolves
|
||||
nothing and falls back to `new BlockContextBlockItem(id, null, null)`, or — if the id was reused by
|
||||
a different item — resolves to the WRONG one. Which of the two happens was not measured, so the
|
||||
claim here is only that the reference is orphaned; the second case is the worse one and is the
|
||||
reason not to write this off.
|
||||
- `PlayoutItem.GuideGroup` is set to `effectiveBlock.TemplateItemId` and persisted, while
|
||||
`ReplaceTemplateItemsHandler` deletes and reinserts. `GuideGroup` is the XMLTV grouping key, so a
|
||||
reused id colliding with one already persisted on older items is a HAZARD rather than a
|
||||
demonstrated consequence — unmeasured. Note the block path groups by
|
||||
`{GuideStart, GuideFinish, GuideGroup}` together (`ChannelGuideProjector.ProjectBlock`), a plain
|
||||
`GroupBy` and NOT contiguity-sensitive, so two non-adjacent runs sharing the triple would merge;
|
||||
the contiguity logic is the flood projector, which never sees a `TemplateItemId`-derived
|
||||
`GuideGroup`.
|
||||
|
||||
The accurate statement is narrower: **no progression or enumerator state is keyed to those ids**, so
|
||||
the replace does not reset scheduling position — but two persisted *diagnostic/grouping* fields do
|
||||
embed them and are orphaned by it. Degradation, not data loss. The evidence originally offered
|
||||
(`BlockKey` keys off parent id plus `DateUpdated` ticks, never `BlockItem.Id`) is true and supports
|
||||
only that narrower claim. The other full-replace handlers reconcile by id rather than delete-and-reinsert:
|
||||
`ReplaceProgramScheduleItemsHandler` (#252's scoped positional/no-op reconcile),
|
||||
`ReplacePlayoutAlternateScheduleItemsHandler` (over `ProgramScheduleAlternate`) and
|
||||
`ReplacePlayoutTemplateItemsHandler` (over `PlayoutTemplate`) — neither of the last two touches the
|
||||
`TemplateItem` rows listed among the four above, despite the names.
|
||||
|
||||
Only the FIRST of those three is demonstrably protecting identity-keyed state: `PlayoutScheduleItemFillGroupIndex`
|
||||
holds an FK to `ProgramScheduleItemId`, whereas `PlayoutTemplateId` and `ProgramScheduleAlternateId`
|
||||
appear nowhere in the solution, model snapshot included (verified 2026-08-21). The other two reconcile
|
||||
by id for their own reasons, and attributing that to progression state would be a causal story the
|
||||
schema does not support.
|
||||
|
||||
The general lesson is the one this record nearly failed to learn: a universal negative in a durable
|
||||
document reads as a checked fact and talks the next reader out of checking. Bound it to what was
|
||||
actually enumerated, and date it.
|
||||
|
||||
|
||||
**Why this is not a repo-wide mechanical check.** Finding "every full-replace path" means deciding
|
||||
which PUT/POST handlers replace a whole collection versus patch fields — a judgement about intent,
|
||||
and the population is *sites in code*, which `testing.guard-derives-population-from-source` explicitly
|
||||
scopes out as needing find-all-references tooling this repo does not have (#777). A global lint would
|
||||
be a name matcher over handler classes ending `Replace…Handler`, which misses the ones that do not
|
||||
and flags the ones that do it correctly. Per-site derivation, applied when a full-replace path is
|
||||
touched, is the enforceable version.
|
||||
|
||||
**Residue, tracked rather than implied closed.** The SPA builds request objects field-by-field in six
|
||||
screens (`normalizeForSave` and its equivalents). A *required* field omitted there fails `tsc`; an
|
||||
*optional* field added to the DTO and not carried through compiles clean and drops silently — #754's
|
||||
mechanism narrowed to optional fields, with no equivalent of the MCP write-tool test on the SPA side.
|
||||
That is the highest-probability recurrence in the repo today and is filed separately rather than
|
||||
folded in here.
|
||||
@@ -0,0 +1,182 @@
|
||||
---
|
||||
key: testing.guard-derives-population-from-source
|
||||
title: '2026-08-13 — a guard derives its population from the authoritative source and asserts set equality, never a filter and never a sample (#774)'
|
||||
status: active
|
||||
since: '2026-08-13'
|
||||
supersedes: none
|
||||
superseded-by: none
|
||||
rule: 'A guard that asserts a COMPLETENESS property enumerates its population from a machine-readable authoritative source — the enum, the generated OpenAPI document, the parsed workflow YAML, the provider list — and asserts SET EQUALITY in BOTH directions against it. It may not narrow that population with a filter, a `Where`, a `grep` or an early `continue` before the assertion, because a filter cannot see the member that is MISSING: the member whose absence is the defect is precisely the one the predicate excludes. A hand-written literal list of members is the same defect in slower motion — a filter frozen at authoring time, correct on the day it was written and unable to report the day it stopped being. Two boundaries bound the rule rather than weaken it. FIRST, filtering to select the SUBJECT of a PER-MEMBER property is legitimate and is not this defect: the excluded members satisfy the property vacuously, so the filtered walk and the whole walk assert the same thing (`ToolCatalogTests.Every_Query_Parameter_Should_Be_A_Declared_Property` filters to tools that declare query parameters, and a tool declaring none has nothing to check). The defect is filtering the population before a COMPLETENESS claim, which is what makes an absent member unrepresentable (#757 filtered on `QueryParameters is {Count: > 0}` and so could not see a tool that should have declared one and did not). SECOND, a population of VALUES always has an external authoritative source and this rule applies directly; a population of SITES IN CODE has no such list, needs find-all-references tooling, and is tracked separately in #777 — do not stretch a set-equality assertion over it. Distinguish the guard SCOPE (which subsystems it covers — a reviewed policy choice, legitimately hand-written) from the guard POPULATION (the members inside that scope — always derived). When the scope itself MIRRORS an authoritative source, the mirror needs its own equality check or a dated staleness marker, or the guard is complete within a scope that has silently gone stale. The canonical worked example in this repo is `ToolCatalogTests.Every_Tool_Should_Declare_Exactly_Its_OpenApi_Query_Parameters`; the canonical residual gap is `MARKED_JOBS` in `scripts/tests/test_ci_dropped_step_guard.py`. WHEN THE POPULATION IS FILES (#806), the authoritative source is the GIT INDEX and never a filesystem walk. A walk is not merely a weaker enumerator, it answers a question about the MACHINE rather than about the repo: it reports build output, generated shims and editor droppings, and it differs between CI and every checkout, so the same guard asserts a different population in each place. Derive with `git ls-files`, take direct children only unless a nested population is stated and wanted, and assert existence rather than filtering on it, because filtering is what makes a missing member unrepresentable. This is an instantiation and not a blanket rewrite: the question per guard remains whether it makes a COMPLETENESS claim over TRACKED files, and a walk that assembles a fixture or selects the SUBJECT of a per-member property stays a walk with its reason written down.'
|
||||
signals: 'guard derives population · set equality both directions · never filter never sample · a filter cannot see the missing member · hardcoded list is a frozen filter · missing vs phantom · unreachable vs phantom query parameter · anti-vacuity count guard · accumulate drift do not fail fast · scope versus population · dated mirror of branch protection · `status_check_contexts` · `MARKED_JOBS` · values versus sites-in-code · paths: `ErsatzTV.Mcp.Tests/ToolCatalogTests.cs`, `scripts/tests/test_ci_dropped_step_guard.py`, `web/src/api/pageSizeCallSites.guard.test.ts` · file population from the git index · never a filesystem walk · untracked shims redden every checkout · `.husky/_/` · direct children not rglob · paths: `scripts/tests/tracked_files.py`, `scripts/tests/test_guard_populations_derive_from_git.py`, `docs/guard-inventory.md` · issues: #806, #778, #774, #773, #757, #671, #650, #644, #633, #616, #503, #403'
|
||||
mechanics: 'Both directions are named separately in the failure message — `missing`/`unreachable` (in the source, absent from the guarded set) and `phantom` (in the guarded set, absent from the source) — because the two are different defects and a single "sets differ" line invites fixing one and re-running.'
|
||||
---
|
||||
|
||||
Across the 95 closed issues carrying a `## Closing record`, the largest family is one error wearing
|
||||
four costumes: **reasoning about a representative instead of the population** (#773 §3 Family A — 27
|
||||
of the 69 process-failure records, 39%). This repo has already invented the cure four times, under
|
||||
four names, and never written it down once. That is Family A operating on our own process.
|
||||
|
||||
**The three targets it hits, because a rule aimed at only one of them misses.** The *fix* can sample
|
||||
(#671 covered 4 of 10 media types; #403 was silent at 5 of 6 dispatch sites). The *verification* can
|
||||
sample (#616 and #757 both filtered on the property whose absence WAS the defect; #633's
|
||||
`ShouldContain("capped at 100")` passed against the text `"capped at 1000"`). And the *guard* can
|
||||
sample (#644 grepped for an inflated `pageSize`, so #650's at-cap truncations were structurally
|
||||
invisible to it). Same mistake, three altitudes.
|
||||
|
||||
**Why a filter is not merely weaker but structurally blind.** A predicate over the population
|
||||
partitions it into examined and excluded. A completeness claim asks whether some member is absent
|
||||
altogether. The absent member is in neither partition — it is not a row the filter rejected, it is a
|
||||
row that was never produced — so no strengthening of the predicate can reach it. This is why the
|
||||
fix is never "tighten the filter"; it is to obtain the population from somewhere that knows the
|
||||
whole of it and compare.
|
||||
|
||||
**The canonical shape**, `ToolCatalogTests.Every_Tool_Should_Declare_Exactly_Its_OpenApi_Query_Parameters`,
|
||||
does five things and each earns its place. It reads `accepted` from the generated OpenAPI document
|
||||
rather than from a list in the test. It walks `ToolCatalog.All` with no filter. It computes BOTH
|
||||
`accepted.Except(declared)` and `declared.Except(accepted)`, because an unreachable parameter and a
|
||||
phantom one are opposite defects. It carries an anti-vacuity floor, so a walk that reached nothing
|
||||
cannot report success. And it accumulates the whole drift set instead of throwing on the first
|
||||
mismatch — failing fast hands back one member at a time and invites fixing them one at a time,
|
||||
which is how the #754 twin stayed hidden.
|
||||
|
||||
**The boundary that keeps this rule honest.** `Every_Query_Parameter_Should_Be_A_Declared_Property`
|
||||
sits eight lines away and DOES filter, correctly. Its property is per-member — each declared query
|
||||
parameter must be a declared schema property — and a tool declaring none satisfies it vacuously, so
|
||||
the filtered walk and the full walk assert the same thing. Read the two together before applying
|
||||
this record: the trigger is not the token `.Where(`, it is a completeness claim standing behind one.
|
||||
A rule stated as "no filters in guards" would flag the correct one and, once waved through twice,
|
||||
stop being read at all.
|
||||
|
||||
**Scope is not population, and this repo's best guard shows why the distinction matters.**
|
||||
`scripts/tests/test_ci_dropped_step_guard.py` is the model for deriving a population: `_marked(job)`
|
||||
reads the marked steps out of the parsed workflow, and
|
||||
`test_the_guard_expects_EXACTLY_the_set_of_marked_keys_in_the_right_bucket` asserts set equality
|
||||
against the guard's own argv. Inside its scope it is exemplary. Its SCOPE, though —
|
||||
`MARKED_JOBS = ("test", "migrations")` — is a hand-written mirror of the required contexts on
|
||||
`main`, annotated `Read live on 2026-08-10`. Those contexts are machine-readable
|
||||
(`GET /repos/{owner}/{repo}/branch_protections` → `status_check_contexts`), and nothing in this repo
|
||||
compares the two: `grep -rn 'status_check_contexts\|branch_protection'` over `*.py`/`*.sh`/`*.yml`
|
||||
returns nothing. Verified 2026-08-13 — the live contexts still match the comment, so this is a
|
||||
latent gap, not a live defect. But add a fourth required context and it acquires no dropped-step
|
||||
guard, silently, and the file that models the rule is the one that cannot report it. A dated comment
|
||||
is a claim about the past; it is not a check.
|
||||
|
||||
**A weak detector is the symptom-keyed mistake, demonstrated rather than argued.** #774 wrote this
|
||||
rule and then, in the same change, built a guard that violated it: a test asserting set equality
|
||||
between the H10 verdict vocabulary's two shell copies, by extracting each with a regex. The
|
||||
authoritative source it should have derived from does not exist — that is the entire problem, and
|
||||
detector C's answer is to delete one copy, not to compare two. Six cold-review rounds each produced
|
||||
another shell construction that either escaped the extractor (an unquoted value, a glob arm, an
|
||||
indented reassignment, a comment-first arm) or made it red on a correct tree (a `<word>) state=` in
|
||||
a heredoc, a column-zero `esac` in a string truncating the scope). Every round's fix was locally
|
||||
correct; the sequence never converged, because a regex over shell source is not a shell parser and
|
||||
cannot be made into one.
|
||||
|
||||
It was withdrawn, not patched a seventh time. **The residue is that the duplication is now
|
||||
unmitigated** (#788 removes the second copy, which is the fix that was available all along), and the
|
||||
lesson is the one this record already stated on theory: when the authoritative source is missing,
|
||||
the answer is to create one, never to approximate it with a predicate over text. A guard whose
|
||||
accompanying prose a reviewer can falsify each round is worse than no guard, because — by this
|
||||
record's own argument — a guard described as sound stops being re-examined.
|
||||
|
||||
**When the population is FILES, the authoritative source is the git index (#806).** The worked
|
||||
examples above are an enum and a generated document, both unambiguously authoritative, and the
|
||||
record was silent on the commonest population in this repo's own guards: files in a directory. Every
|
||||
one of them answered with a filesystem walk, and #778 measured what that costs by getting the same
|
||||
population wrong three times in one PR — a content filter that omitted `git fetch`, a non-recursive
|
||||
`glob` that missed four nested files, and finally `rglob`, which enumerated `.husky/_/`: 17 husky
|
||||
shims generated by `npm ci`, gitignored and untracked. That last one made the guard **red on every
|
||||
developer checkout and green in CI**, whose `script-tests` job pip-installs but never runs `npm ci`.
|
||||
|
||||
The reason a walk keeps losing is not that each traversal was written carelessly; two of the three
|
||||
were the obvious correction to the one before. It is that the disk answers a question about the
|
||||
MACHINE and the guard is asking one about the REPO. Those coincide often enough for a walk to look
|
||||
right and diverge exactly where generated output lands, which is to say wherever the tooling is
|
||||
installed and nowhere else. The index is the repo's own statement of what it contains — the same
|
||||
set of files every checkout receives from a clone, and excluding untracked files by construction
|
||||
rather than by an exclusion list somebody maintains. It is not immutable and it is per-worktree; the
|
||||
claim is not that it never changes, but that it changes only through a deliberate git operation — staging, a checkout, a
|
||||
reset, a merge — whereas the disk changes whenever a build runs. Note what that buys over `.gitignore`-awareness: `.husky/_/`
|
||||
happens to carry its own `.gitignore`, but a stray `foo.sh` in `.claude/hooks/` carries nothing, and
|
||||
only the index knows it is not part of the repo.
|
||||
|
||||
The direction of the failure is worth naming, because it inverts the usual worry about a guard.
|
||||
Under-enumeration hides a defect; this over-enumerated, and reddened correct trees. A guard that
|
||||
fails everywhere except where it runs is not a cautious guard, it is a guard nobody reads — and it
|
||||
had done that to the artifact whose entire thesis is population correctness.
|
||||
|
||||
**This did not become "replace every glob", and the boundary is the same one drawn above.** The
|
||||
question per guard is whether it makes a completeness claim over TRACKED files. `_repo_copy` in
|
||||
`scripts/tests/test_ci_release_path_scan_job.py` assembles a fixture and asserts nothing about
|
||||
which files it found; it takes its file list from the index for HERMETICITY, which is a different
|
||||
reason, and its docstring distinguishes the two.
|
||||
`scripts/tests/test_ci_dropped_step_guard.py` has no filesystem population at all — it reads the
|
||||
parsed workflow. Converting either would have been a change with no defect behind it, which spends
|
||||
the credibility this rule needs when it does bite. The per-guard verdicts, including the two
|
||||
no-change ones and the decisions corpus recorded as unexamined rather than cleared, are tabled in
|
||||
`docs/guard-inventory.md`.
|
||||
|
||||
**The residue, named.** `git ls-files` reports INDEX entries, so a guard joins the population when
|
||||
it is STAGED rather than when the file appears. Nothing local runs these checks — `.husky/pre-commit`
|
||||
runs lint-staged, the decisions guard, the root-PNG check and `dotnet format`, and no husky hook runs
|
||||
pytest — so the red arrives from `pr-checks.yml::script-tests` on the PR. A file deleted from the
|
||||
working tree but not yet staged is still listed; `tracked_paths` asserts existence rather than
|
||||
filtering it out, because a filter is what makes a missing member unrepresentable, which is this
|
||||
record's first paragraph applied to its own implementation.
|
||||
|
||||
**Two traps specific to converting an existing guard, both of which this rule caught inside its own
|
||||
implementation.** FIRST, follow the data to where members are actually ADMITTED, not just to where
|
||||
the walk starts: a scrape that reads its caller files from the index and then admits the paths they
|
||||
name on `Path.exists()` is half-derived and reads as fully derived. SECOND, a proof that a
|
||||
derivation excludes untracked files must remove EVERY member in turn, not one. `derived_guard_files`
|
||||
unions four contributors; a single victim is always drawn from whichever sorts first, so a mutant
|
||||
putting only one contributor back on a filesystem walk passes. A sample cannot see the source it did
|
||||
not draw from — this record's opening argument, one level down, inside the artifact written to
|
||||
enforce it. Exhaustive removal is cheap — about a second at the population sizes here.
|
||||
|
||||
**Removal is only half the property, and the second half must not itself be machine-dependent.** A
|
||||
source contributing ONLY untracked members adds and never removes, so a removal-based check has
|
||||
nothing of its to take away: an `rglob` appending `.husky/_/` leaves the removal proof GREEN. Scope
|
||||
that claim on both axes, because it is narrower than it first reads. A broader `rglob` that also
|
||||
displaces tracked members DOES redden removal, so the blind spot is the append-only shape rather
|
||||
than every filesystem walk; and the append-only shape is blind only where the walked directory
|
||||
yields nothing — with the shims present its members are there to remove and removal reddens too. The
|
||||
gap is therefore an append-only source that is empty ON THIS MACHINE, which is exactly the CI
|
||||
checkout, and exactly where a guard going quiet is invisible.
|
||||
|
||||
The obvious complement — arrange an untracked file and require it not to enter — is a trap this
|
||||
change fell into and backed out of twice, and the reason is worth more than the rule. Writing probe
|
||||
files into the checkout under test means a probe in the `test_*.py` scope is a file pytest COLLECTS
|
||||
mid-session, probe names collide across concurrent runs, `finally` does not survive a SIGKILL, and a
|
||||
concurrent `git add -A` can stage one — defects in the test rather than in the thing tested.
|
||||
Neutralising the shared derivation and requiring the population to go empty is clean but misses the
|
||||
`.husky/_/` source on any machine where `.husky/_/` does not exist, which is the `script-tests`
|
||||
checkout, since that job never runs `npm ci`. Both formulations reproduce the green-in-CI /
|
||||
red-on-a-laptop asymmetry this record exists to abolish, inside the proof written to abolish it.
|
||||
|
||||
**So watch for the property that needs no arranged state: a directory LISTING issued while the
|
||||
derivation runs.** Listing is the commonest way a derivation discovers a member the index does not
|
||||
know about, and a walk issued during the derivation is caught on any machine — an `rglob` fails even
|
||||
where the directory it walks is empty, because the evidence is the call rather than what it
|
||||
returned. State its reach honestly, and state it once: what is observed is any
|
||||
call that goes THROUGH ONE OF THE SPIES, whenever it happens — the check's docstring works through
|
||||
the instances, and this record does not copy them, because the copy drifted from the original inside
|
||||
a single commit. It is a regression guard against the shapes that arrive by accident, not a
|
||||
boundary, and a guard sold as a boundary stops being re-examined.
|
||||
|
||||
**Three drafts of that one sentence were wrong, all in the same direction, and the third was wrong
|
||||
in the copies after the original had been fixed** — which is the completeness rule biting the prose
|
||||
that describes it. "Synchronously inside the call" was falsified by a thread finishing during the
|
||||
drain; "while the patch is active" was falsified by a spy reference captured inside the window and
|
||||
invoked after it, which still records. Each draft named the mechanism the author had in mind rather
|
||||
than the one that decides, and each understated the coverage. Understating is the safe direction and
|
||||
still worth correcting: a limit stated too narrowly invites someone to build the case it appears to
|
||||
exclude. The durable lesson is the one this record already gives for populations — do not keep a
|
||||
second copy of a statement that is still being corrected. Its complement is removal, which catches the shape that admits a HARDCODED
|
||||
path without listing anything (`if (REPO_ROOT / "x").exists(): add` — the defect this change shipped
|
||||
in its own first round). Neither alone is the property.
|
||||
|
||||
**What this record does not cover.** A population of *sites in code* — the places that dispatch on a
|
||||
value — has no external enumerator. #403 is that case: `PlaybackOrder`'s values are enumerable, but
|
||||
the defect was 5 of 6 dispatch sites, and nothing lists dispatch sites. That residue needs
|
||||
find-all-references tooling and is tracked in #777, where the relevant language server is currently
|
||||
broken. Do not stretch set equality over it and call the class closed.
|
||||
@@ -0,0 +1,95 @@
|
||||
---
|
||||
key: testing.guard-ships-with-mutation-proof
|
||||
title: '2026-08-13 — a guard ships with a proof it can go red: disarm that clause alone and a named test must fail (#775)'
|
||||
status: active
|
||||
since: '2026-08-13'
|
||||
supersedes: none
|
||||
superseded-by: none
|
||||
rule: 'A guard is not considered tested because a test involving it passes. It ships with a MUTATION PROOF: remove or disarm THAT GUARD''S CLAUSE ALONE, and a NAMED test must go red. ONE NAMED EXCEPTION, with its limits, because the rule degenerates without it: where the guard IS a test (a checker enforcing a repo invariant, with no separate script behind it), disarming it makes it ABSENT rather than red, so the proof is the contrapositive — INTRODUCE THE DEFECT THE GUARD EXISTS TO CATCH into an isolated copy of the guarded artifact, and the named test must go red. That is a mutation of the guarded SYSTEM rather than of the assertion, and it is admissible ONLY for checker-guards and ONLY when the mutation was executed and witnessed. It is NOT a licence to grade an ordinary script-guard MUTATION for having a bad-input test: feeding a script an input its clause rejects is BEHAVIOUR-ONLY, which is what three rows were regraded for. A file-level grade under this exception covers the clause its cited case actually mutates, not every assertion that later lands in the same file. Three things this excludes, each of which has already shipped here as a green suite over a dead check. FIRST, a behavioural test — one that feeds the guard a good input and a bad input and checks it passes and fails — proves the guard REACTS, never that it is LOAD-BEARING; #685 had two guards on one condition where deleting either left the whole suite green while every behavioural test passed. SECOND, mutating the WHOLE FILE does not count (#510): a whole-file revert cannot show that a test reaches a particular clause, so the mutation must target the clause. THIRD, the guard being WIRED is not the guard RUNNING — #631''s suite was invoked by no CI job, #751''s step was dropped by the runner and the job reported success in 6s against a normal 14-17s, and #719''s new logic was never connected to stdin. Every guard that DERIVES A POPULATION also carries an ANTI-VACUITY assertion, because the characteristic failure of a completeness check is reporting that it proved everything while its population was empty; a guard with no population has nothing for such an assertion to be about, and stating it universally reads as coverage the unproven rows do not have. Mechanical enforcement is possible for the BOOKKEEPING and not for the JUDGEMENT, and the split is the decision: `docs/guard-inventory.md` lists every guard file with its Kind, its Proof class (`MUTATION`/`BEHAVIOUR-ONLY`/`NONE`) and a `file::function` ref, and `scripts/tests/test_guard_inventory.py` derives the guard population from the GIT INDEX and the call sites (#806), asserts SET EQUALITY against the rows, and resolves every claimed ref to a real `def`. So a new guard cannot ship unclassified and a renamed test cannot leave a row silently claiming coverage. Whether a row claiming `MUTATION` is telling the truth is no longer left to review: `testing.mutation-claims-are-executed` (#790) requires each such row to carry a DECLARED clause mutation that is applied to an isolated copy of the repository on every run, with the row''s own named test required to go red.'
|
||||
signals: 'mutation proof · delete the guard alone see red · disarm the clause not the file · behaviour-only is not a proof · anti-vacuity assertion · guard wired is not guard running · a green job with no step output · `docs/guard-inventory.md` · set equality against the row set · proof ref resolves to a real def · the unproven majority is a moving figure — read it off the inventory · PROOF kind stops the regress · hook wiring is not hook existence · paths: `scripts/tests/test_guard_inventory.py`, `scripts/tests/test_ci_dropped_step_guard.py`, `scripts/ci-prove-ban-detects.sh` · issues: #775, #773, #751, #756, #719, #685, #631, #621, #510, #445'
|
||||
mechanics: 'Proof classes are a closed vocabulary enforced by the inventory test; a `TOOLING` row may not claim a proof. `scripts/ci-prove-ban-detects.sh` is the one guard that runs its own mutation at CI time rather than in pytest, because the thing it proves — that the ban test is not deselected — is disarmable from inside pytest configuration.'
|
||||
---
|
||||
|
||||
**Vacuous verification is the single most common recorded failure shape in this corpus — 17 of 69
|
||||
records (25%), of which about half are not sampling errors at all: the check was simply dead**
|
||||
(#773 §3 Family B). The rule already exists in prose, in #685's record and in project memory. It is
|
||||
enforced nowhere, which is why it keeps being relearned. This is a hooks problem wearing a docs
|
||||
problem's clothes.
|
||||
|
||||
**The six ways a check has already been dead here**, because the list is more convincing than the
|
||||
principle. `scripts/tests/` was invoked by no CI job (#631) — *a test suite that runs nowhere is not
|
||||
a safety net, it is a document*. Deleting the single line calling `record_wing_faults()` from
|
||||
`main()` left the suite green while a real record vanished (#621). A `${{ }}` inside a shell
|
||||
**comment** made the runner drop the step, and the job reported success in 6s where the working
|
||||
version takes 14–17 (#751). New pre-push logic was never wired to receive stdin (#719). `if ! cmd;
|
||||
then status=$?` reads bash's logical negation, so a failing spec run exited 0 (#445). And two guards
|
||||
on one condition masked each other, so deleting either left the suite green and the boundary test
|
||||
pinned nothing (#685).
|
||||
|
||||
Note what those have in common: in every case a human had read the guard and believed it worked. The
|
||||
guard was not subtly wrong, it was *absent*, and absence is invisible from the outside because a
|
||||
dead check and a satisfied check produce the identical green.
|
||||
|
||||
**Why behavioural coverage is not the same claim.** A test that drives the guard with a good input
|
||||
and a bad input establishes that the guard's logic responds to its argument. It says nothing about
|
||||
whether that logic is *connected* — to the runner, to the caller, to the exit code anyone reads.
|
||||
#719 and #631 would both have passed such a test on the day they shipped dead. This is why
|
||||
`docs/guard-inventory.md` grades `BEHAVIOUR-ONLY` separately from `MUTATION` rather than counting
|
||||
them together: they answer different questions, and adding them up is how the unproven majority gets
|
||||
reported as covered. The current split is in the inventory's own summary line, which is derived-checked.
|
||||
|
||||
**The model, and what makes it the model.**
|
||||
`test_ci_dropped_step_guard.py::test_dropping_ANY_single_step_FAILS_the_guard` removes each marked
|
||||
step's marker *one at a time* and asserts the guard rejects the run — every step, explicitly not a
|
||||
sample, on the argument that the interesting drop is whichever step someone actually edits. Around
|
||||
it sit a positive control (the guard passes when every step marked itself), a negative control (a
|
||||
fired skip gate still requires the always-steps, so the gate is not a blanket off-switch), and an
|
||||
anti-vacuity check that `assert` with an empty expectation set REFUSES instead of printing *All 0
|
||||
expected step(s) executed* and exiting 0. Four kinds, none substituting for another.
|
||||
|
||||
**What was mechanised here, and the line it stops at.** The bookkeeping half is fully mechanical and
|
||||
now enforced: the inventory's population is derived, compared for set equality in both directions,
|
||||
and every `Proof ref` is resolved to a real `def`. A new guard cannot be added without being
|
||||
classified; a renamed test cannot leave a row claiming a proof that evaporated. That closes the two
|
||||
ways this decays silently. The judgement half — *is this row's `MUTATION` claim true?* — was left with
|
||||
review here, and is mechanised by `testing.mutation-claims-are-executed` (#790). A generic mutation
|
||||
runner for shell hooks was considered and rejected at this point: it would have to know which clause
|
||||
of a 90-line hook is the guard, and a runner that guesses would manufacture exactly the
|
||||
confident-but-empty coverage this record exists to prevent. That objection stands and is what the
|
||||
later harness is built around — the clause is DECLARED per row rather than inferred, which is the
|
||||
only form of runner this record's argument permits.
|
||||
|
||||
**What "disarm the clause" means when the guard IS a test, because the rule degenerates otherwise.**
|
||||
For a guard implemented as a shell script with a separate test, disarming is literal: delete the
|
||||
clause, watch the named test redden. For a guard implemented AS a test — `test_guard_inventory.py`,
|
||||
`test_ci_image_pin_population.py` — deleting the guard does not make anything red, it makes the test
|
||||
*absent*. The equivalent proof is the contrapositive: **introduce the defect the guard exists to
|
||||
catch, and the named test must go red.** This is the named exception in the `rule` above, not a
|
||||
second standard sitting beside it — an earlier draft stated it only here, and cold review correctly
|
||||
objected that a record carrying two incompatible criteria cannot adjudicate a future grade: one
|
||||
reviewer would reject the self-referencing rows on the frontmatter, another accept them on the body.
|
||||
It is a real mutation of the guarded system, not of the test, and it is what was executed here — a deleted inventory row, a new unclassified hook, a stale
|
||||
proof ref, a `SHIP*` case arm the extractor could not parse, and each of four ways a container job
|
||||
can lose its pin, applied to every registered job rather than a sample. Cold review pushed back on
|
||||
calling these mutation proofs; the pushback is right that they mutate INPUT rather than the
|
||||
assertion, and wrong that this makes them weaker — for a checker, the input IS the system under
|
||||
guard. Recorded because the distinction will come up again.
|
||||
|
||||
Where that review was straightforwardly right: neutering `pin_population_faults` wholesale (20 of 25
|
||||
red) is coarser than disarming one clause at a time, and coarse enough that a single surviving
|
||||
clause would not be noticed. #790 replaced it with a single-clause mutation — disarming the
|
||||
against-the-registry comparison alone reddens the proof, because a job that loses its `container:`
|
||||
block leaves the two derived sets equal and only that comparison notices. Which guards admit
|
||||
clause-level proof and which do not is now measured per row rather than assumed.
|
||||
|
||||
**Applied to itself, which is the only honest test of a rule like this — and it failed twice before
|
||||
it passed.** The population guard's first draft compared two derived sets that shrank together, so
|
||||
it was blind to a removed `container:` block: the exact defect `testing.guard-derives-population-
|
||||
from-source` exists to prevent, in the guard enforcing it. Running the mutations caught that. Then
|
||||
the inventory's own completeness check shipped with a path pattern that excluded `scripts/tests/`,
|
||||
so the three guard files added in that very commit had no rows and the check stayed green — caught
|
||||
not by running anything, but by a cross-family reviewer. And the summary counts were a hand-kept
|
||||
mirror of the table that was wrong on arrival, which both reviewers found independently. Three
|
||||
self-inflicted instances of two rules, in the change introducing them. That is the argument for
|
||||
mechanising the bookkeeping rather than trusting care, made by the authors of the rules failing to
|
||||
follow them while writing them down.
|
||||
@@ -0,0 +1,223 @@
|
||||
---
|
||||
key: testing.hook-reports-its-own-execution
|
||||
title: '2026-08-14 — every hook records that it fired and what it decided, through one shared sink (#776)'
|
||||
status: active
|
||||
since: '2026-08-14'
|
||||
supersedes: none
|
||||
superseded-by: none
|
||||
rule: 'Every script in `.claude/hooks/` sources `scripts/hook-fire-log.sh` and calls `etv_hook_fire_begin <its-own-name> <label> <capture|stream>` as its FIRST act, before anything reads stdin. Two records are appended per invocation — a `fire` record on entry and an `exit` record carrying the exit status and the decision — to a session-scoped JSONL log. THE DECISION IS READ FROM WHAT THE HOOK ACTUALLY EMITTED, never declared by the hook author: Claude Code hooks (`capture` mode) always exit 0 and communicate by PRINTING JSON, so their stdout is diverted and replayed, and the recorded decision is parsed from those bytes; git hooks (`stream` mode) decide by EXIT CODE and their stdout is live progress text a human is watching, so it is not diverted and the decision is the status. That split is not a tuning knob — capturing a slow pre-push hook''s output would hold it back until the end and read as a hang, and inferring a git hook''s decision from absent JSON would put the report back into the guessing business this record exists to end. The population is DERIVED from `.claude/hooks/*.sh` by `scripts/tests/test_hook_fire_log.py`, so a new hook is uninstrumented-and-red rather than silently unobserved, and the report lists every hook that EXISTS rather than every hook that appears in the log — a report built from the log alone can only show hooks that fired, which makes the never-fired hook, the one finding worth having, invisible. THE INSTRUMENTATION MUST BE INVISIBLE TO THE HARNESS, and this is the load-bearing half: it sits in the stdin and stdout path of the most authoritative guards in the repo, so a differential test drives EVERY hook with and without it over a payload matrix and demands byte-equal stdout and equal exit status. It fails OPEN in exactly one direction — if the log cannot be written the hook behaves exactly as before — because observability that breaks a guard is worse than the blindness it replaces. Two mechanical traps are pinned by tests rather than left to care: stdout must be replayed from the FILE, since `out=$(cat f)` strips trailing newlines and delivers a guard''s JSON one byte short with no parser anywhere to complain; and stdin must never be slurped when it is a TTY, because an interactive `git commit` hands its hooks a terminal and `cat` would block forever, hanging the commit the instrumentation was added to observe.'
|
||||
signals: 'hook fire log · did this hook actually fire · PreToolUse leaves no transcript record · make the system report it rather than infer it · capture vs stream mode · byte-equal stdout differential · fail-open observability · never slurp a tty · derived hook population · never-fired row · paths: `scripts/hook-fire-log.sh`, `scripts/tests/test_hook_fire_log.py`, `docs/guard-inventory.md`, `docs/defect-shapes-773.md` · issues: #776, #773, #775, #774, #756, #751, #719, #685, #631'
|
||||
mechanics: 'Read side: `scripts/hook-fire-log.sh report [--all|--json|--dir D|--session S]`. Logs default to `$HOME/.cache/ersatztv/hook-fire/<session-id>.jsonl`, outside the repo, overridable with `ETV_HOOK_FIRE_LOG_DIR`; `ETV_HOOK_FIRE_DISABLE=1` turns recording off without editing a hook. The session id comes from the payload''s `.session_id`, because this harness version does not export one to hooks. Records carry no tool arguments — only hook, label, event, tool name, phase, status and decision — so the log is not a second copy of the transcript.'
|
||||
---
|
||||
|
||||
**The blind spot this closes.** In this harness version only `Stop` hooks emit a structured
|
||||
transcript record. `PreToolUse` and `PostToolUse` hooks — which is *every* guard this repo actually
|
||||
relies on: merge consent, worktree ownership, BOM, agent model and RAM, the golden-file ban — leave
|
||||
no durable execution trace at all. When #773 tried to measure whether they fire, the only available
|
||||
signal was **filename mentions in settings dumps and prior audits**, which is not evidence of
|
||||
execution. So the guards carrying the most authority were the ones whose execution could only be
|
||||
inferred, and a silently broken hook would have looked identical to a working one indefinitely.
|
||||
|
||||
That is Family B (the check never ran) applied to the hook layer itself, and this project has
|
||||
already paid for it twice one layer down: #751's guard that reported success in 6 seconds having run
|
||||
nothing, and #756's fail-open twin that 30 green tests could not see. #756's standing lesson is the
|
||||
whole of this record — **make the system report it rather than infer it**.
|
||||
|
||||
**Why the decision is parsed from the emitted bytes rather than declared.** The obvious design has
|
||||
each decision site announce itself. `pretooluse-merge-consent.sh` alone reaches its output through a
|
||||
`decide` helper from roughly forty call sites, so that design is forty edits in the most
|
||||
load-bearing guard in the repo, and it records **what the author meant** — which is the inference
|
||||
being abolished, reintroduced one layer up. Capturing what the hook printed records what the
|
||||
*harness* sees. It cannot drift from the decision because it **is** the decision, and it needed no
|
||||
edit to any decision site.
|
||||
|
||||
**The risk this takes on, stated plainly.** Reading stdin before the hook and diverting its stdout
|
||||
puts this code directly in the path of every guard. Get it wrong and a `deny` is swallowed while the
|
||||
log cheerfully reports the guard firing — strictly worse than the blindness it replaces, and
|
||||
undetectable from the outside for the same reason the original gap was. So the proof obligation is
|
||||
not a unit test of the sink: a wrapper can be perfect alone and lethal in context.
|
||||
`test_instrumentation_changes_NOTHING_the_harness_can_see` runs every hook in both forms and
|
||||
compares stdout bytes, stderr bytes and exit status.
|
||||
|
||||
**A differential test is only as good as the cases it drives, and this one had to be measured rather
|
||||
than asserted.** A hook that emits nothing and exits 0 makes the comparison `("", 0) == ("", 0)`,
|
||||
which passes on a wrapper that discards everything. Measured on the first version of the matrix: 160
|
||||
of 165 comparisons were empty-against-empty and 165 of 165 compared exit 0 against exit 0, so
|
||||
deleting the entire replay left the test green for four hooks, two of which issue `deny`. The three
|
||||
places that claimed the matrix "reaches each hook's deciding branch" — including this record — were
|
||||
simply wrong. The structure that replaces the claim:
|
||||
|
||||
- a broad, cheap payload matrix, which reaches the deciding branch of a few hooks;
|
||||
- `positives()`, which CONSTRUCTS the rest — a sibling worktree carrying a foreign owner marker, an
|
||||
`*ersatztv*`-named repo with a staged BOM `.cs`, `memory_pressure` stubbed at 5% and 15%, a real
|
||||
bare remote so the three pre-push guards can be driven to a genuine block, and a stub Gitea so
|
||||
`prepush-donewhen` is exercised rather than exempted;
|
||||
- `test_the_AB_is_not_VACUOUS_for_any_hook`, which fails unless each hook has at least one case
|
||||
where the control side emits output or exits non-zero.
|
||||
|
||||
One hook is exempt and the exemption is stated rather than assumed: `posttooluse-worktree-marker.sh`
|
||||
emits nothing and exits 0 by design — its entire output is a marker file — so its transparency is
|
||||
asserted on that file instead.
|
||||
|
||||
**Five defects this shipped with, all found by cold review or by the tests it prompted.** Recorded
|
||||
because each was invisible to every other check, and because the ranking is instructive: the worst
|
||||
one was surfaced by a finding ranked *Low*.
|
||||
|
||||
| Defect | Why nothing else would catch it | Pinned by |
|
||||
|---|---|---|
|
||||
| `exec 0<"$sin" 2>/dev/null` sent the hook's **entire stderr** to /dev/null for its whole life — `exec` with no command redirects the shell permanently. Every husky guard's user-facing output is stderr, so guards blocked while explaining nothing | the A/B compared only stdout and exit status | `test_stderr_is_NOT_silenced` |
|
||||
| The fd restore lived inside the same conditional as the replay, and the replay read the PATH — so an unlinked-but-open scratch file meant a swallowed `deny` and an exit 0 | needs `mktemp` stubbed or the file removed mid-run | `test_output_SURVIVES_a_vanished_stdout_tempfile` |
|
||||
| `sed`/`tr` in the classifier are locale-aware, so one 0xE9 byte in a `permissionDecisionReason` printed `illegal byte sequence` AT THE HARNESS and filed a real `deny` as `output` | no test put a non-UTF-8 byte in a decision | `LC_ALL=C`, plus the stderr comparison in the A/B |
|
||||
| Payload fields were read from a `head -c 65536` prefix, so a large `tool_response` pushed `session_id` past the cap and those fires filed under `unknown-session` — reading as NEVER FIRED in the report | a truncating read is a sampling error, and no fixture was over 64 KB | fields now read from the file, unbounded |
|
||||
| `out=$(cat f)` strips trailing newlines, so a hook's JSON reached the harness one byte short | every JSON parser accepts it; only a byte comparison objects | `test_stdout_is_replayed_BYTE_EXACT` |
|
||||
| An unconditional stdin slurp blocks forever on a TTY, hanging every interactive `git commit` | the pipe-fed tests cannot reach a tty; needs a real pty | `test_a_TTY_stdin_is_not_slurped` |
|
||||
|
||||
**One mechanism was WITHDRAWN rather than fixed, and it is recorded here because the next reader
|
||||
will otherwise re-add it.** A `trap … INT TERM HUP` was added so a hook killed by the harness
|
||||
timeout would not lose output it had already printed. It produced a defect in each of three
|
||||
consecutive rounds — a handler ending in `exit "$?"` that turned a signal into consent and let a
|
||||
push reach `main`; a `local sig=…` that clobbered `$?` so a killed guard was *recorded* as having
|
||||
passed; and a second signal mid-flush that discarded the output entirely.
|
||||
|
||||
**A correction, because the first version of this record justified the withdrawal with a number that
|
||||
is wrong.** It claimed, as measured fact, that trapping made a hook take 30 s to die where it had
|
||||
taken 1 s. That came from a review finding accepted without re-measuring, and the A/B refutes it:
|
||||
signalling the pid gives control **4.01 s** and trapped **4.12 s**; signalling the process group
|
||||
gives control **0.002 s** and trapped **0.054 s**. The stall is bash deferring a fatal signal until
|
||||
its foreground command finishes, and it happens **identically without any trap** — the control
|
||||
stalls too. The trap cost ~0.1 s, not 29 s. Recorded rather than quietly fixed, because a decision
|
||||
record carrying a fabricated measurement is worse than one carrying none: it reads as checked and
|
||||
talks the next reader out of checking.
|
||||
|
||||
**A second correction, in the same paragraph as the first.** The replacement rationale asserted that
|
||||
the round-3 sink "returned rc 0 on SIGINT where the uninstrumented control returns −2". That is
|
||||
**inverted**, and it was again taken from a review summary without re-measuring — the identical
|
||||
mistake, one round later, in the artifact written to correct it. The measured 12-cell matrix
|
||||
({group, pid} × {TERM, INT, HUP} × {control, round-3 trapped, HEAD}):
|
||||
|
||||
| | TERM | INT | HUP |
|
||||
|---|---|---|---|
|
||||
| group kill — control / trapped (`6f006a1b7`) / HEAD | −15 / −15 / −15 | −2 / −2 / −2 | −1 / −1 / −1 |
|
||||
| pid kill — control / trapped (`6f006a1b7`) / HEAD | −15 / −15 / −15 | **0 / −2 / 0** | −1 / −1 / −1 |
|
||||
|
||||
Reproduce: spawn a `stream`-mode hook that prints a decision then `sleep`s, in its own session;
|
||||
signal 1 s in with `os.killpg` or `os.kill`; read the wait status. Independently re-measured, five
|
||||
repeats per cell, every cell unanimous, stdout byte-identical in all runs — and the "would not have
|
||||
caught round 3" claim carries its own positive control: patching the trapped handler's re-raise back
|
||||
to `exit 0` turns the same test red on all three signals.
|
||||
|
||||
So the trapped sink returned **−2 where the control returns 0** — a real transparency violation, but
|
||||
in the *restrictive* direction, which also makes "the consent bug survived" the wrong
|
||||
characterisation. **HEAD matches the control in all twelve cells.** And the claim that the
|
||||
exit-status assertion "catches the SIGINT case" is false as written: the test signals the process
|
||||
group, and in every group cell the trapped sink is indistinguishable, so it would not have caught
|
||||
round 3 at all. The test asserts the transparency of the current code, which is what it is for; it
|
||||
is not evidence that the trap was bad.
|
||||
|
||||
**What actually justifies the withdrawal**, with the two false reasons removed. Three defects in
|
||||
three rounds, each real and measured at the time — a handler that turned a signal into consent and
|
||||
let a push reach `main`, a `local` that clobbered the status so a killed guard was recorded as
|
||||
passing, and a nested signal that discarded the output. One measured transparency violation that
|
||||
survived the fixes (pid + SIGINT, above). And a rescue window that is sub-millisecond, because every
|
||||
capture-mode hook prints its decision and exits immediately. Machinery needing a signal-to-status
|
||||
map, an idempotent flush, correct re-raise semantics and `BASHPID` handling, to rescue that window,
|
||||
is not worth its failure modes.
|
||||
|
||||
What is lost by withdrawing it, enumerated rather than waved past, because
|
||||
`process.enumerate-workaround-behaviors-before-deleting` exists for precisely this move:
|
||||
|
||||
| Behaviour the trap provided | Status now |
|
||||
|---|---|
|
||||
| A killed hook's `exit` record | **LOST.** This is why `report` counts `fire` records, not `exit` records — a hook that fired and was killed still shows as having fired |
|
||||
| Stdout a killed hook had already printed | **LOST**, and near-unreachable: every capture-mode hook prints its decision and exits immediately, a sub-millisecond window |
|
||||
| Cleanup of the two temp files on a kill | **LOST**, bounded by `$TMPDIR` cleanup |
|
||||
| Idempotent flush (`ETV_HOOK_FIRE_FLUSHED`) | **RETAINED** — cheap, and the failure it prevents is silent |
|
||||
|
||||
The invariant that replaces the rescue is asserted directly: **under a signal, an instrumented hook
|
||||
is indistinguishable from an uninstrumented one** — compared on exit status, stdout, stderr and time
|
||||
to die. It **signals the process group, not the pid**, and that is not a detail: bash defers a fatal
|
||||
signal until its foreground command finishes, so signalling the pid alone makes both sides take the
|
||||
full sleep and the timing assertion distinguishes nothing.
|
||||
|
||||
Its limit, stated because the alternative is a coverage claim that is not true: **the group-kill
|
||||
arrangement makes the test blind to the round-3 trapped sink**, which passes every assertion there.
|
||||
The divergence that sink actually had is a pid-kill one (the table above). This test pins the
|
||||
transparency of the code that ships; it is not a proof that any earlier version was worse.
|
||||
|
||||
**A SECOND withdrawal, on the same reasoning.** A `test_no_redirection_in_the_sink_PRECEDES_its_stderr_redirect`
|
||||
generalised the `exec … 2>/dev/null` rule to any command by pattern-matching shell source. It went
|
||||
through four iterations, each fixing a false positive or a false negative the last had introduced —
|
||||
`>/dev/null 2>&1` flagged wrongly, then operators preceded by a digit (`exec 0<`) missed entirely,
|
||||
then multi-command lines flagged wrongly — and cold review then constructed more of both
|
||||
(`[[ "$x" < "$y" ]] 2>/dev/null`, a line continuation, a `>` inside a quoted string). It is deleted
|
||||
rather than patched a fifth time, following the precedent in `docs/guard-inventory.md`: a weak
|
||||
detector is itself the symptom-keyed mistake, and another regex patch adds confidence without
|
||||
soundness. The narrower `test_no_exec_in_the_sink_carries_a_STDERR_REDIRECT` is retained — it is a
|
||||
sound match on a fixed keyword and covers the dangerous subset, which is every case that has
|
||||
actually occurred.
|
||||
|
||||
**One transparency limit, measured and stated rather than papered over.** On a group kill with a
|
||||
live foreground child, bash prints its own job-control notice (`Terminated: 15 sleep 5`) that an
|
||||
uninstrumented hook does not. Isolated: sourcing the sink alone does not produce it, and a no-op
|
||||
`EXIT` trap does not either — doing ANY work at exit keeps the shell alive the ~50 ms bash needs to
|
||||
print, and doing work at exit is the sink's entire purpose. The A/B exempts exactly that notice and
|
||||
nothing else.
|
||||
|
||||
**The exemption is doing real work and the honest reading is uncomfortable**, so it is stated
|
||||
plainly: one observed form of the notice is
|
||||
`…/scripts/hook-fire-log.sh: line 109: <pid> Hangup: 1 sleep 25` — bash attributing its own
|
||||
message to whichever line of the sink was executing, which means the instrumentation's path appears
|
||||
on a guard's stderr where an uninstrumented hook emits nothing. That is a genuine, if small, leak,
|
||||
and the exemption hides it. It is accepted rather than fixed because it cannot be suppressed while
|
||||
the sink does any work at exit, and it costs one line in a run that is already being killed.
|
||||
|
||||
**CI found what six review rounds on one platform could not, and it was a live guard defect.** The
|
||||
non-vacuity assertion demands that `pretooluse-bom-guard.sh` reach a `deny`. On the Linux runner it
|
||||
could not, because the guard detects a BOM with `xxd -p` — and **`xxd` ships with vim and is absent
|
||||
there**, so the command substitution yielded empty, never matched, and the guard passed every BOM
|
||||
silently. It has been fail-open on any host without vim for its whole life. `od -A n -t x1 -N 3` is
|
||||
POSIX, byte-identical on both platforms, and is what it uses now.
|
||||
|
||||
Two things follow. The defect was invisible to every round of review because all of them ran on
|
||||
macOS, where `xxd` exists — **the environment was a sampled population too**, and the only reason it
|
||||
surfaced is that a test asserted the guard must actually decide something. And two of the three CI
|
||||
failures were my *tests* being platform-specific (a bare `Terminated` job notice where macOS prints
|
||||
`Terminated: 15 sleep 5`; locales that do not exist on the runner, so bash warned and the
|
||||
assertion measured the missing locale rather than the fix). A test suite is code, and it inherits
|
||||
the same blind spot as the code it checks.
|
||||
|
||||
**The generalisable lesson is about the fixing, not the bugs.** This took three review rounds, and
|
||||
every round found that the previous round's *fix* had introduced something beside its target. The
|
||||
signal fix is the sharpest case: it corrected a lost log line and, in doing so, turned a killed
|
||||
`main`-protection guard into a rubber stamp. The rule this repo already has —
|
||||
`process.consistency-fix-new-code-scrutiny`, a fix is new code and gets reviewed as new code —
|
||||
applies with full force to instrumentation, because the fix and the thing it instruments occupy the
|
||||
same execution path.
|
||||
|
||||
Two of those fixes read as done and were not, which is the more useful pattern to recognise. The
|
||||
path-traversal hardening reused the record scrubber, which deliberately permits `/` and `.`, so
|
||||
`../../escaped` passed through untouched: **passing a value through a scrub is not the same as
|
||||
passing it through the right scrub.** And `exec … 2>/dev/null` was fixed twice and reintroduced a
|
||||
third time, eight lines below the comment forbidding it — which is why
|
||||
`test_no_exec_in_the_sink_carries_a_STDERR_REDIRECT` now enforces the rule instead of restating it.
|
||||
Enumerating instances loses to removing the class, and the third instance is what proves it.
|
||||
|
||||
One measurement note, since it cost a round of confusion: a shared `TMPDIR` between the two halves
|
||||
of the differential let the control's one-shot throttle marker suppress the instrumented run, so the
|
||||
test reported a swallowed decision that never happened. A differential whose halves share mutable
|
||||
state stops measuring the difference and measures the state.
|
||||
|
||||
**What the first measurement immediately produced**, which is the argument for the whole thing. A
|
||||
live run recorded `pretooluse-agent-model` firing on an `Agent` dispatch and deciding `no-op` where
|
||||
`ask` was expected. Under the old regime that would have been an unanswerable suspicion. Instead the
|
||||
question was settled by dumping the real payload: the dispatch carried `model: haiku`, so `no-op`
|
||||
was correct and there was no bug. Note what did *not* happen — no argument from the hook's source
|
||||
about what it "must" do, which this repo's record says has been wrong repeatedly. One printed line
|
||||
settled it in one run.
|
||||
|
||||
**The limit, stated rather than implied.** A zero in the report means *not observed in this window*,
|
||||
which is two different findings wearing one number: a hook that is broken, and a hook whose trigger
|
||||
genuinely did not occur. The report says so in its own footer rather than presenting a zero as a
|
||||
verdict. `pretooluse-nav-guard` and `pretooluse-merge-consent` are the standing examples — neither
|
||||
can be exercised without either a browser session or a real merge, so their zeroes are expected and
|
||||
distinguishing them stays a human judgement.
|
||||
@@ -0,0 +1,84 @@
|
||||
---
|
||||
key: testing.mutation-claims-are-executed
|
||||
title: '2026-08-22 — a MUTATION grade is EXECUTED every run, from a declared clause, or it is not that grade (#790)'
|
||||
status: active
|
||||
since: '2026-08-22'
|
||||
supersedes: none
|
||||
superseded-by: none
|
||||
rule: 'A `MUTATION` row in `docs/guard-inventory.md` is not a statement that someone once witnessed a red. It carries a DECLARED clause mutation in `scripts/tests/mutation_manifest.py`, and `scripts/tests/test_mutation_harness.py` applies that mutation to an isolated copy of the repository on every run and requires the row''s OWN named test to go red. The manifest and the MUTATION rows are compared for SET EQUALITY in both directions, so a row cannot claim the grade without a mutation and a mutation cannot outlive the grade it justifies. EXIT STATUS IS NOT THE VERDICT: each entry also declares the DIAGNOSTIC its red must carry, matched against pytest''s exception output alone, because pytest reports a crashing test exactly as it reports a detecting one and a red for an unrelated reason is evidence about nothing. WHERE THE GUARD IS ITSELF A TEST, `target` may differ from `guard` and the exact-once check applies to the declared TARGET. Two shapes are admissible and the choice is not free. Where the guard''s assertion IS the check — a completeness comparison against a Markdown inventory — the mutation goes into the guarded ARTIFACT, per `testing.guard-ships-with-mutation-proof`''s checker-guard exception, because mutating such a checker''s own POPULATION demonstrates a false POSITIVE while proving nothing about the detection the row claims. Where the guard is a test module wrapping a separately mutable DETECTOR or helper, the clause may be in that detector, since disarming it is a real clause disarm and the module''s own assertion is what notices. THE MUTATION IS DECLARED, NEVER INFERRED: a harness that guessed which clause of a 90-line hook is the guard would manufacture the confident-but-empty coverage this exists to prevent, which is why `testing.guard-ships-with-mutation-proof` rejected a generic runner. Where a proof test already names its clause in source, the manifest reuses THAT string, so a retarget in either place is caught by the other. COARSENESS IS RECORDED, NOT HIDDEN: each entry is graded `CLAUSE` or `DETECTOR`, and a `DETECTOR` entry — one whose detector accumulates faults from independent arms, so disarming any single arm leaves its proof test green — must CARRY the finer mutation that survived, which is re-run every time and required to keep surviving. Guards that are not graded `MUTATION` each carry a STATED reason in that same manifest, keyed on the guard and compared for SET EQUALITY against the inventory''s `GUARD` rows in both directions — so a new guard cannot arrive without someone writing what a proof would need, and a reason cannot outlive the row it is about. Keying the reason on the row''s GRADE instead is tautological (a new guard inherits one and nobody looks at it) and a pinned COUNT moves only on net change; both were tried and are rejected. The sandbox is a real git repository built from `git ls-files` with working-tree content, never a filesystem walk.'
|
||||
signals: 'mutation harness · declared clause not inferred · MUTATION row is executed · set equality manifest vs inventory · CLAUSE vs DETECTOR granularity · surviving finer mutation is re-run · pytest exit code 1 is the only red that counts · every other exit status is rejected · positive control before any mutation · sandbox from git ls-files · paths: `scripts/tests/test_mutation_harness.py`, `scripts/tests/mutation_manifest.py`, `scripts/tests/mutation_harness_lib.py`, `docs/guard-inventory.md` · issues: #790, #775, #774, #778, #806'
|
||||
mechanics: 'The harness builds ONE sandbox per session — tracked files only, from `git ls-files -s`, in an environment with every `GIT_*` variable stripped so an exported `GIT_DIR`/`GIT_COMMON_DIR`/`GIT_CONFIG_*` cannot point its `init`/`add`/`commit` at the real repository — and resets it between mutations with `git reset --hard` TO A BASELINE COMMIT RECORDED OUTSIDE THE REPOSITORY, plus `git clean -qffdx`. Both halves matter: bare `git reset --hard` resets to whatever HEAD is, so a proof test that COMMITS inside the sandbox moves HEAD onto a commit carrying the mutant and every later reset restores it faithfully; and a baseline held as a REF inside the sandbox is one more thing a proof can move. `core.hooksPath`, `commit.gpgsign` and `core.worktree` are pinned on every git invocation for the same reason. The `-ff` removes a nested repository a proof may have left. It runs the full set of named proof tests UNMUTATED first, inside the fixture rather than as a separate test so ordering is a dependency and not a convention. Only pytest exit code 1 counts as red; every other status is rejected. That is what closes the two ways a proof ref goes stale, and they are measured rather than assumed: with an explicit `file.py::function` node id a missing file and a missing function both exit 4, while 5 needs a collection that succeeded and selected nothing. The sandbox skips the `ErsatzTV-macOS` gitlink and recreates the `.claude/skills/jellyfin` symlink as a symlink. Cost: ~14s, against a ~4min `script-tests` suite.'
|
||||
---
|
||||
|
||||
**What #775 left open, in its own words: "nothing checks that a row claiming `MUTATION` is telling
|
||||
the truth."** It said so deliberately — the bookkeeping half (population, set equality, proof refs
|
||||
resolving to a real `def`) was mechanised, and the judgement half was left with review. Both cold
|
||||
reviews of #774/#775 came back to the same place: the witnessing had happened once, by hand, and
|
||||
**hand-run evidence decays the moment someone edits the guard.** Two false `MUTATION` grades were
|
||||
caught by *reading* in the first draft, and a third round would likely have found more.
|
||||
|
||||
**Why a generic runner was rejected, and what changed.** #775 rejected a mutation runner for shell
|
||||
hooks on the grounds that it "would have to know which clause of a 90-line hook is the guard". That
|
||||
objection is correct and is not answered by better inference — it is answered by not inferring. The
|
||||
clause is *declared*, one entry per row, and for most rows the declaration already existed: the
|
||||
proof tests name their own clauses in source (`= "efbbbf" ]; then`, `UNSET_CLAUSE`, `if [ "$RC"
|
||||
-eq 0 ]; then`). Reusing those strings rather than inventing parallel ones is what makes the two
|
||||
halves catch each other's drift.
|
||||
|
||||
**What a green harness proves, and what it does not.** It proves the proof ref names a test that
|
||||
still exists and still collects; that the declared clause still occurs, exactly once, in the entry's
|
||||
declared `target`, which is not always the guard's own file; and — via the positive control — that every named proof test is GREEN on the
|
||||
unmutated sandbox, so "the mutation was noticed" cannot be confused with "the test was already red".
|
||||
The proofs that carry a disarm of their own run it during that control; the two that are plain
|
||||
production set-equality checks have none, and the harness supplies theirs. It does **not** prove the declared clause is the only thing the guard hangs on.
|
||||
For three entries (measured 2026-08-22: the BOM guard, `build_decisions_catalog.py`
|
||||
and `prove-fix.sh`) the redness arrives through the proof test's own "this clause has moved, RETARGET
|
||||
it" assertion rather than through changed behaviour. That is the intended reading rather than a hole:
|
||||
those tests perform their behavioural disarm themselves on every green run; what they could not do
|
||||
was notice their own clause reference going stale. Every other entry reddens behaviourally, and each
|
||||
one's declared diagnostic says which.
|
||||
|
||||
**The `DETECTOR` grade is the honest half, and it exists because measuring found a case.** #775's
|
||||
record concedes that neutering `pin_population_faults` wholesale is "coarser than disarming one
|
||||
clause at a time — coarse enough that a single surviving clause would not be noticed". Running the
|
||||
finer mutations settled which guards actually admit clause-level proof: all but one do,
|
||||
including
|
||||
`pin_population_faults`, where disarming the against-the-registry comparison alone reddens the proof
|
||||
because a job that loses its `container:` block leaves the two derived sets equal. One does not.
|
||||
`instrumentation_faults` accumulates from four independent arms and a stripped hook trips three of
|
||||
them at once, so disarming any single arm leaves the proof test green — measured, by running
|
||||
`if not _SOURCES_SINK.search(text):` → `if False:` and watching it pass. That surviving mutation is
|
||||
carried as data and re-run on every suite: if it ever starts reddening, the guard has become
|
||||
clause-provable and the entry must be regraded. A grade that cannot decay quietly is the point.
|
||||
|
||||
**Three shapes that were tried and rejected, because each looks like verification and is not.**
|
||||
Emptying a checker's own population reddens its proof with an `IndexError` — a crash, not a
|
||||
detection, and any breakage of the derivation satisfies it equally. Shrinking the guard-inventory
|
||||
checker's hook glob reddens its proof by making every real row report as PHANTOM: a false positive,
|
||||
which says nothing about the missing-row detection the row claims. And planting a phantom row in
|
||||
`docs/remote-state-inventory.md` reddens a proof test that builds a phantom of its own and asserts
|
||||
the difference is exactly one entry — so the declared diagnostic appears through fixture
|
||||
contamination, in the opposite direction from the one the row claims. All three are replaced by
|
||||
artifact-level mutations that reach the guard's production assertion — a deleted inventory row, a
|
||||
renamed one — and `expect` is what makes the difference visible instead of arguable.
|
||||
|
||||
**Stating the other guards, and the two cheaper shapes that do not work.** Keying the reason on the
|
||||
row's *grade* is tautological: a new guard graded `NONE` inherits a sentence automatically and nobody
|
||||
ever looks at that particular guard. Pinning the *count* of undeclared guards is no better, because
|
||||
it moves only on net change — one guard arriving as another is promoted leaves it unchanged. What
|
||||
ships is a per-guard mapping compared for set equality against the inventory's `GUARD` rows. That is
|
||||
a hand-maintained table, which is normally the duplication family to avoid (#774's wrong summary
|
||||
counts, #788's verdict vocabulary) — but bidirectional set equality is what makes
|
||||
`docs/guard-inventory.md` itself safe, and it is what turns "state the ones you cannot declare" into
|
||||
something a new guard cannot slip past. Set equality ALONE is not sufficient and the gap is
|
||||
demonstrable: every comparison here reduces rows through a set or a dict, so a DUPLICATED row is
|
||||
invisible to all of them and a table contradicting itself reports full coverage. Duplicate rejection
|
||||
is a separate, explicit check, and copying the set-equality design without it recreates the hole.
|
||||
|
||||
**Two words that are not interchangeable.** A `NONE` row means the row nominates no proof; it does
|
||||
not mean the guard is untested. `scripts/ci-prove-ban-detects.sh` is graded `NONE` and is driven end
|
||||
to end by `test_ci_release_path_scan_job.py`, and eight hooks are driven through their deciding path
|
||||
by `test_hook_fire_log.py` — whose matrix asserts instrumentation TRANSPARENCY, that the wrapped and
|
||||
unwrapped runs agree, never that the decision is right or that a particular clause produced it. A
|
||||
statement that reads "no test drives this" over those guards sends the next session's proof work in
|
||||
the wrong direction, so each entry says which of the two is actually missing.
|
||||
@@ -0,0 +1,687 @@
|
||||
# Recurring defect shapes across the closed-issue corpus (ersatztv#773)
|
||||
|
||||
Root-cause analysis over every closed issue in the tracker, the detectors that would have caught
|
||||
each class, and an audit of the tooling already configured in this repo. This is an **analysis
|
||||
artifact**, not a rulebook: the rules it argues for land as decision records and follow-up issues,
|
||||
which are linked per class below.
|
||||
|
||||
Measured 2026-08-13 against `origin/main` at `f9f8f65ce`. **The issue corpus is a snapshot drawn at
|
||||
19:45 that day** — a boundary that matters, because #767 closed nineteen minutes later and is
|
||||
therefore absent from every count here (§3.7).
|
||||
|
||||
---
|
||||
|
||||
## 1. Method, and what these numbers can and cannot support
|
||||
|
||||
**Corpus.** All 349 closed issues (`state=closed&type=issues`, 7 pages, 349 distinct numbers,
|
||||
#1–#757 — pull requests excluded). Of those, **95 carry a `## Closing record`** comment. Each
|
||||
record was classified by one of five independent agents against a written taxonomy, with
|
||||
`NEW:<name>` available so the taxonomy could not be self-fulfilling. A sixth agent re-rated a
|
||||
stratified 15-issue sample blind, as an inter-rater control. A seventh sampled 25 of the 254
|
||||
pre-convention closed issues from their raw bodies and comments, to test whether the shapes exist
|
||||
outside the era that writes records.
|
||||
|
||||
**Three limits, stated up front because they bound every number below.**
|
||||
|
||||
1. **Record coverage is not uniform.** The `## Closing record` convention starts at #520
|
||||
(2026-07-21). Coverage is 31/32 for #600–699 and 4/69 for #1–99. So frequencies over the 95
|
||||
records are frequencies over *recent* practice.
|
||||
2. **Corpus composition shifts underneath the measurement.** The share of records describing no
|
||||
process failure at all falls from 12/19 (#60–#438) to 0/19 (#616–#671). Early issues are
|
||||
feature work; recent issues are largely CI/process hardening. Guard-shaped defects therefore
|
||||
dominate the recent records *partly by construction* — we have lately been building guards, so
|
||||
our defects are in guards.
|
||||
3. **A closing record is the closing author's self-report.** Raters were told to flag records whose
|
||||
own claims outrun what they describe, and several did (#60, #651, #691, #720). But the corpus
|
||||
systematically cannot contain a defect nobody noticed.
|
||||
|
||||
**What limit 2 does *not* undermine.** The obvious worry — that these classes are an artifact of
|
||||
recent guard-building — was tested and **refuted**. The backward sample found high-confidence
|
||||
instances of the same shapes well before the convention existed: twin-missed at #215 (REST playout
|
||||
mutations skip the build-lock gating the Blazor path had) and #403 (an unhandled enum silent at 5 of
|
||||
6 dispatch sites); wrapper drift at #198 and #287 (OpenAPI spec diverged from runtime JSON);
|
||||
masking guards at #232, whose SPA `PENDING_GRACE_TICKS` "existed only to paper over the resulting
|
||||
dishonest 200", and #234, where a blind `finally`-unlock was "papering over the stranded lock, so
|
||||
the two had to land together"; overclaim at #1 ("the earlier 'confirmed' was based on one curl test
|
||||
and several unverified assertions"). The convention changed the **density and legibility** of the
|
||||
evidence, not the existence of the shapes.
|
||||
|
||||
Classification of pre-convention issues is harder but mostly possible — of the five the backward
|
||||
rater flagged as unclassifiable, four (#94, #172, #265, #377) do carry a closing comment with a
|
||||
clear root cause on re-inspection, and only **#157** (an unresolved flaky-test report with a single
|
||||
comment) is genuinely evidence-starved. The `insufficient-evidence` verdict is therefore rarer than
|
||||
first reported, which if anything strengthens the backward result.
|
||||
|
||||
---
|
||||
|
||||
## 2. What the corpus actually says
|
||||
|
||||
95 records; **69 describe a process failure**, 26 describe none. Percentages are of the 69.
|
||||
|
||||
| Rank | Class | n | % | Shape |
|
||||
|---|---|---|---|---|
|
||||
| 1 | **Vacuous verification** | 17 | 25% | A check was green having done no work |
|
||||
| 2 | **Twin-missed** | 14 | 20% | The fix hit one instance and missed its structural twin |
|
||||
| 3 | **Symptom-keyed guard** | 9 | 13% | The guard keyed on the symptom seen, not the defect's mechanism |
|
||||
| 4 | *(new classes — see §3)* | 8 | 12% | Proposed by raters as fitting nothing above |
|
||||
| 5 | **Environment divergence** | 7 | 10% | Behaviour differs by interpreter/provider/hardware, untested there |
|
||||
| 6 | **Wrapper drift** | 5 | 7% | A hand-maintained mirror drifted from the contract it mirrors |
|
||||
| 7 | **Overclaim / stale claim** | 3 | 4% | An assertion stronger than its evidence, or since gone false |
|
||||
| 7 | **String-predicate churn** | 3 | 4% | A regex/grep predicate needing repeated rounds — but see §3.7: the *rounds* are cross-cutting, only the *substrate* is string-shaped |
|
||||
| 9 | **Identity-not-capability** | 2 | 3% | Authorization scoped by *who*, not by *what is being changed* |
|
||||
| 10 | **Masking guards** | 1 | 1% | Two guards on one condition; one hid the other's total failure |
|
||||
|
||||
**This reorders the issue's own ranking.** #773 put twin-missed first (9 instances) and vacuous
|
||||
verification second (8), from a 50-issue sample. Over the full record corpus they swap: **vacuous
|
||||
verification is the most common single shape.** Overclaim also drops sharply as a *primary* cause
|
||||
(4%) — raters overwhelmingly assigned it as a *secondary*. It is better understood as a modifier
|
||||
that rides on another failure than as a class with its own detector.
|
||||
|
||||
**Inter-rater control (measured, not asserted).** A sixth rater re-classified a stratified 15-issue
|
||||
sample blind. **Raw agreement 12/15 (80%)** across an 11-category scheme. The three disagreements
|
||||
are informative rather than noise:
|
||||
|
||||
- **#177** (twin-missed → no-failure) and **#521** (no-failure → twin-missed) disagree at the
|
||||
"did a process failure occur" boundary and point in **opposite directions**, so that boundary is
|
||||
noisy but not systematically biased. The 26 clean-record count should be read as ±3.
|
||||
- **#649** was rated twin-missed by one and wrapper-drift by the other — which is precisely the blur
|
||||
that Family C in §3 exists to resolve. Both raters named the other class as secondary.
|
||||
|
||||
The top-three separation is far larger than the disagreement. Treat ±1 rank as noise.
|
||||
|
||||
---
|
||||
|
||||
## 3. Consolidation: four families, one of which the issue's taxonomy missed
|
||||
|
||||
### Family A — reasoning about a representative instead of the population
|
||||
|
||||
**#773's central hypothesis, confirmed, and it extends further than proposed.** The issue asked
|
||||
whether twin-missed and vacuous verification are the same underlying error. They are — and
|
||||
symptom-keyed guards belong with them. All three are the same mistake at different targets:
|
||||
|
||||
- **twin-missed** — the *fix* was applied to a sample of the population,
|
||||
- **vacuous-by-sampling** — the *verification* sampled an empty or unrepresentative subset,
|
||||
- **symptom-keyed guard** — the *guard* matched only the member that was noticed.
|
||||
|
||||
**Size: 27 of 69 (39%)** — 9 twin-missed, 9 symptom-keyed, 9 vacuous-by-sampling. See §3.6 for the
|
||||
full partition; the number is smaller than the naive merge because records that belong to Families
|
||||
C and D are **not** double-counted here.
|
||||
|
||||
The merge is not merely rhetorical; it predicts a single detector, and the corpus already contains
|
||||
that detector, reinvented several times without anyone noticing it was the same rule:
|
||||
|
||||
> "the by-id handler covered 4 of 10 media types… **Sweep by FIELD, not by the call site the issue
|
||||
> names**" — #671
|
||||
> "#644's guard keyed on the *symptom* (an inflated pageSize) rather than the *defect*… an at-cap
|
||||
> request was structurally invisible to it" — #650
|
||||
> "filtered on tools that *already* declared `pageNum`, so a tool wrapping a paged endpoint with no
|
||||
> paging args escaped it entirely" — #616
|
||||
> "the pre-existing test filters `Where(t => t.QueryParameters is {Count: > 0})`, so a tool that
|
||||
> lost its query parameters escaped it entirely" — #757
|
||||
|
||||
Four sessions, four reinventions of one rule → **detector A** in §4.
|
||||
|
||||
**Scope limit, because it bounds the highest-value detector in this document.** The distinction is
|
||||
what kind of thing the population is made of.
|
||||
|
||||
- **A population of *values*** — enum members, OpenAPI operations, registered tools — always has an
|
||||
external authoritative source, and detector A applies directly. `Enum.GetValues<T>()` is that
|
||||
source for an enum, which is precisely #503's fix for a catch-all `default` that silently absorbed
|
||||
an unhandled value.
|
||||
- **A population of *sites in code*** — the places that dispatch on a value, rather than the values
|
||||
themselves — has no external list to assert against. #403 is the case: `PlaybackOrder` is an enum,
|
||||
so its *values* are enumerable, but the defect was that 5 of 6 **dispatch sites** failed to handle
|
||||
one, and no artifact anywhere enumerates those sites. That residue needs find-all-references
|
||||
tooling, which is broken here and tracked in #777, not a set-equality assertion.
|
||||
|
||||
### Family B — the check never ran at all
|
||||
|
||||
**Size: 9 of 69 (13%)** — the 8 vacuous records that are not sampling errors, plus the one masking-guards
|
||||
record. Here the check was *dead*, not mis-aimed. `scripts/tests/`
|
||||
invoked by no CI job (#631); a validator whose call site could be deleted with the suite still green
|
||||
(#621); a `${{ }}` in a shell **comment** silently dropping a step while the job reported success in
|
||||
6s (#751); `if ! cmd; then status=$?` reading bash's logical negation, so a failing spec run exited 0
|
||||
(#445); new pre-push logic never wired to receive stdin, "dead code that every unit test still passes
|
||||
over" (#719). A sampling detector does nothing here. This needs **detector B**: proof the guard can
|
||||
go red.
|
||||
|
||||
### Family C — two copies of one thing
|
||||
|
||||
**Size: 9 of 69 (13%)** — the 5 wrapper-drift records plus the 4 twin-missed records whose twin is
|
||||
literally a second copy (#510, #649, #711, #756). Those 4 are counted **here and not in Family A**.
|
||||
|
||||
`.claude/` and its byte-identical `.codex/` mirror where only one was in the `PROTECTED` list (#711);
|
||||
an advisory local hook and the server-enforced workflow where four rounds of hardening landed on the
|
||||
copy with the lower stakes (#649); a hand-maintained MCP schema drifting from the generated OpenAPI
|
||||
DTO by one field and silently clearing it (#754, again #757); a hand-rolled frontmatter parser
|
||||
accepting YAML that PyYAML rejects (#674); a dropped-step guard added to one required workflow and
|
||||
not its fail-open twin (#756).
|
||||
|
||||
The preferred fix is **deletion** — one implementation, called from both places, as #649 did by
|
||||
extracting `scripts/pr-changed-files.sh` and recording `ci.shared-pr-file-enumeration`. (#649 also
|
||||
shows why the duplication is dangerous: before the extraction the guard was "safe only by
|
||||
redundancy," and a failing exit status "previously left the *entire* suite green.")
|
||||
**But dedup is a preference, not a law, and #711
|
||||
is the counterexample that proves it.** That record deliberately *kept* its enumerative list:
|
||||
*"the list stays ENUMERATIVE rather than derived — a derived rule would have to be evaluated against
|
||||
the very file list being classified, putting more moving parts inside a security predicate to save
|
||||
one line per new tooling directory."* Deriving a security predicate from the input it judges is
|
||||
worse than maintaining two entries. So: dedup by construction where the duplication is incidental;
|
||||
keep the enumeration where deriving it would feed the judged input back into the judge.
|
||||
|
||||
### Family D — check-and-use race over mutable state *(the taxonomy missed this entirely)*
|
||||
|
||||
**Size: 5 of 69 (7%).** Three raters proposed overlapping new classes — *unprompted by the taxonomy*,
|
||||
which had no bucket for this, though they shared a corpus and a brief and so were not strictly
|
||||
independent. The stronger evidence is the **blind** re-rater, who saw none of their output and
|
||||
proposed the same class again for #536 under its own name (`toctou-partial-atomicity` vs
|
||||
`half-atomic-toctou`).
|
||||
|
||||
Four of the five (#536, #622, #706, #707) were tagged `NEW` by their raters. **#632 was rated
|
||||
twin-missed as its primary**, with the race named as its secondary; it sits here because the
|
||||
mechanism is the race, and it is counted here rather than in Family A.
|
||||
|
||||
| # | The race |
|
||||
|---|---|
|
||||
| 536 | Check-then-act split by an `await`; the write side was atomic, the read side a stale `Volatile.Read` |
|
||||
| 622 | Merge consent bound to a head sha, then Gitea's async auto-merge evaluated against a later head |
|
||||
| 632 | Head sha bound, but retargeting the PR base changes the diff while moving neither sha nor status |
|
||||
| 706 | Status writes are read-then-write with no compare-and-set; an older run can finish last and win |
|
||||
| 707 | Paged file enumeration diffs each page against the base's *live* tip, so a mid-paging advance drops rows |
|
||||
|
||||
**Five instances — larger than identity-not-capability, string-predicate churn and overclaim
|
||||
individually**, and #773's taxonomy had no bucket for it. The unifying property:
|
||||
a check and the action it authorizes are separated in time over state that can change in between,
|
||||
with nothing pinning a version. #622's own record puts it exactly: *"The gate was never bypassed —
|
||||
it was satisfied against a snapshot that stops being true."*
|
||||
|
||||
### Two further cross-era classes, from the pre-convention sample
|
||||
|
||||
- **Fail-open by default** — a surface that defaults to permissive when config is absent or a step
|
||||
vanishes: API writes open when `Api:WriteKey` is unset (#280); a dropped step is fail-CLOSED in
|
||||
`review-verdict.yml` but fail-OPEN in `docker-build.yml` (#751, #756, #768); a jq-1.6-inert
|
||||
classifier (#647). Distinct from identity-not-capability: nothing is mis-scoped, the *default*
|
||||
is wrong.
|
||||
- **Destructive replace** — a full-replace path that silently destroys state a reconcile-by-id
|
||||
would have kept: a schedule PUT resetting fill-group progression, which #252 fixed with a
|
||||
"scoped positional/no-op reconcile" so that a no-op PUT-back stops being destructive; a
|
||||
full-replace wrapper missing one field and clearing it (#754); a dedup fix that turned a
|
||||
duplicate row into permanent data loss because the remove filter used a different key (#500).
|
||||
|
||||
### 3.6 The partition, stated exactly
|
||||
|
||||
The families overlap conceptually — a twin-missed record can be read as "another member of a
|
||||
population" or as "a second copy" — so membership is given as an explicit partition, **mechanically
|
||||
verified to cover all 69 records exactly once**. Read a family's n as a disjoint count, not a tally
|
||||
of everything the family's description could fit:
|
||||
|
||||
Family letters A–E are the ones §4 names detectors for, and those letters correspond. **This table
|
||||
assigns no letters below E.** §4's detectors F and G address fail-open and destructive-replace,
|
||||
which are cross-era classes drawn from the pre-convention sample and are *not* members of this
|
||||
69-record partition — so an F or G in §4 refers to nothing in this table.
|
||||
|
||||
| Family | n | % | Composition |
|
||||
|---|---|---|---|
|
||||
| **A** population reasoned about via a sample | 27 | 39% | 9 twin-missed + 9 symptom-keyed + 9 vacuous-by-sampling |
|
||||
| **B** the check never ran | 9 | 13% | 8 vacuous-by-non-execution + 1 masking-guards |
|
||||
| **C** two copies of one thing | 9 | 13% | 5 wrapper-drift + 4 twin-missed-by-duplication |
|
||||
| **E** environment divergence | 7 | 10% | |
|
||||
| **D** check-and-use race over mutable state | 5 | 7% | 4 rated `NEW` + #632 rated twin-missed with the race secondary |
|
||||
| — overclaim / stale claim | 3 | 4% | no detector — see §4 |
|
||||
| — string-predicate churn | 3 | 4% | no detector proposed; see §3.7 — the round-churn it is named for is cross-cutting |
|
||||
| — identity-not-capability | 2 | 3% | |
|
||||
| — unmerged singletons | 4 | 6% | #60, #503, #586, #688 — real, but one instance each |
|
||||
| **Total** | **69** | — | the **n** column is exact; percentages are rounded to the nearest point and sum to 99% |
|
||||
|
||||
The partition is checked mechanically rather than by eye: the assertion is that the union of the
|
||||
families equals the set of process-failure records, with no overlap. That check immediately caught
|
||||
a transcription slip (#438 omitted from the control set) — a small live demonstration of detector A
|
||||
applied to this analysis's own numbers.
|
||||
|
||||
Two caveats travel with these percentages. The Family A/C boundary is a judgment about whether a
|
||||
twin is "a second copy" or "another member of a population", and #649 is exactly the record the two
|
||||
raters split on. And per §1 limit 2, these are proportions **over a recent, guard-heavy corpus** —
|
||||
the backward sample establishes that the shapes existed before, not that they occurred in these
|
||||
ratios.
|
||||
|
||||
### 3.7 Round-churn is a property, not a class — and the fix text is the fourth sampling target
|
||||
|
||||
The ranked table lists **string-predicate churn** as a class, inherited from #773's own taxonomy
|
||||
("a parser / string-matching predicate needing repeated rounds"). That name identifies a *substrate*
|
||||
when the evidence identifies a *cross-cutting property*. Counting records that narrate three or more
|
||||
review rounds:
|
||||
|
||||
| Family | n | records with ≥3 rounds |
|
||||
|---|---|---|
|
||||
| A — population via a sample | 10 | #460, #496, #616, #633, #644, #650, #671, #684, #726, #743 |
|
||||
| B — the check never ran | 5 | #445, #620, #631, #685, #751 |
|
||||
| C — two copies of one thing | 5 | #510, #649, #754, #756, #757 |
|
||||
| E — environment divergence | 5 | #491, #643, #647, #648, #668 |
|
||||
| D — check-and-use race | 3 | #622, #632, #706 |
|
||||
| identity-not-capability | 2 | #697, #698 |
|
||||
| **string-predicate churn** | 2 | #578, #629 |
|
||||
| overclaim | 1 | #651 |
|
||||
| **Total** | **33 of 69** | every family represented |
|
||||
|
||||
*Method: every one of the 69 records was read for round language — 48 mention "round" at all, 33 state
|
||||
an explicit count of three or more. Two attributions are loose: #647 narrates four rounds that
|
||||
happened on #643, and #629 counts a round it deliberately did not run.*
|
||||
|
||||
**The class named after the phenomenon holds 2 of the 33.** Round-churn should therefore be read the
|
||||
way §2 reads overclaim — a modifier riding on another failure, not a class with its own detector.
|
||||
#697 settles it independently: three BLOCKED rounds on credential scoping, no parser anywhere near it;
|
||||
#698 ran six.
|
||||
|
||||
One honest qualification against over-correcting. String predicates *are* disproportionately
|
||||
round-prone — 2 of that class's 3 records, against 10 of Family A's 27. The naming error is not that
|
||||
the association is false; it is that the class was defined by the substrate where the property was
|
||||
noticed, so 31 instances outside it had nowhere to be counted.
|
||||
|
||||
**Why the rounds happen, which is the part that generalises.** §3 Family A names three things that
|
||||
get sampled instead of enumerated — the fix, the verification, the guard. There is a fourth: **the
|
||||
review scope.** A re-review briefed to "check the reported finding" samples; the population is the
|
||||
whole changed artifact, and the text written to *fix* the last round is unreviewed by construction at
|
||||
the moment it is written. That is why each round's defect lands in the newest prose rather than in
|
||||
the text under review.
|
||||
|
||||
**Two limits on this section, since it is the one making a claim about its own production.**
|
||||
|
||||
- **33 is a floor, not a total.** It counts records that *state* a round count; a record that ran
|
||||
four rounds without narrating them is invisible here. Do not read the 36 remaining records as
|
||||
single-round work.
|
||||
- **Counting this by pattern rather than by enumeration under-reports it by roughly half.** A regex
|
||||
over the corpus finds 18 of these 33 — the misses narrate their rounds in the same words as the
|
||||
hits. If you re-derive this number, read all 69.
|
||||
- **#767 — the case #773 describes as taking eight review rounds — is excluded by snapshot timing,
|
||||
not by absence.** The corpus was drawn at 19:45 on 2026-08-13; #767 closed at 20:04, nineteen
|
||||
minutes later, and now carries a full `## Closing record`. So the worst instance of the phenomenon
|
||||
is missing from every count in this document, including the 33 above. It is named here rather than
|
||||
quietly folded in, because re-drawing the corpus would move every denominator in §2 and §3.6 and
|
||||
the snapshot boundary has to sit somewhere.
|
||||
|
||||
Reading its record changes nothing structurally and confirms Family A twice over — it reinvents
|
||||
detector A independently, for the fifth and sixth times in this repo: *"Refuting two variants of a
|
||||
channel is not clearing the channel"* (the working attacks through that channel were never tried),
|
||||
and *"Enumerating **shapes** of a command loses. Nine disarms across two rounds; running the
|
||||
command settles them together."* That last sentence is detector A in one line — stop sampling the
|
||||
shapes you thought of, execute the population.
|
||||
|
||||
---
|
||||
|
||||
## 4. Detectors
|
||||
|
||||
Ranked by instances covered per unit of build cost. "Rule status" distinguishes a **docs problem**
|
||||
(no rule exists) from a **hooks problem** (the rule exists and nothing enforces it) — they have
|
||||
different fixes, and conflating them is why several of these recurred.
|
||||
|
||||
| # | Detector | Covers | Rule status |
|
||||
|---|---|---|---|
|
||||
| **A** | **A guard derives its expected set from the authoritative source and asserts set equality — never `filter`, never a sample.** The population comes from the enum / the OpenAPI doc / the workflow YAML / the provider list, and the assertion is equality against it. A filter over the population cannot see the member that is missing. | Family A: 27 records (39%), minus the code-structural residue noted in §3 | **rule-missing** as a general rule; instantiated ad hoc in #616, #644→#650, #671, #757 |
|
||||
| **B** | **Every guard ships with a proof it can go red: delete or disarm *that guard alone* and the suite must fail.** Not "a test exists" — a mutation. | Family B: 9 records (13%) | **rule-present-unenforced** — stated in #685's record and in project memory, enforced nowhere |
|
||||
| **C** | **Dedup by construction.** When two copies of one rule exist, delete one and have both callers invoke it — *unless* deriving the list would feed the judged input back into the judge, which is #711's reasoned exception. | Family C: 9 records (13%) | **rule-missing** |
|
||||
| **D** | **Anything read-then-written against live Gitea/remote state pins a version or uses compare-and-set.** See the honesty note below — this is a fix pattern, and its *detector* is detector A applied to an enumerated inventory of such sites. | Family D: 5 records (7%) | **rule-missing** (partially addressed for the merge path by the per-sha required check from #622) |
|
||||
| **E** | **Run the check under the interpreter/provider it will actually run under**, and preflight-log the version. | Environment divergence: 7 records | **rule-present** and working — `scripts/jq-preflight.sh` + `ci.jq-version-contract` closed the jq axis after #643/#647/#648. **Unclosed axes**: SQLite-vs-MySQL query semantics (#668), GPU generation (#505), CI-VM speed (#512) |
|
||||
| **F** | **Test the DENY path with the production config value.** #756's lesson generalised: a fixture that omits a field tests only the default, so a fail-open in the production value stays invisible. Parametrise the whole matrix, including "config absent". | Fail-open-by-default: ~5 records | **rule-missing** |
|
||||
| **G** | **Full-replace endpoints assert their complete field list in a test**, and prefer reconcile-by-id over delete-and-reinsert where child state exists. | Destructive replace: ~3 records | **rule-missing** |
|
||||
|
||||
**Detector A is the highest-value single change in this analysis** — one convention, 39% of the
|
||||
recorded process failures over this corpus, and it is already proven four times in this repo under
|
||||
four different names. Two honesty notes on that ranking, because it drives #774's priority:
|
||||
|
||||
- **The 39% is measured over a recent, guard-heavy corpus** (§1 limit 2). The backward sample shows
|
||||
the shape predates that corpus; it does not show the *proportion* holds across eras. Read it as
|
||||
"the largest family in the work we have been doing lately," which is still the right basis for
|
||||
prioritising the next change, but not as a timeless property of the project.
|
||||
- **Detector A also applies to review scope, which is free.** Per §3.7, a re-review briefed to
|
||||
"check the reported finding" samples the artifact. Briefing it to sweep the whole changed artifact
|
||||
enumerates it. This costs one sentence in a review brief and is the only detector here with no
|
||||
build step at all.
|
||||
- **Detector D is weaker than its neighbours in this table and is listed anyway.** A, B, F and G
|
||||
each name a check that fails when violated. D names a *fix pattern* with no general lint — you
|
||||
cannot mechanically spot "this code should have pinned a sha." What makes it actionable is that
|
||||
the population is small and enumerable: the handful of scripts that touch live remote state. So
|
||||
D's real detector is detector A applied to that inventory, which is why #778's scope is "enumerate
|
||||
every such site and mark each pinned / CAS / knowingly-unsafe" rather than "write a linter."
|
||||
|
||||
### Classes where no mechanical detector is plausible — stated rather than papered over
|
||||
|
||||
The issue explicitly asked for this, and inventing a weak detector here would itself be the
|
||||
symptom-keyed-guard mistake.
|
||||
|
||||
- **Overclaim / stale claim.** No check can tell that a sentence is stronger than the evidence
|
||||
behind it. Partial mitigations exist and should not be oversold: `stale-after` frontmatter (#603)
|
||||
dates a claim, and #578's retracted-term grep catches a *specific known* retraction propagating
|
||||
into generated artifacts. Neither detects a fresh overclaim. This stays a review responsibility.
|
||||
- **Per-task review blindness** (#60). A review scoped to one task's diff structurally cannot see a
|
||||
defect that only exists once several tasks compose — five Important findings at #60 were invisible
|
||||
to six per-task reviews and surfaced only in a whole-branch pass. The fix is a *process* step
|
||||
(a whole-branch review before close), not a check.
|
||||
- **Omitted brief constraint** (#586). A delegated brief silent on a hazard gets a plausible-but-wrong
|
||||
default — "an omitted rule isn't an unenforced rule, it's a rule replaced by whatever default the
|
||||
agent reaches for." A brief lint is conceivable but would be a keyword matcher, i.e. exactly the
|
||||
string-predicate class. A hazards checklist in the brief template is the honest ceiling.
|
||||
- **Guard-parity by verb** (#458), and re-deriving an inherited exemption against a new failure mode
|
||||
(#484). Both need judgment about whether a prior rationale still applies.
|
||||
|
||||
### A meta-finding about this repo's own knowledge base
|
||||
|
||||
`docs/decisions/records/` holds **189 active records**, and the two most relevant to this analysis
|
||||
(`testing.enumerating-guard-identity-not-position`, `ci.required-job-step-execution-markers`) are
|
||||
extraordinarily detailed — each a full account of one incident. The corpus grows **one record per
|
||||
instance**. That is Family A operating on our own process: we are enumerating cases rather than
|
||||
removing the mechanism. Detector A, C and B are class-level rules precisely because the per-instance
|
||||
record has already been tried 189 times.
|
||||
|
||||
---
|
||||
|
||||
## 5. Part 2 — audit of the tooling already configured
|
||||
|
||||
Judged against the measured classes above, per the issue's sequencing. Everything here was run, not
|
||||
assumed.
|
||||
|
||||
### 5.1 The LSPs: two of three are broken, and the one that matters most is the most broken
|
||||
|
||||
#773 asked whether routine LSP use would catch Family A (find *all* references rather than the one
|
||||
in front of you). It would help — **and it is not available.** Measured in this session:
|
||||
|
||||
| LSP | State | Evidence |
|
||||
|---|---|---|
|
||||
| `csharp-lsp` | **Broken — cannot initialize** | `findReferences` on `ChannelPlaylist.ToM3U()` → `System.InvalidOperationException: .NET SDK cannot be resolved, because libhostfxr.dylib cannot be found inside /opt/homebrew/Cellar/dotnet/10.0.302/bin/host/fxr`. That directory **does not exist** — Homebrew's dotnet layout is not what MSBuildLocator expects. `dotnet --version` works (10.0.302), so builds are fine; only the language server is dead. |
|
||||
| `typescript-lsp` | **Broken at the repo root** | `Could not find a valid TypeScript installation… ensure that the "typescript" dependency is installed in the workspace`. `typescript` lives in `web/node_modules`, not at the workspace root. Configuration problem, not a missing dependency. |
|
||||
| `pyright-lsp` | **Works** | `documentSymbol` on `scripts/decisions_lib.py` returned the full symbol tree; `findReferences` on `active_files` correctly returned 4 references across 3 files, including cross-file hits in `decisions_validate.py` and `migrate_decisions_split.py`. |
|
||||
|
||||
**Two conclusions, and the second is the sharper one.**
|
||||
|
||||
1. The LSP that would help most is the one that is dead. The residue detector A cannot reach —
|
||||
populations of *sites in code* rather than of values — is overwhelmingly C#: #403 (5 of 6
|
||||
dispatch sites) and #671 (a by-id handler covering 4 of 10 media types) are "find every site"
|
||||
problems, and `csharp-lsp` cannot answer a single query. (#510 is *not* an example here despite
|
||||
looking like one: its record names duplication as the root cause and its switch keys on a plain
|
||||
enum, so it belongs to Family C and `Enum.GetValues<T>()` is its authoritative source.)
|
||||
2. **A dispatched subagent could not reach the LSP tool at all.** The agent tasked with testing the
|
||||
three LSPs reported `ToolSearch` returning "No matching deferred tools found" for every query,
|
||||
while the same tool resolved immediately in this main session. So the standing note
|
||||
*"workflow agents must use csharp-lsp"* is doubly rotten: the server is broken **and** the agents
|
||||
it addresses cannot invoke it even when it works. This is a live instance of overclaim/stale
|
||||
claim (§2 rank 7) sitting in our own guidance.
|
||||
|
||||
**Resolved 2026-08-14 (#777), and one claim above needs correcting.** Both servers now work and both
|
||||
answer a real cross-file query in this repo; the full setup, traps and verification are in
|
||||
`docs/local-lsp-tooling.md`, and the rule is `session.local-code-intelligence`.
|
||||
|
||||
| Row above | Resolution |
|
||||
|---|---|
|
||||
| `csharp-lsp` | `MSBuildLocator` needs a dotnet root that owns `host/fxr`, which Homebrew's `bin` does not and its `libexec` does. Fixed by `env.DOTNET_ROOT` in `.claude/settings.local.json`. `findReferences` on `ChannelPlaylist.ToM3U()` returns the declaration plus its 5 call sites, excluding the mention of the name in a comment that `grep` matches. |
|
||||
| `typescript-lsp` | No configuration lever exists — v5 dropped `--tsserver-path`, and a plugin `lspServers` entry cannot pass `initializationOptions`. Fixed by making the package resolvable from the workspace root (a gitignored root `node_modules/typescript` link). Returns 20 references across 7 files for `canLeaveCurrentScreen`. |
|
||||
|
||||
**The correction to conclusion 2**, which matters because the guidance it judges is still in use: the
|
||||
note *"workflow agents must use csharp-lsp"* names the **`csharp-lsp` MCP server** tools
|
||||
(`csharp_set_workspace`, `csharp_diagnostics`, `csharp_references`, …), not the `LSP` tool. MCP tools
|
||||
**are** reachable from a subagent. The subagent measurement above is correct about the `LSP` tool and
|
||||
was generalised one step too far: the note was unsatisfiable because that MCP server's `.mcp.json`
|
||||
entry named a dotnet install that no longer existed, so it never started — not because agents cannot
|
||||
invoke it. With the entry repaired the server serves 16 tools, so the guidance becomes satisfiable —
|
||||
inferred from the MCP boundary generally (326 subagent MCP calls across four other servers), not yet
|
||||
measured for `csharp-lsp` from a subagent. What must be briefed explicitly is **which surface**: pointing a subagent at the `LSP` tool
|
||||
is still an instruction it cannot obey.
|
||||
|
||||
### 5.2 Python: the global instruction and this repo disagree, and the repo is silent
|
||||
|
||||
`~/.claude/CLAUDE.md` instructs `ruff check`, `ruff format --check`, and `pyright` after modifying
|
||||
Python. Measured against reality:
|
||||
|
||||
| Claim | Measured |
|
||||
|---|---|
|
||||
| ruff/pyright run in CI or hooks | **No.** `grep -rn "ruff\|pyright" .gitea/workflows/ .husky/` returns **nothing**. |
|
||||
| What *does* run on Python | `pr-checks.yml` → `decisions-guard` (`decisions_validate.py`, `build_decisions_catalog.py --check`) and `script-tests` (`pytest scripts/tests`). Neither Husky hook touches Python. |
|
||||
| `ruff check scripts/` | **47 errors** — but 28 are `E702` (semicolons) and **7 `S105` "hardcoded password" are false positives** on test stubs (`env["ETV_GITEA_TOKEN"] = "stub"`). The real cleanup is small. |
|
||||
| `ruff format --check scripts/` | **10 of 20 files would be reformatted.** |
|
||||
| `pyright scripts/` | **2 errors**, both `reportMissingImports` for `etv_client` in `scripts/scripted-schedules/entrypoint.py` — a package resolvable only in that script's deploy environment. Effectively clean. |
|
||||
| Repo-level ruff config | **None.** No `ruff.toml`/`pyproject.toml`. Ruff silently falls back to whichever `~/.config/ruff/ruff.toml` the operator's machine happens to have. |
|
||||
|
||||
**The last row is the finding that matters**, and it is Family E (environment divergence) in our own
|
||||
toolchain: Python lint behaviour here is a function of an un-versioned file on one laptop. A second
|
||||
machine lints differently, or not at all. One of the two must move — either the repo adopts a
|
||||
committed ruff config and enforces it, or the global instruction stops claiming this repo enforces
|
||||
something it does not.
|
||||
|
||||
**Resolved 2026-08-21 (#780): the repo moved.** `ruff.toml` is committed at the root and the
|
||||
`script-tests` job runs `ruff check` + `ruff format --check` under a pinned `ruff==0.12.11` over an
|
||||
explicit population from `git ls-files` — not `ruff check .`, which an `exclude` in the right config
|
||||
scope silently empties into a green run. The pre-fix state, reproducible rather than
|
||||
asserted — against `706674272`, the base this landed on, with the committed config dropped in:
|
||||
|
||||
```
|
||||
mkdir -p ~/scratch/m780 && git archive 706674272 | tar -x -C ~/scratch/m780
|
||||
git show 01f7a89e8:ruff.toml > ~/scratch/m780/ruff.toml # a sha: the file is not on main pre-merge
|
||||
cd ~/scratch/m780 && ruff check . ; ruff format --check . # ruff 0.12.11
|
||||
# -> Found 74 errors. / 20 files would be reformatted, 13 files already formatted
|
||||
```
|
||||
|
||||
(`;` not `&&` — `ruff check` exits 1, which would swallow the second command. Not `/tmp`: macOS purges
|
||||
it. Redirect the config into place *before* running anything: an empty `ruff.toml` is valid, so a
|
||||
failed `git show` leaves ruff silently using its own defaults and printing a different number.)
|
||||
|
||||
The row above measured **47** eight days earlier against the operator's global config; the tree grew
|
||||
and the configs differ, so the two numbers are not comparable and neither supersedes the other. Two
|
||||
of the 74 are `RUF100` on suppressions that were already in the tree before this change — they exist
|
||||
in this count only because the committed config enables that rule.
|
||||
|
||||
Of the 74, **57 were fixed in code** (most of them by the `ruff format` pass itself, which splits the
|
||||
40 semicolon statements) and **17 carry a per-site `# noqa` with its reason inline**: 8 `S105` on stub
|
||||
credentials handed to the real hooks by `scripts/tests`, 9 `E501` on one-line JSON and shell fixtures.
|
||||
The `S105`s are deliberately per-site rather than a directory exemption, so a real credential pasted
|
||||
into a fixture later still reddens the gate. Only `S101` is exempted directory-wide for `scripts/tests/**`,
|
||||
because a test suite asserts.
|
||||
|
||||
`RUF100` is selected, which is what keeps that split honest: a `# noqa` that suppresses nothing still
|
||||
reads as a suppression, and it is invisible without this rule. Three were live when it was switched
|
||||
on: one on a site that had already been fixed in code, plus the two counted above — one whose rule had
|
||||
stopped firing, one for a rule this config never enables.
|
||||
|
||||
`pyright` stayed ungated: its only findings are the `etv_client` imports in the row above, and gating
|
||||
it would put a node toolchain on the git-only `small` lane to find nothing. Rationale, the exemption
|
||||
list and the measured exclude matrix: `ci.python-lint-ruff-config-committed`.
|
||||
|
||||
### 5.3 Configured vs actually invoked
|
||||
|
||||
Measured over the session transcript corpus (811 files under
|
||||
`~/.claude/projects/-Users-timothy-ersatztv/`, this session excluded).
|
||||
|
||||
| Tool | Configured | Actually invoked | Verdict |
|
||||
|---|---|---|---|
|
||||
| `gitea` MCP | project `.mcp.json` **and** user scope (duplicate, identical values) | Heavy, same-day | Keep; de-duplicate the config |
|
||||
| `ssh-mcp` | project `.mcp.json` | 16 transcripts, same-day | Keep |
|
||||
| `mempalace` MCP | user scope | 10 transcripts, **last hit 2026-07-25** (19 days) | Keep, but see below |
|
||||
| **`LSP` (all three servers)** | 3 plugins enabled + `csharp-lsp` in `enabledMcpjsonServers` | **0 calls in 811 transcripts** | Broken *and* unused — both C# and TS servers repaired 2026-08-14 (#777); the zero is the baseline a future measurement is compared against |
|
||||
| `context7` | enabled, `CONTEXT7_API_KEY` set | 0 calls | Dead |
|
||||
| `playwright` | enabled | 145 calls, last 2026-07-25 | Keep |
|
||||
| `superpowers` | enabled | 29 transcripts, last 2026-08-04 | Keep |
|
||||
| `feature-dev`, `ralph-loop`, `security-guidance` | enabled | 0 skill invocations | Dead as configured |
|
||||
| `codex` plugin | enabled | 0 *skill* invocations — but the documented workflow is `codex exec` via Bash, which this search cannot see | Not dead; measured the wrong surface |
|
||||
| `nuget`, `docker-mcp` | in `.mcp.json`, explicitly **disabled** | 0 (expected) | Correctly off |
|
||||
|
||||
The zero for `LSP` is verified rather than assumed: the same query shape returns 23,661 `Bash` calls
|
||||
and 5,102 `Read` calls over the same corpus, so the search was demonstrably capable of finding hits.
|
||||
**Re-derive it with that positive control**, not on its own — `rg -c` over multiple files prints
|
||||
`path:count`, so the obvious way to total it sums the paths and returns zero for everything.
|
||||
|
||||
**Re-measured 2026-08-14 (#781), and the table above needs two corrections plus seven additions.**
|
||||
The population here was assembled **by hand**; deriving it from the config files instead
|
||||
(`~/.claude/settings.json` `enabledPlugins`, `~/.claude.json` `mcpServers`, the project `.mcp.json`,
|
||||
`.claude/settings.local.json`) surfaces **seven enabled plugins the table omits** — `serena`,
|
||||
`code-review`, `code-simplifier`, `commit-commands`, `claude-md-management`, `skill-creator`,
|
||||
`frontend-design`. An audit whose completeness rests on remembering every entry is exactly the
|
||||
failure `testing.guard-derives-population-from-source` exists to prevent, applied to an audit.
|
||||
|
||||
The corpus is also larger than counted: **964** transcripts via `rglob`, not 811 — a top-level-only
|
||||
glob sees 209 here and manufactures false zeros. Positive control on the same query: 24,762 `Bash`,
|
||||
5,275 `Read`, 4,037 `Edit`.
|
||||
|
||||
| Row above | Re-measured |
|
||||
|---|---|
|
||||
| `mempalace` "last hit 2026-07-25 (19 days)" | **31 calls, last seen 2026-08-14.** In active use; the gap was a snapshot artifact, not disuse. |
|
||||
| `codex` "0 skill invocations" | **113 `codex exec` calls across 17 sessions** — the third-heaviest tool in the corpus. The caveat was right; this is the number. |
|
||||
| `LSP` "0 calls" | Not zero. The handful of calls are from sessions *testing whether the LSP works* — including §5.1's own probes. |
|
||||
| `gitea` "duplicate, identical values" | Duplicated but **not identical** — the user-scope entry carries `"type": "stdio"`. **Resolved:** the *project* copy was removed, not the user one, because `server-management` and `homelab-docs` have no `.mcp.json` and depend on user scope. |
|
||||
| *(absent)* `serena` | Enabled; starts and initializes standalone (Serena 1.28.1) but its tools never appear in a session and `ToolSearch` cannot resolve them. Cause unproven → **#799**. |
|
||||
| *(absent)* six further plugins | Three (`claude-md-management`, `skill-creator`, `frontend-design`) were enabled **on 2026-08-14**, so their zeros mean "not yet tried". The rest are reachable and simply unused. |
|
||||
|
||||
**The issue's framing does not survive the measurement, and that is the finding.** "Retire what is
|
||||
enabled and never invoked" treats a zero as evidence of uselessness. These zeros split **four** ways:
|
||||
**broken** (`csharp-lsp` MCP and two LSP servers — #777), **unreachable** (`serena`), **just enabled**
|
||||
(three plugins), and **measured on the wrong surface** (`codex`). Establishing *why* a counter is zero
|
||||
is a precondition for acting on it — and on this evidence almost nothing should be removed. What was
|
||||
actually wrong was **duplication** and **breakage**, not excess. The one entry still reading as
|
||||
coverage it does not provide is `nuget`, which names the same dead `/usr/local/share/dotnet` path
|
||||
#777 fixed elsewhere; it is inert only because it is explicitly disabled.
|
||||
|
||||
**Hooks are the good news.** All 13 scripts in `.claude/hooks/` are wired from either
|
||||
`.claude/settings.json` or `.husky/*`, and no settings entry points at a missing path — there are
|
||||
**no dead hook scripts**, contrary to the issue's suspicion. Husky hooks have genuine fired-output
|
||||
evidence in transcripts (`husky - dotnet format found whitespace/BOM issues`, `husky - commit message
|
||||
missing Co-Authored-By trailer`, `husky - refusing to commit root-level screenshot(s)`).
|
||||
|
||||
Wiring was the strongest claim available for the hook rows when this table was written. It is no
|
||||
longer the ceiling — §5.4 is now measured — so read the hook rows against
|
||||
`scripts/hook-fire-log.sh report`, not against this paragraph.
|
||||
|
||||
### 5.4 Whether our own hooks fire — now measured (#776 closed this)
|
||||
|
||||
**The finding as originally recorded.** In this harness version only `Stop` hooks emit a structured
|
||||
record (`stop_hook_summary`/`hookInfos`). `PreToolUse` and `PostToolUse` hooks — which is *every*
|
||||
guard that matters here: merge consent, worktree, BOM, agent model/RAM — leave no durable execution
|
||||
trace. What this audit could count for those hooks was filename mentions in settings dumps, which is
|
||||
not evidence of execution. So the guards this repo relies on most were exactly the ones whose
|
||||
execution could only be inferred: Family B (the check never ran) applied to the hook layer itself,
|
||||
after this project had already paid for it twice at the CI layer (#751, #756).
|
||||
|
||||
**What replaced the inference.** Every hook now records its own execution through one shared sink,
|
||||
`scripts/hook-fire-log.sh` (`testing.hook-reports-its-own-execution`): a `fire` record on entry and
|
||||
an `exit` record carrying the status and the decision, where **the decision is parsed from the bytes
|
||||
the hook actually emitted** rather than declared by its author. Read it with
|
||||
`scripts/hook-fire-log.sh report [--all]`.
|
||||
|
||||
**Measured 2026-08-14** — a scripted headless session (a plain Bash call, a Bash call carrying
|
||||
`ETV_UPDATE_GOLDENS=1`, a `Write` to a SPA file, and one `Agent` dispatch) plus a real `git commit`
|
||||
and a `git push --dry-run` in a worktree. Snapshot boundary: this is one deliberately-constructed
|
||||
window, not a corpus statistic.
|
||||
|
||||
| Hook | Event | Fires | Decisions observed |
|
||||
|---|---|---|---|
|
||||
| `pretooluse-bash-guard` | PreToolUse/Bash | 2 | `deny` 1, `no-op` 1 |
|
||||
| `pretooluse-bom-guard` | PreToolUse/Bash | 2 | `no-op` 2 |
|
||||
| `pretooluse-worktree-guard` | PreToolUse/Bash | 2 | `no-op` 2 |
|
||||
| `posttooluse-worktree-marker` | PostToolUse/Bash | 1 | `no-op` 1 |
|
||||
| `design-sync-reminder` | PreToolUse/Write + Stop | 4 | `context` 1, `block` 1, `no-op` 2 |
|
||||
| `pretooluse-agent-model` | PreToolUse/Agent | 2 | `no-op` 2 |
|
||||
| `pretooluse-agent-ram` | PreToolUse/Agent | 2 | `no-op` 2 |
|
||||
| `decisions-guard` | git pre-commit | 2 | `pass` 2 |
|
||||
| `prepush-clean-worktree-check` | git pre-push | 1 | `pass` 1 |
|
||||
| `prepush-donewhen` | git pre-push | 1 | `pass` 1 |
|
||||
| `prepush-rebase-check` | git pre-push | 1 | `pass` 1 |
|
||||
| `pretooluse-nav-guard` | PreToolUse/navigate | **0** | not exercised — needs a live browser session |
|
||||
| `pretooluse-merge-consent` | PreToolUse/PR write | **0** | not exercised — needs a real merge attempt |
|
||||
|
||||
**11 of 13 hooks are confirmed firing, with the decision each reached.** The `deny` row is the load-
|
||||
bearing one: `pretooluse-bash-guard` did not merely run, it *blocked* the `ETV_UPDATE_GOLDENS=1`
|
||||
probe, so at least one guard in this set is demonstrably live rather than merely present.
|
||||
|
||||
**Re-verified against the shipped implementation.** The table was first measured against an early
|
||||
version of the sink, and the classifier changed materially afterwards, so the run was repeated
|
||||
against the final code: `pretooluse-bash-guard` `deny`+`no-op`, `pretooluse-bom-guard` and
|
||||
`pretooluse-worktree-guard` `no-op`, `posttooluse-worktree-marker` `no-op`, `design-sync-reminder`
|
||||
`context`+`block`+`no-op` — identical decisions. The repeat run exercised the five Claude hooks a
|
||||
Bash/Write session reaches; the `Agent` pair and the four git hooks are carried over from the
|
||||
original run and were not re-measured.
|
||||
|
||||
**Reproducing this table.** It is a constructed window, not a corpus statistic, and it is not
|
||||
reproducible from a reader's own `report` output — running the test suite alone would not produce
|
||||
it, and before `scripts/tests/conftest.py` landed, running the suite actively polluted the default
|
||||
log with synthetic fires. To re-derive: point `ETV_HOOK_FIRE_LOG_DIR` at an empty directory, run a
|
||||
headless session exercising the four tool paths above, then a `git commit` and a
|
||||
`git push --dry-run`, and read `scripts/hook-fire-log.sh report --all --dir <that directory>`.
|
||||
|
||||
**A zero is two findings wearing one number** — a hook that is broken, and a hook whose trigger did
|
||||
not occur — and the report says so rather than presenting a zero as a verdict.
|
||||
|
||||
For the two zeroes here, what is established and what is not, kept apart deliberately.
|
||||
**Established:** both script bodies work. `pretooluse-nav-guard` and `pretooluse-merge-consent` are
|
||||
each driven to their deciding branch in `scripts/tests/test_hook_fire_log.py` — a `deny` on an
|
||||
`/iptv/` URL and an `ask` on a merge call — emitting the correct decision with the instrumentation
|
||||
in place. **Not established:** that the harness would dispatch to them. Those tests invoke the
|
||||
scripts directly, so they bypass registration and matcher dispatch entirely; a typo in
|
||||
`.claude/settings.json`, a settings file that never loaded, or a matcher that does not match would
|
||||
leave both tests green while the production zero still meant broken wiring. A working script is a
|
||||
necessary condition, not the finding.
|
||||
|
||||
So these two zeroes remain genuinely ambiguous, and neither can be resolved without the thing that
|
||||
resolves it: a live browser session, or a real merge attempt. Manufacturing a merge to observe the
|
||||
merge guard is a worse idea than the gap it would close. Distinguishing the two meanings of zero
|
||||
stays a human judgement — `testing.hook-reports-its-own-execution` says so, and an earlier draft of
|
||||
this paragraph quietly contradicted it by treating "the script works" as "the wiring works".
|
||||
|
||||
**The measurement earned its keep on its first run.** `pretooluse-agent-model` was recorded deciding
|
||||
`no-op` on an `Agent` dispatch where `ask` was expected. Previously that would have been an
|
||||
unanswerable suspicion about a guard nobody could observe; instead the real payload was dumped and
|
||||
it carried `model: haiku`, so `no-op` was correct and there was no defect. The general form is the
|
||||
standing #756 lesson — **make the system report it rather than infer it** — and note what it
|
||||
displaced: an argument from the hook's source about what it "must" do, which is the reasoning shape
|
||||
§3 measures going wrong repeatedly.
|
||||
|
||||
### 5.5 Tools we do *not* have that would address a named class
|
||||
|
||||
Each is justified against specific issues, not general merit; anything that could not be tied to
|
||||
recorded instances is left out rather than padded in.
|
||||
|
||||
| Tool | Class it addresses | Justification |
|
||||
|---|---|---|
|
||||
| **Stryker.NET** (mutation testing for C#) | Family B — detector B, mechanised | Detector B currently relies on an author remembering to write a mutation proof. Stryker generates them. #621 (a guard whose call site could be deleted with the suite green), #685 (two guards where deleting either left the suite green) and #719 (logic never wired to stdin) are all surviving-mutant detections by construction. Cost is real (mutation runs are slow), so scope it to the guard/validator projects rather than the whole solution. |
|
||||
| ~~`shellcheck`~~ — **tested and rejected** | (would have been Family B, shell half) | The obvious candidate for #445 (`if ! cmd; then status=$?`, where bash sets `$?` to the *logical negation* so a failing run exited 0). **Measured: it does not catch it.** ShellCheck 0.11.0 on that exact construct reports nothing, and even `-o all` returns only an unrelated brace-style nit; run against our 13 hook scripts it yields 2 `SC2034` unused-variable warnings. Recorded here as a negative result so it is not proposed again on plausibility. |
|
||||
| **A duplicate-code detector** (`jscpd` or equivalent) | Family C | #440 shipped two byte-identical Lucene escapers in different files; #711 and #649 are the same shape at the config/CI layer. A duplication report would have surfaced all three at authoring time — subject to #711's exception, so it should advise, not block. |
|
||||
| **An enum-exhaustiveness analyzer rule** for C# `switch` *statements* | Family A, code-structural residue | #503's catch-all `default` silently absorbed an unhandled enum value and produced byte-identical output to a handled one. Switch *expressions* already warn; statements do not. This is a rule to enable in the analyzers we already run (Meziantou), not a new dependency — the cheapest item here. |
|
||||
|
||||
**Explicitly not proposed:** a "brief linter" for omitted delegation constraints (#586) and any
|
||||
overclaim detector. §4 argues both are implausible, and inventing them here to look thorough would
|
||||
be the symptom-keyed mistake. Nothing in the corpus suggests a tool we lack would beat detector A
|
||||
for Family A — the four in-repo reinventions show the convention works when applied; the failure is
|
||||
that it was never written down once.
|
||||
|
||||
**Evidence status of these rows, which differs and matters.** Only the shellcheck row was
|
||||
*executed* — and running it refuted the reason it had been proposed, which is why it is struck
|
||||
through. The three surviving rows are **reasoned from precedent**: each names issues whose recorded
|
||||
mechanism the tool addresses, but none has been run against this repo to confirm it fires. That is a
|
||||
weaker warrant and should be discharged before any of them is adopted — a tool justified by "it
|
||||
plausibly catches this class" is the same species of claim as the guards this document criticises.
|
||||
Treat these three as candidates to test, not findings.
|
||||
|
||||
**On MemPalace**, the 19-day gap deserves care rather than a verdict: `CLAUDE.md` makes it the
|
||||
documented discovery path for decisions, so either sessions are skipping the documented path, or
|
||||
they are correctly reading `docs/decisions/README.md` directly (which the same contract permits, and
|
||||
which is authoritative). The measurement cannot distinguish those, and I am not going to guess.
|
||||
|
||||
---
|
||||
|
||||
## 6. Follow-ups
|
||||
|
||||
This issue is analysis; it spawns implementation rather than doing it.
|
||||
|
||||
| Issue | Detector / finding | Covers | Priority |
|
||||
|---|---|---|---|
|
||||
| #774 | **A** — a guard derives its population from the authoritative source and asserts set equality | Family A, 27 records (39%) | high |
|
||||
| #775 | **B** — every guard ships a mutation proof: delete that guard alone, see red | Family B, 9 records (13%) | high |
|
||||
| #776 | Make hooks report that they fired — PreToolUse/PostToolUse execution is unobservable | the whole hook layer | high — **DONE**, §5.4 is measured |
|
||||
| #777 | `csharp-lsp` and `typescript-lsp` are broken; the "workflow agents must use csharp-lsp" note is stale | partial mitigation for Family A | medium |
|
||||
| #778 | **D** — pin a version or use compare-and-set for read-then-write against live remote state | Family D, 5 records | medium |
|
||||
| #779 | **F + G** — test the deny path with the production config value; assert full-replace field lists | fail-open + destructive-replace, ~8 records | medium |
|
||||
| #780 | Reconcile Python tooling — commit a ruff config and enforce it, or stop claiming we do | environment divergence in our own toolchain | medium |
|
||||
| #781 | Retire enabled-but-never-invoked plugins/MCP servers; de-duplicate the `gitea` server | config that reads as coverage | low |
|
||||
|
||||
**Detector C (dedup by construction) has no issue of its own on purpose.** It is not a thing to build;
|
||||
it is the shape the fixes in #774 and #778 should take when they find two copies of one rule — with
|
||||
#711's exception carried along, since that record deliberately keeps its enumeration and is a
|
||||
counterexample rather than a supporting case.
|
||||
|
||||
Not filed, deliberately: overclaim/stale-claim, per-task review blindness, and omitted brief
|
||||
constraints. §4 argues no plausible mechanical detector exists for these, and filing an issue for
|
||||
each would produce exactly the weak enumerating guard this analysis recommends against.
|
||||
@@ -0,0 +1,420 @@
|
||||
# Guard inventory (ersatztv#774 / #775)
|
||||
|
||||
Every executable guard **file** in this repo, what it blocks, and whether it ships a proof it can go
|
||||
red. `scripts/tests/test_guard_inventory.py` derives the population from the **git index** and the
|
||||
workflow/hook call sites and asserts **set equality** against the `Guard` column, so a new guard
|
||||
cannot be added without acquiring a row here, and a row cannot name a proof that does not exist.
|
||||
The index rather than a filesystem walk since ersatztv#806 — the practical consequence is that a new
|
||||
guard joins the population when it is **staged**, not when the file appears. Nothing local runs
|
||||
these checks at all: `.husky/pre-commit` runs lint-staged, the decisions guard, the root-PNG check
|
||||
and `dotnet format`, and `grep -rn pytest .husky/` returns nothing. The runner is
|
||||
`pr-checks.yml::script-tests`, `on: pull_request`, so the red arrives in CI — plus
|
||||
`docker-build.yml:723` on the release path, which re-runs two of these files
|
||||
(`test_ci_dropped_step_guard.py` and `test_ci_release_path_scan_job.py`) as a `needs:` of `build`.
|
||||
|
||||
**Read `docs/decisions/records/testing/guard-derives-population-from-source.md` and
|
||||
`…/guard-ships-with-mutation-proof.md` before editing a guard or adding a row.**
|
||||
|
||||
## Columns
|
||||
|
||||
- **Kind** — `GUARD` (it can block a commit, a push, a tool call, a merge or a CI job), `TOOLING`
|
||||
(it does work but asserts nothing; listed so its absence from the guard set is a recorded
|
||||
decision, not an oversight), or `PROOF` (a `scripts/tests/` file whose job is to prove another
|
||||
guard). `PROOF` exists to stop a regress: once test files entered the population, every mutation
|
||||
proof became a row wanting a proof of its own. A `scripts/tests/` file that enforces a repo
|
||||
invariant with no separate guard behind it is a `GUARD`, graded normally, and may cite a mutation
|
||||
case in its own file.
|
||||
- **Proof** — `MUTATION`: a clause-level mutation **was executed** and this named test **was
|
||||
witnessed red**. The test either performs the disarm itself (a monkeypatch, a deselection, a
|
||||
removed marker) or a recorded incident shows it going red when the clause went away.
|
||||
`BEHAVIOUR-ONLY`: a real test drives the guard through its real entry point with good and bad
|
||||
input, but no one has demonstrated that removing the clause reddens it. `NONE`.
|
||||
- **Proof ref** — `file.py::function`, verified to exist by the inventory test.
|
||||
|
||||
The distinction between `MUTATION` and `BEHAVIOUR-ONLY` is the whole point of #775 and is not a
|
||||
grading curve. A behavioural test proves the guard reacts to its input; only a mutation proves the
|
||||
guard is *load-bearing*. #685 shipped two guards on one condition where deleting either left the
|
||||
suite green, and every behavioural test passed throughout.
|
||||
|
||||
**The definition above is the second one.** The first read "a named test disarms this guard's clause
|
||||
and asserts red", and three rows were graded `MUTATION` against it that do not disarm anything —
|
||||
they feed the real script an input only that clause rejects (empty stdin, a short page, a full first
|
||||
page). Cold review caught it, and the objection is right twice over: those tests are the same
|
||||
species as ones graded `BEHAVIOUR-ONLY` eight rows away, so the column was being applied as a curve
|
||||
on the very day it was introduced. They are regraded. The surviving criterion is *witnessed*, not
|
||||
*plausible* — "removing this clause would surely redden that test" is an argument, and this table
|
||||
exists because arguments of that shape have been wrong here six times.
|
||||
|
||||
## Scope limit, stated rather than implied
|
||||
|
||||
This inventory covers guard **files**, discovered by reading `.claude/hooks/*.sh`, `.husky/*` and
|
||||
`scripts/tests/test_*.py` **out of the git index**, plus every `scripts/…` path referenced by a
|
||||
workflow or a hook. The classes below are outside that population. They are listed because the first
|
||||
version of this section named only the first one, and cold review found that the very guards this
|
||||
inventory shipped with were sitting in the gap:
|
||||
|
||||
1. **Guards inline in workflow YAML** — most importantly `pr-checks.yml:ci-image-pin`. "Which jobs
|
||||
are guards" needs a judgement call per job the filesystem cannot supply. Two were audited under
|
||||
#774 and one fixed; extending the population is tracked in #786.
|
||||
2. **C# and TypeScript guards** — `ErsatzTV.Mcp.Tests/ToolCatalogTests.cs` and
|
||||
`web/src/api/pageSizeCallSites.guard.test.ts` are both structural guards and neither has a row.
|
||||
3. **Mentions counted as call sites.** The `scripts/…` scrape matches any occurrence, including
|
||||
inside a comment or an `::error::` string. `scripts/update-openapi.sh` is named in a
|
||||
`pr-checks.yml` error message, so removing the step that runs it would leave its row intact.
|
||||
4. **Nested and non-lowercase paths** beyond `scripts/tests/` — a guard under
|
||||
`scripts/scripted-schedules/`, or with an uppercase name, is invisible to the scrape.
|
||||
5. **Non-`.sh` hooks** — the hook pattern is `*.sh` only. (`.yaml` workflows are no longer in this
|
||||
gap: the caller scan matches `*.yml` and `*.yaml` since #806, because Gitea accepts both.)
|
||||
6. **Transitive calls** — a script invoked only by another script, rather than by a workflow or
|
||||
hook, is not discovered.
|
||||
7. **Non-`test_` modules under `scripts/tests/`** — the pattern is `test_*.py`, so `conftest.py`,
|
||||
`mutation_harness_lib.py`, `mutation_manifest.py` and `tracked_files.py` are outside the
|
||||
population and hold no rows. They are not guards (they assert nothing on their own), but the
|
||||
middle two ARE what `test_mutation_harness.py` is made of, so gutting either would take that
|
||||
guard with it. What catches that is the guard's own row: its declared mutation targets
|
||||
`mutation_harness_lib.py`, and its proof test refuses to run if the clause it names has moved.
|
||||
`tracked_files.py` has no such backstop — it is load-bearing for every module that imports it
|
||||
(#806), and an edit to it is covered only by those modules' own proofs. Recorded rather than
|
||||
force-fitted: a row for a library would need a `Kind` the vocabulary does not have.
|
||||
8. **Nested workflow directories** — the workflow scope is direct children of `.gitea/workflows`, so
|
||||
a tracked `.gitea/workflows/nested/x.yml` is invisible to the caller scan and to
|
||||
`test_ci_image_pin_population.py`. Left as scope rather than widened: whether Gitea executes
|
||||
nested workflow files was not verified here, and widening on an unverified premise risks a
|
||||
permanent red on a correct tree, which is how a correct guard gets deleted.
|
||||
|
||||
Hook **wiring** is checked (`test_every_hook_file_is_actually_WIRED` reads `.claude/settings.json`
|
||||
and the husky hooks with full-line comments stripped), so a hook file whose registration is deleted
|
||||
fails rather than keeping a row that reads as coverage. Its limit, stated because the check reads
|
||||
stronger than it is: **it is a substring test for the basename, not a parse of the invocation.**
|
||||
`: # .claude/hooks/decisions-guard.sh disabled` still reads as wired, and conversely a hook invoked
|
||||
through a wrapper or a constructed path reads as unwired. It catches deletion, which is the common
|
||||
case; it does not catch deliberate disablement. The check does not extend to the `scripts/` half at
|
||||
all.
|
||||
|
||||
## File populations and where they come from (ersatztv#806)
|
||||
|
||||
Every guard here whose members are FILES derives them from the **git index**, never a filesystem
|
||||
walk. The disk is not an authoritative source: it reports build output and editor droppings and
|
||||
differs per machine, so a guard derived from it asserts a different population in CI than on the
|
||||
laptop of the person it is meant to stop. `scripts/tests/tracked_files.py` is the single derivation
|
||||
and carries the full rationale; `scripts/tests/test_guard_populations_derive_from_git.py` proves it,
|
||||
in both directions: removing EVERY member of each registered derivation from the index one at a time
|
||||
and requiring it to disappear while still on disk, and watching for a directory LISTING issued while
|
||||
the derivation runs (reading files stays allowed). Removal alone is blind to a source that
|
||||
contributes only untracked members — an `rglob` reaching `.husky/_/` adds and never removes — and
|
||||
any check phrased as "an untracked file must not enter" is itself machine-dependent, because the
|
||||
untracked file has to exist. Watching for the call needs no arranged state.
|
||||
|
||||
That second check is a regression guard against the accidental shapes, not a boundary: what is
|
||||
observed is any call that goes **through one of the spies**, whenever it happens — the spy records
|
||||
into a list that outlives the patch, so a reference captured during the window and invoked after it
|
||||
still counts. Whether the call goes through a spy is what decides, not when. The instance list — what
|
||||
never reaches a spy at all — lives in the check's own docstring and is deliberately not restated
|
||||
here, because a second copy of it drifted from the first within one commit.
|
||||
|
||||
One deliberate exception to "from the index" sits in the same file: the registration check lists
|
||||
`scripts/tests/test_*.py` from disk on purpose, because it is a superset check over what pytest
|
||||
collects — an untracked stray there makes it MORE demanding, never blind, whereas using the index
|
||||
would let an unstaged new guard escape registration.
|
||||
|
||||
Its own limit, stated because a check described as complete stops being re-examined: it finds
|
||||
derivations by PARSING each `test_*.py` for an import of the shared helper, so a module that derives
|
||||
a file population some other way is invisible to it, and no mechanical check can close that (#774
|
||||
reached the same conclusion about detecting filter-shaped guards by token).
|
||||
|
||||
The audit #806 asked for, recorded whichever way it came out, because "we looked and left it" and
|
||||
"we never looked" are indistinguishable a year later:
|
||||
|
||||
| Guard | Population | Completeness claim over tracked files? | Outcome |
|
||||
| --- | --- | --- | --- |
|
||||
| `test_guard_inventory.py` | `.claude/hooks/*.sh`, `.husky/*`, `scripts/tests/test_*.py`, workflow/hook callers | **yes** — set equality against this table | converted to the index; `.husky/_/` had been excluded only because `_` is a directory, so the obvious "make it recursive" edit would have reintroduced #778's defect here |
|
||||
| `test_hook_fire_log.py` | `.claude/hooks/*.sh` | **yes** — every hook must be instrumented | converted; an untracked scratch `.sh` used to demand instrumentation and redden the suite on that checkout alone |
|
||||
| `test_ci_image_pin_population.py` | `.gitea/workflows/*.yml` + `*.yaml` | **yes** — "docker-build is the ONLY workflow pinning the toolchain image" | converted, and `*.yaml` added: Gitea accepts both spellings, so a `.yaml` workflow was structurally invisible while the test read as covering all of them |
|
||||
| `test_pr_changed_files.py` | `.gitea/workflows/*.y*ml` | **yes** — "no OTHER workflow writes the review-verdict status" | converted. Not on #806's list and found by cold review on the pushed head: an untracked `.yaml` dropped in `.gitea/workflows/` reddened two guards while absent from the index — the issue's list of files to assess was a starting point, not the population |
|
||||
| `web/src/api/pageSizeCallSites.guard.test.ts` | `web/src/**/*.{ts,tsx,mts,cts}` via `import.meta.glob` | **yes** — an unregistered discovered site fails | DEFERRED to #819, assessed not skipped. The glob is a documented workaround: `@types/node` is deliberately out of `tsconfig.app.json`, and wiring it in was tried and reverted (it leaked Node's `setTimeout` into the app project and broke three unrelated tests), so there is no `node:child_process` to reach the index from. Over-enumerates, so it fails loudly rather than going blind |
|
||||
| `test_ci_release_path_scan_job.py` | `.gitea/workflows/*.y*ml` + `scripts/**` | **no** — a fixture assembling a tmp harness, asserted about behaviour not membership | takes its file LIST from the index anyway, for hermeticity not completeness: `shutil.copytree` copied whatever was on disk, including untracked files and `scripts/__pycache__`, into a tree whose behaviour the probes then measure. Content still comes from the working tree. The copy is not a git repo, so the two files this step RUNS may not use the helper — see the fixture docstring |
|
||||
| `test_ci_dropped_step_guard.py` | the parsed workflow document | **no filesystem population at all** | unchanged; its residual is `MARKED_JOBS`, a SCOPE mirror of the required contexts on `main`, which #806 does not close |
|
||||
|
||||
`test_remote_state_inventory.py` fixed its own population under #778 and kept a private copy of the
|
||||
derivation; #806 folded it onto the shared one, so that module is covered by the proof above like
|
||||
the rest. Across the whole change, every module that derived a file population its own way now goes
|
||||
through `tracked_files.py` — one implementation of the rule instead of one per module. The
|
||||
registered derivations are listed in `DERIVATIONS` in
|
||||
`scripts/tests/test_guard_populations_derive_from_git.py`; this page deliberately keeps no count
|
||||
of them.
|
||||
|
||||
**Still on filesystem walks, deliberately out of scope:** the decisions corpus
|
||||
(`scripts/decisions_lib.py`'s `active_files()`, and the suites over it). Its members are `docs/`
|
||||
Markdown with no generated-file pressure and a different lifecycle, and folding it in here would
|
||||
have been the reflex this milestone argues against — a change with no defect behind it. It is
|
||||
recorded as unexamined rather than as cleared.
|
||||
|
||||
## Inventory
|
||||
|
||||
| Guard | Blocks | Kind | Proof | Proof ref |
|
||||
| --- | --- | --- | --- | --- |
|
||||
| `.claude/hooks/decisions-guard.sh` | a commit | GUARD | NONE | — |
|
||||
| `.claude/hooks/design-sync-reminder.sh` | the first Stop after a UI change (one-shot, then allows) | GUARD | NONE | — |
|
||||
| `.claude/hooks/posttooluse-worktree-marker.sh` | nothing (writes the marker the worktree guard reads) | GUARD | MUTATION | `test_worktree_ownership_guard.py::test_MUTATION_a_marker_hook_that_stops_WRITING_makes_the_guard_go_quiet` |
|
||||
| `.claude/hooks/prepush-clean-worktree-check.sh` | a push with uncommitted changes in the pushed set | GUARD | NONE | — |
|
||||
| `.claude/hooks/prepush-donewhen.sh` | a direct push to `main` with unticked Done-when boxes | GUARD | NONE | — |
|
||||
| `.claude/hooks/prepush-rebase-check.sh` | a push from a branch behind `origin/main` | GUARD | BEHAVIOUR-ONLY | `test_prepush_rebase_check_tag_exemption.py::test_zero_ref_lines_does_not_exempt` |
|
||||
| `.claude/hooks/pretooluse-agent-model.sh` | an Agent dispatch naming no model (asks) | GUARD | NONE | — |
|
||||
| `.claude/hooks/pretooluse-agent-ram.sh` | an Agent dispatch under 10% free RAM | GUARD | NONE | — |
|
||||
| `.claude/hooks/pretooluse-bash-guard.sh` | a Bash call setting `ETV_UPDATE_GOLDENS` | GUARD | NONE | — |
|
||||
| `.claude/hooks/pretooluse-bom-guard.sh` | a commit/push carrying a BOM in a touched `.cs` | GUARD | MUTATION | `test_bom_guard_detection.py::test_DISARMING_the_BOM_comparison_stops_detection` |
|
||||
| `.claude/hooks/pretooluse-merge-consent.sh` | a PR merge without derived consent | GUARD | BEHAVIOUR-ONLY | `test_merge_consent_exemption.py::test_protected_path_on_a_LATER_page_is_still_seen` |
|
||||
| `.claude/hooks/pretooluse-nav-guard.sh` | a browser navigate to a streaming URL | GUARD | NONE | — |
|
||||
| `.claude/hooks/pretooluse-worktree-guard.sh` | a commit/merge in a foreign worktree | GUARD | MUTATION | `test_worktree_ownership_guard.py::test_MUTATION_disarming_the_guards_MARKER_READ_stops_the_deny` |
|
||||
| `.husky/commit-msg` | a commit with no `Co-Authored-By` trailer | GUARD | NONE | — |
|
||||
| `.husky/pre-commit` | a commit failing lint-staged, decisions, root-PNG or format | GUARD | NONE | — |
|
||||
| `.husky/pre-push` | a push failing any pre-push hook or the SPA gate | GUARD | MUTATION | `test_prepush_unsets_git_env.py::test_MUTATION_DELETING_the_unset_lets_drift_through_silently` |
|
||||
| `scripts/build_decisions_catalog.py` | the `decisions-guard` job, on a stale catalog | GUARD | MUTATION | `test_build_catalog_check_path.py::test_MUTATION_disarming_the_stale_comparison_stops_detection` |
|
||||
| `scripts/check-doc-narrative.py` | nothing, by design (advisory `::warning::` only, exits 0 on every path (the error/degradation arms are defensive and unproven — see the record) — `docs.no-session-narrative` says a string predicate over prose may not be load-bearing) | TOOLING | NONE | — |
|
||||
| `scripts/check-kickoff-guard.sh` | the `decisions-guard` job, on a revived #237 reference | GUARD | NONE | — |
|
||||
| `scripts/check-review-verdict.sh` | the merge-consent hook's verdict classification | GUARD | BEHAVIOUR-ONLY | `test_check_review_verdict.py::test_falseopen_token_must_be_a_whole_word` |
|
||||
| `scripts/ci-detect-already-validated.sh` | nothing directly (feeds the skip gate) | GUARD | NONE | — |
|
||||
| `scripts/ci-detect-docs-only.sh` | nothing directly (feeds the skip gate) | GUARD | NONE | — |
|
||||
| `scripts/ci-peak-anon.sh` | nothing (samples container memory) | TOOLING | NONE | — |
|
||||
| `scripts/ci-prove-ban-detects.sh` | the release path, if the delimiter ban is disarmed | GUARD | NONE | — |
|
||||
| `scripts/ci-step-ran.sh` | the two required contexts, on a dropped step | GUARD | MUTATION | `test_ci_dropped_step_guard.py::test_dropping_ANY_single_step_FAILS_the_guard` |
|
||||
| `scripts/decisions_validate.py` | the `decisions-guard` job, on a lifecycle fault | GUARD | MUTATION | `test_decisions_validate.py::test_main_actually_CALLS_the_wing_scan` |
|
||||
| `scripts/e2e-functional.sh` | the Functional E2E job, on a failed HTTP contract assertion | GUARD | NONE | — |
|
||||
| `scripts/e2e-local.sh` | nothing (boots a local instance) | TOOLING | NONE | — |
|
||||
| `scripts/e2e-ui.sh` | nothing (drives the Playwright flows) | TOOLING | NONE | — |
|
||||
| `scripts/hook-fire-log.sh` | nothing (records that each hook fired, and reports it) | TOOLING | NONE | — |
|
||||
| `scripts/jq-preflight.sh` | the `script-tests` job, on a jq version change | GUARD | BEHAVIOUR-ONLY | `test_jq_preflight.py::test_below_the_floor_is_LOUD` |
|
||||
| `scripts/post-review-verdict.sh` | nothing (writes the verdict status) | GUARD | BEHAVIOUR-ONLY | `test_post_review_verdict.py::test_never_retargets_the_verdict_at_the_new_head` |
|
||||
| `scripts/pr-changed-files.sh` | the verdict exemption, on an incomplete enumeration | GUARD | BEHAVIOUR-ONLY | `test_pr_changed_files.py::test_a_SHORT_page_does_not_end_the_enumeration` |
|
||||
| `scripts/prove-fix.sh` | the `prove-fix` job, on a commit whose `Proves:` trailer names a test that passes without the fix | GUARD | MUTATION | `test_prove_fix.py::test_MUTATION_disarming_the_UNPROVEN_clause_reddens_the_refusal_test` |
|
||||
| `scripts/update-openapi.sh` | nothing (regenerates the spec) | TOOLING | NONE | — |
|
||||
| `scripts/tests/test_bom_guard_detection.py` | the `script-tests` job | PROOF | NONE | — |
|
||||
| `scripts/tests/test_build_catalog.py` | the `script-tests` job | PROOF | NONE | — |
|
||||
| `scripts/tests/test_build_catalog_check_path.py` | the `script-tests` job | PROOF | NONE | — |
|
||||
| `scripts/tests/test_check_review_verdict.py` | the `script-tests` job | PROOF | NONE | — |
|
||||
| `scripts/tests/test_check_doc_narrative.py` | the `script-tests` job | PROOF | NONE | — |
|
||||
| `scripts/tests/test_ci_dropped_step_guard.py` | the `script-tests` job | PROOF | NONE | — |
|
||||
| `scripts/tests/test_ci_image_pin_population.py` | the `script-tests` job, when a container job loses its pin | GUARD | MUTATION | `test_ci_image_pin_population.py::test_a_single_job_losing_its_pin_is_DETECTED` |
|
||||
| `scripts/tests/test_ci_release_path_scan_job.py` | the `script-tests` job, on a weakened release-path scan job | GUARD | NONE | — |
|
||||
| `scripts/tests/test_decisions_lib.py` | the `script-tests` job | PROOF | NONE | — |
|
||||
| `scripts/tests/test_decisions_validate.py` | the `script-tests` job | PROOF | NONE | — |
|
||||
| `scripts/tests/test_guard_inventory.py` | the `script-tests` job, on an unclassified guard or a stale proof ref | GUARD | MUTATION | `test_guard_inventory.py::test_the_inventory_covers_exactly_the_guards_that_exist` |
|
||||
| `scripts/tests/test_guard_populations_derive_from_git.py` | the `script-tests` job, on a guard whose file population admits a file git does not track | GUARD | MUTATION | `test_guard_populations_derive_from_git.py::test_no_derivation_admits_an_untracked_file` |
|
||||
| `scripts/tests/test_hook_fire_log.py` | the `script-tests` job, on a hook that stops reporting that it fired, or whose reporting changes what the harness sees | GUARD | MUTATION | `test_hook_fire_log.py::test_a_hook_that_LOSES_its_instrumentation_is_DETECTED` |
|
||||
| `scripts/tests/test_jq_preflight.py` | the `script-tests` job | PROOF | NONE | — |
|
||||
| `scripts/tests/test_merge_consent_base_change.py` | the `script-tests` job | PROOF | NONE | — |
|
||||
| `scripts/tests/test_merge_consent_exemption.py` | the `script-tests` job | PROOF | NONE | — |
|
||||
| `scripts/tests/test_merge_consent_required_check.py` | the `script-tests` job | PROOF | NONE | — |
|
||||
| `scripts/tests/test_migration_equivalence.py` | the `script-tests` job | PROOF | NONE | — |
|
||||
| `scripts/tests/test_mutation_harness.py` | the `script-tests` job, when a `MUTATION` row's declared clause no longer reddens the test the row names | GUARD | MUTATION | `test_mutation_harness.py::test_MUTATION_disarming_the_DIAGNOSTIC_gate_accepts_a_red_for_the_wrong_reason` |
|
||||
| `scripts/tests/test_post_review_verdict.py` | the `script-tests` job | PROOF | NONE | — |
|
||||
| `scripts/tests/test_prepush_unsets_git_env.py` | the `script-tests` job | PROOF | NONE | — |
|
||||
| `scripts/tests/test_pr_changed_files.py` | the `script-tests` job | PROOF | NONE | — |
|
||||
| `scripts/tests/test_prepush_rebase_check_tag_exemption.py` | the `script-tests` job | PROOF | NONE | — |
|
||||
| `scripts/tests/test_prove_fix.py` | the `script-tests` job | PROOF | NONE | — |
|
||||
| `scripts/tests/test_remote_state_inventory.py` | the `script-tests` job, on an executable that talks to a remote service with no row in `docs/remote-state-inventory.md` | GUARD | MUTATION | `test_remote_state_inventory.py::test_every_in_scope_file_has_a_row_and_every_row_names_a_real_file` |
|
||||
| `scripts/tests/test_worktree_ownership_guard.py` | the `script-tests` job | PROOF | NONE | — |
|
||||
|
||||
## The `MUTATION` column is executed, not asserted
|
||||
|
||||
Every row graded `MUTATION` carries a DECLARED clause mutation in `scripts/tests/mutation_manifest.py`.
|
||||
`scripts/tests/test_mutation_harness.py` applies each one to an isolated copy of this repository and
|
||||
requires that row's own named test to go red; the manifest and the `MUTATION` rows are compared for
|
||||
set equality in both directions, so the grade and the mutation cannot drift apart. Adding a row
|
||||
graded `MUTATION` without declaring its clause fails the suite. Full contract and its limits:
|
||||
`docs/decisions/records/testing/mutation-claims-are-executed.md`.
|
||||
|
||||
Two things that column still does not say. It does not say the declared clause is the ONLY thing the
|
||||
guard hangs on — for three rows (measured 2026-08-22) the redness arrives through the
|
||||
proof test's own "this clause has moved, RETARGET it" assertion rather than through changed
|
||||
behaviour, which catches the recorded proof going stale but not much else. And one row is graded `DETECTOR` rather than `CLAUSE` in the
|
||||
manifest: `instrumentation_faults` accumulates from four independent arms and a stripped hook trips
|
||||
three at once, so no single-arm disarm reddens its proof. That finer mutation is carried as data and
|
||||
re-run every suite, and must keep surviving — if it starts reddening, the guard has become
|
||||
clause-provable and the entry is regraded.
|
||||
|
||||
## What the numbers say
|
||||
|
||||
36 guards, 6 tooling scripts, 19 proof files. **14 guards carry a mutation proof; 6 are
|
||||
behaviour-only; 16 have none.** These figures are asserted against the table by
|
||||
`test_the_summary_counts_match_the_table` — they were wrong in the first draft (28/4/6/3/19 against
|
||||
a table holding 27/5/6/3/18), because a hand-maintained summary of a table is a second copy of it,
|
||||
which is the duplication family this change argues against. Both cold reviewers found the error
|
||||
independently.
|
||||
|
||||
**One guard was WITHDRAWN rather than shipped**, and it is worth a line here because the inventory
|
||||
is where a future session will look for it. A `test_review_verdict_vocabulary_parity.py` asserted
|
||||
set equality between the H10 verdict vocabulary's write side and read side by extracting both from
|
||||
shell source with regexes. Six cold-review rounds each found another shell construction that either
|
||||
escaped it or made it red on a correct tree. It was deleted rather than patched a seventh time: the
|
||||
`testing.guard-derives-population-from-source` record says a weak detector is itself the
|
||||
symptom-keyed mistake, and this was the empirical demonstration. The underlying duplication is real
|
||||
and is now UNMITIGATED — tracked in #788, which fixes it by removing the second copy.
|
||||
|
||||
**What that file asserted, and where each part went** — enumerated because the first attempt at this
|
||||
withdrawal silently dropped one and claimed otherwise, which is
|
||||
`process.enumerate-workaround-behaviors-before-deleting` failing on its own removal:
|
||||
|
||||
| Invariant it asserted | Where it is now |
|
||||
|---|---|
|
||||
| Bidirectional write/read vocabulary parity | **LOST.** The reason for the withdrawal; #788 |
|
||||
| No READ-side word is both positive and negative | **LOST** as a universal property; #788 |
|
||||
| No WRITE-side word is both success and failure | **LOST** as a universal property; #788 |
|
||||
| Strict extractor consumes every case arm | Retired with the extractor; #788 |
|
||||
| `POS_RE`/`NEG_RE` each assigned exactly once | Retired with the extractor; #788 |
|
||||
| Both extractors find non-empty sets (anti-vacuity) | Retired with the extractor; #788 |
|
||||
| A wrong write source makes the extractors disagree | Retired with the extractor; #788 |
|
||||
|
||||
Two guards were ADDED in its place. They are **not** replacements for rows 2 and 3 and the table
|
||||
above deliberately does not list them as such — an earlier draft did, and cold review was right that
|
||||
relabelling a lost invariant as a narrower surviving one is the exact failure this table exists to
|
||||
prevent, committed by the table:
|
||||
|
||||
| Added | What it pins |
|
||||
|---|---|
|
||||
| `test_check_review_verdict.py::test_each_verdict_word_retains_its_established_polarity` | the five established tokens still READ with their established polarity |
|
||||
| `test_post_review_verdict.py::test_each_verdict_word_posts_its_established_polarity` | the five established tokens still POST with their established polarity |
|
||||
|
||||
**Polarity is strictly weaker than disjointness, and the gap is demonstrable.** Put `MERGEABLE` in
|
||||
BOTH `case` arms of `post-review-verdict.sh`: the success arm wins, every polarity assertion stays
|
||||
green, and the withdrawn disjointness test failed. What the two added guards do catch is the
|
||||
dangerous direction — a token that a reviewer means as `BLOCKED` silently reading or posting as
|
||||
approval, which writes a green `review-verdict/h10`. A token added to one script and not the other
|
||||
is untested by either, which is the whole of #788.
|
||||
|
||||
**A guard was WITHDRAWN from `test_hook_fire_log.py` after four iterations**, recorded here because
|
||||
this is where a future session will look for it. A `test_no_redirection_in_the_sink_PRECEDES_its_stderr_redirect`
|
||||
generalised the `exec … 2>/dev/null` rule to any command by pattern-matching shell source. Each
|
||||
iteration fixed a false positive or false negative the previous one introduced — `>/dev/null 2>&1`
|
||||
flagged wrongly, then operators preceded by a digit (`exec 0<`) missed entirely, then multi-command
|
||||
lines flagged wrongly — and cold review then constructed more of both (`[[ "$x" < "$y" ]] 2>/dev/null`,
|
||||
a backslash continuation, a `>` inside a quoted string). Deleted rather than patched a fifth time,
|
||||
on the same reasoning as the vocabulary-parity withdrawal above.
|
||||
|
||||
**`test_hook_fire_log.py` proves THREE clauses, and its row claims two of them.** Coverage (every
|
||||
hook reports that it fired) via `test_a_hook_that_LOSES_its_instrumentation_is_DETECTED`;
|
||||
transparency (the wrapper changes nothing the harness can see) via
|
||||
`test_instrumentation_changes_NOTHING_the_harness_can_see` with its mutation proof
|
||||
`test_DELETING_the_replay_makes_the_differential_go_RED`; and placement (`etv_hook_fire_begin` must
|
||||
precede the stdin read) via `test_begin_placed_AFTER_the_stdin_read_is_DETECTED`. The row's `Blocks`
|
||||
column covers the first two — "stops reporting that it fired, **or whose reporting changes what the
|
||||
harness sees**" — and the `Proof ref` column holds one ref because the column holds one, not because
|
||||
the second is unproven. Placement is proved and unclaimed.
|
||||
|
||||
**Its `test_the_suite_does_not_write_to_the_PRODUCTION_log` is narrower than its docstring**, which
|
||||
says `conftest.py` "must isolate every test, not just this file's". What it checks is that the
|
||||
fixture set `ETV_HOOK_FIRE_LOG_DIR` *for the test currently running* and that a hook it drives in
|
||||
its own sandbox does not touch the real log. It cannot see another suite that snapshots
|
||||
`os.environ` at IMPORT time — before the autouse fixture runs — and hands that stale mapping to its
|
||||
subprocesses. That suite's hooks then write to `$HOME/.cache/ersatztv/hook-fire/` while every
|
||||
assertion stays green, because the fire-log library is fail-open. #785 shipped exactly that defect,
|
||||
inside the file added to prove those very hooks.
|
||||
|
||||
The reproduction is the part worth keeping: restore a module-level `{**os.environ}` snapshot — or
|
||||
leave the helper correct and point a single `env=` argument at one — then run that file and count
|
||||
records for its synthetic session ids. **58 per run**, measured identically on macOS/git 2.55 and
|
||||
Linux/git 2.47.3. (The accumulated total observed before the fix ran to four figures across many
|
||||
runs; that is an observation rather than a reproducible measurement, so check the per-run figure.)
|
||||
|
||||
The pin is `test_worktree_ownership_guard.py::test_driving_a_hook_LANDS_its_records_in_the_ISOLATED_dir`,
|
||||
which asserts the EFFECT — records land in the fixture's dir — rather than the shape of the fix. Its
|
||||
predecessor asserted the helper's return value, and cold review showed that leaves the call site
|
||||
unguarded: `_env()` correct, one `env=` reverted, all tests green, records still leaking. Pinning to
|
||||
a hand-written revert rather than to the property is `verify-against-the-REAL-predecessor`.
|
||||
|
||||
It is still narrower than the property: it guards the launch path it drives. A second launcher in
|
||||
the same file that passed a stale environment would leave it green — measured, 18 records — because
|
||||
the hooks it drives would still log correctly. Every hook in that file goes through one helper
|
||||
today, which is what makes it sufficient there. Generalising it is #809, and the reason that is hard
|
||||
is that the obvious version races against a real session's hooks firing during the run.
|
||||
|
||||
**That unproven set carries no number here on purpose.** It restated the count, drifted the moment
|
||||
the BOM guard was regraded, and `test_the_summary_counts_match_the_table` cannot see it — the parser
|
||||
checks the formatted summary sentence and nothing else. A second hand-maintained copy of a number is
|
||||
the duplication family this file argues against, so the copy is removed rather than corrected: the
|
||||
set is the eight `.claude/hooks/` guards enumerated in the standing list below, and the count is in
|
||||
the summary above. Naming them by event is what made the earlier wording wrong twice — they do not
|
||||
share one event, and `design-sync-reminder.sh` is registered on **both** `PreToolUse` and `Stop` in
|
||||
`.claude/settings.json`, so any "every X hook" phrasing double-counts it.
|
||||
|
||||
**They are now observable but still unproven, and the two words carry different weight.** Observable:
|
||||
every hook records its own execution through `scripts/hook-fire-log.sh`, so "did this hook fire, and
|
||||
what did it decide" is a measurement — run `scripts/hook-fire-log.sh report` (#776). Unproven:
|
||||
nobody has demonstrated any of them is load-bearing — the first group in the standing list below.
|
||||
Observability tells you a guard ran; only a mutation tells you it would have caught anything. The
|
||||
BOM guard is the case that shows why the distinction matters — it was firing on every commit the
|
||||
whole time it was fail-open.
|
||||
|
||||
The gaps are not uniform in cost, and the ranking that matters is *what a silent failure would let
|
||||
through*, not test count. **All four ranked entries now carry clause-level mutation proofs (#785);
|
||||
they are kept here with what each mutation established, because the ranking is the reusable part and
|
||||
because two of them turned out to be worse than the ranking predicted.**
|
||||
|
||||
1. ~~`pretooluse-bom-guard.sh`~~ — **proven, and it was fail-open the whole time.** Ranked first
|
||||
because the defect it guards has recurred three times (#311, #402, #405); that ranking turned out
|
||||
to be right for a worse reason than intended. It detected a BOM with `xxd -p`, and `xxd` ships
|
||||
with vim and is **absent on the Linux CI runner**, so the comparison never matched and every BOM
|
||||
was allowed in silence. `od` now. The lesson for the rows below: an unproven guard is not merely
|
||||
untested, it is a guard whose *current* behaviour nobody has established.
|
||||
2. ~~`pretooluse-worktree-guard.sh` + `posttooluse-worktree-marker.sh`~~ — **proven as a pair.**
|
||||
Four clauses were disarmed and witnessed red: the guard's marker read; the guard's ownership
|
||||
comparison (inverted, because disarming it the other way only makes the guard deny more and every
|
||||
deny assertion stays green); the `commit|merge` alternation, whose `merge` half every other case
|
||||
in the file left untested while guarding the plumbing-merge path; and — the one that could not
|
||||
exist while the halves were tested apart — the *marker hook's write*, asserted against the
|
||||
*guard's* decision. Both hooks are
|
||||
deliberately fail-open, so an absent mechanism and a working one produce the identical "commit
|
||||
allowed".
|
||||
3. ~~`.husky/pre-push:11`'s `unset GIT_DIR GIT_WORK_TREE GIT_INDEX_FILE`~~ — **proven, and the case
|
||||
is the normal one rather than an edge.** Git exports `GIT_DIR` to `pre-push` **when the push
|
||||
comes from a worktree** and not from the main tree; `process.shared-tree-readonly` makes the
|
||||
worktree the mandated way to work here, so every push takes the exposed path. With `GIT_DIR` set
|
||||
and `GIT_WORK_TREE` unset git stops discovering the repo and treats the *current directory* as
|
||||
the work tree, so `cd web && npm run check:api`'s `git diff --exit-code` compares against index
|
||||
paths that do not exist and reports no diff. Both the deletion and the relocation are proved.
|
||||
4. ~~`scripts/build_decisions_catalog.py`~~ — **proven, including the wiring.** The `--check`
|
||||
comparison is mutated directly, and a separate subprocess case runs the command *derived from
|
||||
`pr-checks.yml`* against a copied corpus. That second case is not redundant: replacing
|
||||
`raise SystemExit(main())` with a bare `main()` leaves the script printing
|
||||
`docs/decisions/README.md is stale` on stderr while exiting **0**, and the workflow step reads
|
||||
nothing but the exit code. Only the subprocess case reddens — the #751/#719 shape.
|
||||
|
||||
**What a file-level `MUTATION` grade does and does not claim, because three of these four rows are
|
||||
multi-purpose files.** The grade covers *the clause the cited case actually mutates*, per
|
||||
`testing.guard-ships-with-mutation-proof`; it is not a statement about every line in the file.
|
||||
`.husky/pre-push` runs four other things, and its row asserts only that line 11 is load-bearing —
|
||||
the three hooks it invokes carry their own rows and their own grades. Reading the row as "pre-push
|
||||
is tested" is the same relabelling error the withdrawn-guard table above exists to prevent.
|
||||
Clause-level grading is tracked in #790.
|
||||
|
||||
**The remaining unproven guards, and why each is still `NONE` rather than merely unattended** — the
|
||||
third `## Done-when` box of #785, answered here rather than left implicit. They fall into three
|
||||
groups, and the split is the point: the first two are backlogs, the third is not.
|
||||
|
||||
- **The eight remaining hook guards** — genuinely unproven, and a real backlog. Four are
|
||||
`PreToolUse` (`pretooluse-agent-model.sh`, `pretooluse-agent-ram.sh`, `pretooluse-bash-guard.sh`,
|
||||
`pretooluse-nav-guard.sh`), two are pre-push (`prepush-clean-worktree-check.sh`,
|
||||
`prepush-donewhen.sh`), one is pre-commit (`decisions-guard.sh`) and one is registered on both
|
||||
`PreToolUse` and `Stop` (`design-sync-reminder.sh`); they are grouped by their status, not by
|
||||
their event, because the events do not partition them. They are now
|
||||
*observable* (`scripts/hook-fire-log.sh report`, #776), which is a weaker claim than proven and is
|
||||
stated as such above.
|
||||
- **`.husky/commit-msg` and `.husky/pre-commit`** — unproven, and each carries its own clause, so
|
||||
they are a backlog too rather than "covered by what they call". `pre-commit` dispatches to
|
||||
`lint-staged` and `decisions-guard.sh`, but the root-level-`*.png` refusal and the
|
||||
`dotnet format whitespace --verify-no-changes` block are its own. `commit-msg` is not a dispatcher
|
||||
at all: it is one `grep -q '^Co-Authored-By:'` plus a `MERGE_HEAD` exemption, and nothing tests
|
||||
that the exemption fires only for merges. Neither is covered by what it calls — the trap is to read
|
||||
"it dispatches to guards" as "its own clauses are guarded".
|
||||
- **`ci-detect-already-validated.sh`, `ci-detect-docs-only.sh`, `ci-prove-ban-detects.sh`,
|
||||
`e2e-functional.sh`, `check-kickoff-guard.sh`, `test_ci_release_path_scan_job.py`** — not
|
||||
unattended either, but each needs its own harness rather than a proof written to clear a row.
|
||||
`ci-prove-ban-detects.sh` is the exception noted in `testing.guard-ships-with-mutation-proof`: it
|
||||
runs its own mutation at CI time, because what it proves is disarmable from inside pytest.
|
||||
|
||||
That list is prose and nothing checks it, which is the honest limit — a guard moving out of a group
|
||||
will not redden anything. It is here so the next session inherits *why* a row is `NONE`, which is
|
||||
the distinction #785 asked for; the machine-checked half remains the table.
|
||||
|
||||
Filling the rest is tracked rather than done in one pass, deliberately: a mutation proof written to
|
||||
close a row is the kind of test that passes for the wrong reason.
|
||||
@@ -314,7 +314,12 @@ HARD CONSTRAINTS:
|
||||
compiler/parser, security, migrations, review arbitration) → orchestrator tier; independent review →
|
||||
a different model family than the implementer. Omitting it silently inherits the orchestrator tier, so
|
||||
state the choice out loud. → `process.per-agent-model-routing`
|
||||
- **Local gate + cold-context review BEFORE the push**, never after. → `process.local-gate-before-push`
|
||||
- **Local gate + cold-context review BEFORE the push**, never after — and the push is licensed by a
|
||||
**CLEAN verdict, not by a review having run.** Zero outstanding findings on the current tree, however
|
||||
many rounds that takes; "round 1's findings are fixed" and "the mechanism has been cleared" are both
|
||||
reasons to keep reviewing, not to push. Each extra push auto-cancels the live run, and a cancelled job
|
||||
reads as `failure` at the commit-status endpoint — so an early push manufactures phantom reds on top of
|
||||
the wasted runner time. → `process.local-gate-before-push`, `ci.cancelled-is-not-a-verdict`
|
||||
- **Independent review is mandatory** for locks/concurrency, auth/security, API write-path handlers, DB
|
||||
migrations, or >~150 changed C# lines; a skip must be stated with its reason. → `process.independent-review-rubric`
|
||||
- **Batch your pushes — you cannot cancel a CI run.** Only the operator can cancel, in the browser.
|
||||
@@ -346,8 +351,10 @@ HARD CONSTRAINTS:
|
||||
> **Scope**: *how we work* — orchestration, CI triage, review routing, cross-session hygiene. Each rule
|
||||
> below is one or two lines plus its decision `key:`; the evidence, the incident that produced it and the
|
||||
> full rationale live in that record (`docs/decisions/workflow-process.md` for most of them). **Keep it
|
||||
> that way** — this file is pasted into every session's kickoff, so a paragraph of narrative here is a tax
|
||||
> paid by every future session, while a record is retrieved only when it is needed (#542).
|
||||
> that way** — `docs.no-session-narrative` is the general rule (a doc records the end state; the path to
|
||||
> it goes in the commit message), and it binds here with one extra cost on top: this file is pasted into
|
||||
> every session's kickoff, so a paragraph here is a tax paid by every future session, while a record is
|
||||
> retrieved only when it is needed (#542).
|
||||
>
|
||||
> Editing: prune covered and stale bullets rather than appending — this is not append-only and git keeps
|
||||
> the history. If you add a rule, write the record first and cite it here.
|
||||
@@ -386,7 +393,9 @@ HARD CONSTRAINTS:
|
||||
**Reading a red CI run** (check these before diagnosing your diff)
|
||||
- A **killed** job reports `conclusion: failure` — read the log tail for the `❌ Failure - Main` marker;
|
||||
log timestamps are UTC, the host is UTC+2. → `ci.killed-job-triage`
|
||||
- **`cancelled` is not a verdict** — report FAILED and CANCELLED separately. → `ci.cancelled-is-not-a-verdict`
|
||||
- **`cancelled` is not a verdict**, and the endpoint you poll hides it: `commits/{sha}/status` has no
|
||||
`cancelled` state and reports one as `failure`. Resolve the job `conclusion` via
|
||||
`actions/runs/{id}/jobs` before believing a red. → `ci.cancelled-is-not-a-verdict`
|
||||
- A failure inside a **setup/cache step**, before your code compiles, is environmental.
|
||||
→ `ci.infra-shaped-red-under-load`
|
||||
- A lone **`decisions lifecycle`** red is a known flake: do **nothing**, the operator reruns it. A
|
||||
|
||||
@@ -0,0 +1,190 @@
|
||||
# Local code-intelligence tooling (LSP + the csharp-lsp MCP server)
|
||||
|
||||
What is available for "find every site that references this symbol", how each surface is configured,
|
||||
and how to verify it rather than assume it. Run `scripts/check-local-lsp.sh` to check all of it at
|
||||
once.
|
||||
|
||||
This matters beyond convenience. `docs/defect-shapes-773.md` measures that **39% of recorded process
|
||||
failures** are a fix or a guard applied to a sample instead of the population. The set-equality rule
|
||||
(`testing.guard-derives-population-from-source`) covers populations of *values*; it deliberately does
|
||||
not cover the residue where the population is *sites in code* — #403 (5 of 6 dispatch sites) and #671
|
||||
(a by-id handler covering 4 of 10 media types). Find-all-references is the tool for that residue, and
|
||||
it is overwhelmingly a C# problem here.
|
||||
|
||||
## The two surfaces, and which one a subagent can reach
|
||||
|
||||
This distinction is the whole reason both halves are documented together.
|
||||
|
||||
| Surface | Servers | Who can call it |
|
||||
| --- | --- | --- |
|
||||
| The **`LSP` tool** (Claude Code plugins) | `csharp-ls`, `typescript-language-server`, `pyright-langserver` | **No dispatched subagent has been observed to reach it.** `ToolSearch` returns "No matching deferred tools found" for `select:LSP` in a subagent while the same query resolves in the main session — Claude Code 2.1.232, agent types `general-purpose` and `Explore`, 2026-08-14, plus the independent observation in `docs/defect-shapes-773.md` §5.1. Three observations on one harness version: treat it as measured behaviour to design around, not an architectural guarantee. |
|
||||
| The **`csharp-lsp` MCP server** (`.mcp.json`) | wraps `csharp-ls`; serves `csharp_references`, `csharp_diagnostics`, `csharp_hover`, `csharp_definition`, `csharp_symbols`, `csharp_completions`, `csharp_set_workspace`, … (16 tools) | **Main session and subagents.** Subagents demonstrably call MCP tools here: 326 MCP calls from inside subagent turns across the transcript corpus, spanning four servers (`gitea` 288, `playwright` 26, `ssh-mcp` 6, `mempalace` 6). Not yet demonstrated for `csharp-lsp` specifically — that server only became startable on 2026-08-14 — so this is the general MCP boundary, evidenced, rather than a per-server measurement. |
|
||||
|
||||
So an instruction telling *delegated agents* to use C# code intelligence must point them at the
|
||||
**MCP tools**, never at the `LSP` tool. Briefing harder does not help: an agent that cannot resolve
|
||||
the tool falls back to Grep, and does not announce that it did.
|
||||
|
||||
## Configuration, and the failure each piece prevents
|
||||
|
||||
Both #777 root causes were the same shape — a config naming a path that this machine does not have,
|
||||
with nothing checking. Neither produced a wrong answer; each produced a server that could not start.
|
||||
|
||||
### `csharp-ls` needs a dotnet root that owns `host/fxr`
|
||||
|
||||
`MSBuildLocator` resolves the SDK next to the `dotnet` host it finds and requires a sibling
|
||||
`host/fxr/*/libhostfxr.dylib`. A Homebrew install does not satisfy that from `bin`:
|
||||
|
||||
- `/opt/homebrew/bin/dotnet` → `…/Cellar/dotnet/<v>/bin/dotnet`, and `…/bin/host/fxr` **does not exist**
|
||||
- the real root is `/opt/homebrew/opt/dotnet/libexec`, which does own `host/fxr`
|
||||
|
||||
Unset, the server dies at `initialize` with `".NET SDK cannot be resolved, because libhostfxr.dylib
|
||||
cannot be found inside …/bin/host/fxr"`. Builds are unaffected — `dotnet --version` works — so this
|
||||
is invisible until a language-server query is actually made.
|
||||
|
||||
Set `env.DOTNET_ROOT` to the root that owns `host/fxr` in **`.claude/settings.local.json`**:
|
||||
|
||||
```json
|
||||
{ "env": { "DOTNET_ROOT": "/opt/homebrew/opt/dotnet/libexec" } }
|
||||
```
|
||||
|
||||
Measured 2026-08-14 against `csharp-ls` 0.22.0, workspace `/Users/timothy/ersatztv`, five variants of
|
||||
`initialize` — each of these is sufficient **on its own**, and the control is the only failure:
|
||||
|
||||
| Variant | Value | `initialize` |
|
||||
| --- | --- | --- |
|
||||
| control | inherit the plain shell environment | **fails** (`libhostfxr.dylib` not found) |
|
||||
| `DOTNET_ROOT` | `/opt/homebrew/opt/dotnet/libexec` | OK |
|
||||
| `DOTNET_HOST_PATH` | `/opt/homebrew/opt/dotnet/libexec/dotnet` — the direct libexec **host binary**, not `/opt/homebrew/bin/dotnet` | OK |
|
||||
| `PATH` first entry | `/opt/homebrew/opt/dotnet/libexec` | OK |
|
||||
| `DOTNET_ROOT` + `PATH` (the `~/.codex/config.toml` shape) | both of the above | OK |
|
||||
`DOTNET_ROOT` is chosen because it is also what the MCP server entry and
|
||||
`~/.codex/config.toml` use, so the three agree.
|
||||
|
||||
Settings `env` is read at **session start** and is inherited by the spawned server: the `csharp-ls`
|
||||
process a session starts carries `DOTNET_ROOT=/opt/homebrew/opt/dotnet/libexec` in its environment
|
||||
(`ps eww`, verified 2026-08-14). A session already running when the setting changed keeps the old
|
||||
environment — restart it rather than concluding the fix failed.
|
||||
|
||||
**It goes in the untracked local settings, not the tracked `.claude/settings.json`, deliberately.**
|
||||
The value is one machine's Homebrew prefix. Committing it would export that path to every checkout,
|
||||
and on a host with a working dotnet elsewhere it would point a *working* server at a directory that
|
||||
does not exist — turning a shared config into the same environment-divergence failure this issue
|
||||
fixed. What is committed is the knowledge (this doc, the decision record) and the check
|
||||
(`scripts/check-local-lsp.sh`); the machine-specific value stays machine-local, exactly as
|
||||
`.mcp.json` already is.
|
||||
|
||||
### `typescript-language-server` must resolve `typescript` from the **repo root**
|
||||
|
||||
The workspace root is the repo root, but the package lives in `web/node_modules`, so resolution
|
||||
fails and the server exits with `"Could not find a valid TypeScript installation"`.
|
||||
|
||||
The plugin cannot be configured around it: typescript-language-server v5.1.3 exposes only `--stdio`
|
||||
and `--log-level` (`--tsserver-path` **was removed**), and a tsserver path can otherwise only arrive
|
||||
via `initializationOptions.tsserver.path`, which a plugin `lspServers` entry
|
||||
(`command`/`args`/`extensionToLanguage`) cannot set.
|
||||
|
||||
**Two remedies exist, and they are not equivalent — measured 2026-08-14.** The server resolves
|
||||
TypeScript by walking up from the workspace root and then falling back to `require.resolve`
|
||||
relative to its own install (`lib/cli.mjs`), so a *global* `typescript` is also found. That is what
|
||||
the plugin's own README prescribes (`npm install -g typescript-language-server typescript`), and it
|
||||
is the obvious fix — but here it produces a **worse** failure than the one it cures:
|
||||
|
||||
| Remedy | `initialize` | `findReferences` on `canLeaveCurrentScreen` |
|
||||
| --- | --- | --- |
|
||||
| neither | **fails** — `Could not find a valid TypeScript installation` | n/a |
|
||||
| global `typescript` only (7.0.2, no root link) | succeeds | **empty — 6 polls over ~5 min, always `[]`** |
|
||||
| root `node_modules/typescript` link → workspace 6.0.3 | succeeds | **20 references across 7 files** |
|
||||
|
||||
The global-only row is the dangerous one: the server starts, answers, and answers *nothing*, with no
|
||||
error to notice. A loud refusal is better than a silent empty population, so the root link is the
|
||||
remedy in use here. Only global `typescript@7.0.2` was tested — another global version may behave
|
||||
differently — but that is the point: a global install makes the startup error disappear without
|
||||
proving anything about the answers, so any global-only setup needs its own behavioural check before
|
||||
it is called fixed.
|
||||
|
||||
```bash
|
||||
mkdir -p node_modules
|
||||
ln -sfn "$PWD/web/node_modules/typescript" node_modules/typescript
|
||||
```
|
||||
|
||||
`/node_modules/` is gitignored, so this is a per-checkout step — `scripts/check-local-lsp.sh` reports
|
||||
it when missing. Rooted this way the server reports `Using Typescript version (workspace) 6.0.3 from
|
||||
path ".../web/node_modules/typescript/lib/tsserver.js"` and answers cross-file queries over `web/`
|
||||
exactly as it does when rooted at `web/` directly.
|
||||
|
||||
### The `csharp-lsp` MCP server
|
||||
|
||||
`.mcp.json` is **gitignored**, so its content is not recoverable from this repo — which is precisely
|
||||
how its `command` came to name a dotnet install that no longer exists while
|
||||
`~/.codex/config.toml`'s copy of the same server was migrated. The working entry:
|
||||
|
||||
```json
|
||||
"csharp-lsp": {
|
||||
"command": "/opt/homebrew/opt/dotnet/libexec/dotnet",
|
||||
"args": ["run", "--project",
|
||||
"/Users/timothy/ersatztv/.mcp/csharp-lsp-mcp/csharp-lsp-mcp/src/CSharpLspMcp",
|
||||
"-c", "Release"],
|
||||
"env": {
|
||||
"DOTNET_ROOT": "/opt/homebrew/opt/dotnet/libexec",
|
||||
"PATH": "/opt/homebrew/opt/dotnet/libexec:/opt/homebrew/bin:/Users/timothy/.dotnet/tools:/usr/bin:/bin:/usr/sbin:/sbin"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
The server is a vendored clone of [csharp-lsp-mcp](https://github.com/HYMMA/csharp-lsp-mcp) under
|
||||
`.mcp/` (also gitignored). Drive it with `csharp_set_workspace` on `ErsatzTV.sln` once per session
|
||||
before other calls.
|
||||
|
||||
**The clone does not build as upstream ships it, and that is the part most easily lost.** Upstream
|
||||
targets `net8.0` and its `global.json` pins SDK `8.0.0`; this machine has only SDK 10.0.302, so a
|
||||
fresh clone fails to build and the server never starts — the same end state as the wrong `command`,
|
||||
reached a different way. The working tree is upstream `64185bc` **plus a local retarget**, which is
|
||||
not committed anywhere upstream or here. To reconstruct:
|
||||
|
||||
```bash
|
||||
git clone https://github.com/HYMMA/csharp-lsp-mcp .mcp/csharp-lsp-mcp
|
||||
git -C .mcp/csharp-lsp-mcp checkout 64185bc
|
||||
# retarget for an SDK-10-only host: global.json sdk.version 8.0.0 -> 10.0.0,
|
||||
# CSharpLspMcp.csproj TargetFramework net8.0 -> net10.0, and
|
||||
# Microsoft.Extensions.Hosting / .Logging.Console 8.0.0 -> 10.0.0
|
||||
```
|
||||
|
||||
The alternative is to install the .NET 8 SDK and build upstream unmodified. Either is fine; what is
|
||||
not fine is leaving it undocumented, because `scripts/check-local-lsp.sh` now *starts* the server, so
|
||||
a wrong pin or a missing patch surfaces as a failed smoke test rather than a false pass.
|
||||
|
||||
## Traps
|
||||
|
||||
- **A query issued before the project graph is loaded is answered anyway, and answered wrongly.**
|
||||
Measured 2026-08-14, `findReferences` on the same symbol, cold vs settled:
|
||||
|
||||
| Server | Symbol | Cold answer | Settled answer |
|
||||
| --- | --- | --- | --- |
|
||||
| `typescript-language-server` 5.1.3 (workspace TS 6.0.3) | `canLeaveCurrentScreen` (`web/src/navigationGuard.ts:23`) | **1 location** — the declaration alone | 20 across 7 files |
|
||||
| `csharp-ls` 0.22.0 (`ErsatzTV.sln`) | `ChannelPlaylist.ToM3U()` | **empty `[]`**, repeatedly, while the solution loaded | 6 locations |
|
||||
|
||||
The TypeScript case is the dangerous one: a *non-empty* answer with nothing marking it incomplete.
|
||||
The C# case is loud by comparison — an empty list at least looks unfinished. **Re-issue the query
|
||||
and confirm the count is stable** before treating a reference list as a population. A sample that
|
||||
looks like a population is the exact failure class this tooling exists to prevent.
|
||||
- **Solution discovery finds more than one solution.** `csharp-ls` reports
|
||||
`2 solution(s) found: [ErsatzTV.sln, .mcp/csharp-lsp-mcp/…/CSharpLspMcp.sln]` and loads
|
||||
`ErsatzTV.sln`. If a query ever returns nothing for a symbol that plainly exists, confirm which
|
||||
solution was loaded before concluding anything about the symbol.
|
||||
- **Loading `ErsatzTV.sln` takes minutes**, and a `dotnet build` running concurrently makes it worse.
|
||||
This cost is per session, not per query.
|
||||
- **`pyright-lsp` needs no configuration.** It answered correctly on 2026-08-13 (§5.1) and again on
|
||||
2026-08-14 (`documentSymbol` on `scripts/decisions_lib.py`, full symbol tree). Two dated successes,
|
||||
not a longitudinal claim — it is the control showing a broken C#/TS server is a configuration fault
|
||||
rather than a harness fault.
|
||||
|
||||
## Verifying
|
||||
|
||||
`scripts/check-local-lsp.sh` checks the five preconditions above and prints an actionable remedy per
|
||||
failure. It is **operator-run and wired to no CI job** — every dependency is a developer-machine
|
||||
install, so there is no runner on which a red would mean anything.
|
||||
|
||||
It checks preconditions, not behaviour. The end-to-end confirmation is a real query in a main
|
||||
session — e.g. `findReferences` on `ChannelPlaylist.ToM3U()` (`ErsatzTV.Core/Iptv/ChannelPlaylist.cs`),
|
||||
which returns the declaration plus 5 call sites and, unlike `grep`, excludes the mention of the name
|
||||
in a comment.
|
||||
+41
-3
@@ -186,12 +186,50 @@ Re-adding an already-present item is an **idempotent no-op** (no duplicate rows,
|
||||
referenced id does not exist the whole batch is rejected (`422`). So the flow is: search → add ids →
|
||||
re-run to confirm idempotence.
|
||||
|
||||
### Full-replace writes drop what you omit (`mcp.tool-schema-openapi-parity`)
|
||||
|
||||
**Check each tool's own description — the write tools are not uniform, and one is not uniform with
|
||||
itself.** Three are full replaces, where a field you leave out is not "left unchanged" but written as
|
||||
empty: `ersatztv_update_channel`, `ersatztv_update_schedule`, `ersatztv_update_collection_custom_order`.
|
||||
|
||||
`ersatztv_update_playout` is **mixed, and this is the easy one to get wrong**: `scheduleFile` is
|
||||
leave-unchanged, but `dailyRebuildTime` is always applied — `UpdatePlayoutHandler` sets it to `null`
|
||||
unconditionally before re-applying a supplied value, so calling this tool to set `scheduleFile` while
|
||||
omitting `dailyRebuildTime` **silently clears the daily reset**. Send both, or neither.
|
||||
|
||||
The rest are additive or leave-unchanged and say so: `ersatztv_add_collection_items` is an idempotent
|
||||
add (it does **not** replace membership), `ersatztv_update_collection` leaves an omitted
|
||||
`useCustomPlaybackOrder` alone, and `ersatztv_enable_jellyfin_library_sync` leaves an absent row
|
||||
untouched.
|
||||
|
||||
For the full-replace ones, the GET → edit one field → PUT flow is only safe if the tool can express
|
||||
the whole state, and `ersatztv_update_channel` could not — it omitted `graphicsElementIds`, so that flow
|
||||
silently detached every graphics element (including the On Now/Next overlay) with a `200` and no
|
||||
error, visible only as missing pixels at the next transition. `ersatztv_update_schedule` cleared
|
||||
`padToNearestMinute` the same way (ersatztv#754).
|
||||
|
||||
Both are fixed, and the class is now guarded by two tests in `ToolCatalogTests`, comparing against the
|
||||
generated `ErsatzTV/wwwroot/openapi/v1.json`:
|
||||
|
||||
- Every POST/PUT/PATCH tool declares **exactly** the request-body fields its endpoint accepts, each
|
||||
with a matching type. A new property on a request DTO fails until the catalog declares it.
|
||||
- Every tool — read **and** write — declares **exactly** its endpoint's query parameters. An omitted
|
||||
one is not merely undocumented but *unreachable*, since `ToolArgumentValidator` rejects undeclared
|
||||
arguments; that is how #616 hard-capped two paged tools at the first page, and how
|
||||
`ersatztv_list_playouts` (`query`) and `ersatztv_get_playout_items` (`showFiller`) lost their
|
||||
filters until ersatztv#757.
|
||||
|
||||
When adding a write tool, regenerate the spec (`./scripts/update-openapi.sh`) and add the tool to the
|
||||
pinned list in the body test.
|
||||
|
||||
## Deferred
|
||||
|
||||
Channel create/update (`ersatztv_create_channel` / `ersatztv_update_channel`) wrap a 28-field DTO with
|
||||
nine enum fields. Only `name`/`number`/`ffmpegProfileId` are required; the rest have server-side
|
||||
Channel create/update (`ersatztv_create_channel` / `ersatztv_update_channel`) wrap a large DTO with
|
||||
nine enum fields — 27 body fields on create, and 28 on update, which additionally carries
|
||||
`graphicsElementIds`. Only `name`/`number`/`ffmpegProfileId` are required; the rest have server-side
|
||||
defaults, and the enum fields take the enum **name** (the API validates them). Discover an existing
|
||||
channel's shape and current enum values with `ersatztv_get_channel` before creating/updating.
|
||||
channel's shape and current enum values with `ersatztv_get_channel` before creating/updating — and
|
||||
copy its `graphicsElementIds` through unless you mean to detach them.
|
||||
|
||||
Deliberately **not** exposed in this cautious first write pass:
|
||||
|
||||
|
||||
@@ -0,0 +1,188 @@
|
||||
# Remote-state inventory (ersatztv#778)
|
||||
|
||||
Every **git-tracked** file matching one of these, that reads live remote state and later acts on
|
||||
that read, and whether the read is bound to something that cannot change underneath it:
|
||||
|
||||
| Directory | Files |
|
||||
|---|---|
|
||||
| `scripts/` (recursive, **excluding `scripts/tests/`**) | `*.sh`, `*.py` |
|
||||
| `.claude/hooks/` | `*.sh` |
|
||||
| `.husky/` | all tracked files |
|
||||
| `.gitea/workflows/` | `*.yml`, `*.yaml` |
|
||||
|
||||
The per-directory extensions are stated because the prose once attached them to `scripts/` alone
|
||||
while the guard applied them everywhere, so a `.py` hook would have joined the scope the doc
|
||||
described and acquired no row.
|
||||
|
||||
The scope qualifier is load-bearing, not throat-clearing: this file twice claimed to cover "every
|
||||
executable in this repo" while its own derivation missed real remote readers, so the heading now
|
||||
states exactly what the guard enforces — including the `scripts/tests/` exclusion, so nobody adds a
|
||||
remote-reading test executable expecting a red guard that will stay green. C#/TypeScript guards,
|
||||
`web/`, and anything outside those directories are **not** covered. Scope is limit 3; the
|
||||
files-not-call-sites limit is limit 2.
|
||||
|
||||
`scripts/tests/test_remote_state_inventory.py` derives the population from `git ls-files` and
|
||||
asserts **set equality** against the `Site` column, so a new script that talks to a remote service
|
||||
cannot ship without acquiring a row.
|
||||
|
||||
**Read `docs/decisions/records/process/check-and-use-pins-a-version.md` before adding a row or
|
||||
changing a classification.**
|
||||
|
||||
## Why this file exists rather than a linter
|
||||
|
||||
`docs/defect-shapes-773.md` §4 detector D is a **fix pattern, not a mechanical detector**: there is
|
||||
no general lint for "this code should have pinned a sha." What makes the class actionable is that
|
||||
the population is small and enumerable, so the detector is detector A — derive the population from
|
||||
an authoritative source and assert set equality — applied to this inventory. The inventory is the
|
||||
artifact; the test keeps it from rotting.
|
||||
|
||||
## Columns
|
||||
|
||||
- **Class** —
|
||||
- `PINNED` — the read is bound to an immutable version identifier (a **full** commit sha, an
|
||||
image digest, a monotonic event count), and that binding still holds when the action runs. Two
|
||||
shapes qualify, and the second was missing from the first wording: either the action
|
||||
**re-validates** against the identifier immediately before committing (a snapshot nobody
|
||||
re-checks is not pinned — binding alone is never enough), **or** the check and the use are a
|
||||
single step over a value that cannot move, such as a workflow reading a full sha straight out of
|
||||
its own fixed event payload. What never qualifies is a value captured early and trusted later.
|
||||
The second shape is distinguished from an `N/A` row that says "resolution and use are one step"
|
||||
by the IDENTIFIER, not by the step count: `PINNED` requires the value itself to be immutable (a
|
||||
full sha, a digest), while a one-step read of a MUTABLE identifier — a registry tag, a branch
|
||||
name — is not pinned and is graded on what it authorizes.
|
||||
- `CAS` — the write itself carries a compare-and-set condition the server enforces.
|
||||
- `UNSAFE-KNOWN` — read-then-act with nothing pinning it, **accepted** with the reason stated in
|
||||
the Note. Every row here must say why the residual is tolerable, not merely that it exists.
|
||||
- `N/A` — reads no live remote state, or draws no authorization from what it reads.
|
||||
- **Note** — the window, and what closes or bounds it.
|
||||
|
||||
A row is about a *site*, not a file: a file with two independent reads gets two rows only where the
|
||||
classifications differ; otherwise the strictest applies and the Note names the exception.
|
||||
|
||||
## The inventory
|
||||
|
||||
### Hooks
|
||||
|
||||
| Site | Class | Note |
|
||||
|---|---|---|
|
||||
| `.claude/hooks/pretooluse-merge-consent.sh` — head-sha reads (CI status, H10 status, verdict comments) | `UNSAFE-KNOWN` | Every comparison uses the **full** 40-char `.head.sha` (`${sha:0:7}` appears only in human-readable strings), which is the right identifier — but it is captured ONCE from the PR snapshot at the top of the hook, and is stale in two distinct ways. FIRST, within the run: every later check (CI status, H10 status, verdict comments) is evaluated against that captured sha, so a push landing mid-run is checked against the commit it replaced. This is the same defect that WAS live for `$base_ref` until it was re-read before the branch-protection lookup; the sha is not re-read, and closing it symmetrically is tracked in **#803**. SECOND, after the decision: the hook returns `allow` and a separate call performs the merge. "No async window" was the second overclaim cold review removed from this file. Both are bounded the same way — a head the verdict does not cover cannot inherit the sha-bound required status, so the server refuses it — and neither is bounded by anything in this hook. The merge API accepts an optional `head_commit_id`, which would make the call a true CAS; a PreToolUse hook cannot ADD that argument, only refuse without it, and requiring it changes every merge call's shape — tracked as follow-up rather than smuggled in here. Accepted meanwhile because the window is one tool call wide with no scheduler in it, and the server-side per-sha required check still refuses an unreviewed head. |
|
||||
| `.claude/hooks/pretooluse-merge-consent.sh` — scheduled auto-merge (`merge_when_checks_succeed`) | `UNSAFE-KNOWN` | **Preflight, not a pin** — graded down by cold review, which was right: the hook's own comment concedes the branch-protection read pins nothing, so calling it `PINNED` contradicted this file's definition. What the hook proves is a snapshot; Gitea merges later; and since #778 it also verifies that `review-verdict/h10` is a required check, reading the repo's **full rule list** (never the by-name endpoint, which does no matching and knows nothing about precedence) — nothing can govern the base → deny, unreadable → ask, and a **glob rule that could govern it → ask**, because the hook deliberately does not reimplement Gitea's glob dialect. It also asks when two rule names fold equal, or when either name is non-ASCII, since Gitea's `EqualFold` is Unicode-aware and its rule precedence is not derivable here. That converts an unobserved assumption into an observed precondition and detects drift, but an admin can still weaken the protection *after* the read. Accepted, and the earlier wording here was **circular** — it said the residual was "closed one layer down" by the very branch protection an admin may have removed. It is not closed; it is BOUNDED, and the bound is a trust assumption that should be stated rather than dressed as a mechanism: everything on this path assumes repo-admin branch-protection config is not hostile. If protection is present at preflight and removed afterwards, an unreviewed head can merge, and nothing in this repo would detect it. What the check does buy is that the far commoner case — protection already weakened when the merge is attempted — stops being silent. A PreToolUse hook cannot add `head_commit_id` to the merge call, so it can never convert its own grant into a CAS — it can only refuse, which is what it now does. |
|
||||
| `.claude/hooks/pretooluse-merge-consent.sh` — base-retarget detection | `UNSAFE-KNOWN` | `recorded_base` parsed from the H10 status description is compared against the PR's live `.base.ref` (#632), and it deliberately compares the base **ref** rather than `base.sha` — the tip moves on every unrelated merge, so comparing it would deadlock every open PR. The residual is the same ABA the enumerator has: a name can be retargeted away and back, and the comparison cannot see it. Accepted because the alternative that CAN see it is the monotonic event count, which lives in the workflow that writes the enforced status rather than in this advisory hook. |
|
||||
| `.claude/hooks/pretooluse-merge-consent.sh` — `## Done-when` issue-body read | `UNSAFE-KNOWN` | The issue body carries an `updated_at` that is not used, so a box unticked between the read and the merge is invisible. Accepted: the only actor who can edit the issue is the one requesting the merge, so this is a self-inflicted race with no adversary and no silent-failure mode. |
|
||||
| `.claude/hooks/prepush-donewhen.sh` — `## Done-when` issue-body read | `UNSAFE-KNOWN` | No pin, but the hook's exit code gates the push synchronously — git blocks on this process. Blast radius is near zero regardless: `main` carries `enable_push: false` and `block_admin_merge_override: true`, so the direct push this hook exists to block is refused server-side for every account (#743). This is belt-and-braces over a path the server already refuses. |
|
||||
| `.claude/hooks/pretooluse-nav-guard.sh` | `N/A` | Reads only the proposed tool call's own parameters and decides synchronously; the `curl` mentions in the file are prose, not executed lines. |
|
||||
|
||||
| `.claude/hooks/prepush-rebase-check.sh` — `git fetch origin main`, then `git merge-base --is-ancestor origin/main HEAD` to decide whether to block the push as behind | `UNSAFE-KNOWN` | Fetch-then-decide with no re-validation before the verdict; `origin/main` can advance inside that window. Accepted because the decision is self-correcting and cannot reach `main`: a push allowed on a now-stale read still lands on a feature branch, since direct pushes to `main` are refused server-side (#743), so the worst case is a rebase nag arriving one push later rather than a bad merge. This row exists because the previous token filter did not list `git fetch` and so could not see it at all. |
|
||||
| `.claude/hooks/prepush-clean-worktree-check.sh` — `git fetch origin main`, then diffs `origin/main...HEAD` to scope which dirty files are in the pushed diff | `UNSAFE-KNOWN` | Same fetch-then-decide shape and the same bound: the verdict advises a feature-branch push only, `main` refuses direct pushes server-side (#743), and a stale `origin/main` read at worst lets a dirty-file push through, which the downstream review gate still catches before any merge. |
|
||||
| `.claude/hooks/decisions-guard.sh` | `N/A` | Reads no live remote state — runs `scripts/decisions_validate.py` over the local working tree; no fetch, no HTTP call. |
|
||||
| `.claude/hooks/design-sync-reminder.sh` | `N/A` | Reads no live remote state — compares local `git diff`/`ls-files` output against the session's own edits, never contacts origin. |
|
||||
| `.claude/hooks/posttooluse-worktree-marker.sh` | `N/A` | Reads no live remote state — parses the tool call's own JSON payload and writes a local ownership marker. |
|
||||
| `.claude/hooks/pretooluse-agent-model.sh` | `N/A` | Reads no live remote state — inspects only the proposed Agent call's own `model`/`subagent_type` fields. |
|
||||
| `.claude/hooks/pretooluse-agent-ram.sh` | `N/A` | Reads no live remote state — samples local `memory_pressure -Q` output. |
|
||||
| `.claude/hooks/pretooluse-bash-guard.sh` | `N/A` | Reads no live remote state — pattern-matches the proposed Bash command string for `ETV_UPDATE_GOLDENS=`. |
|
||||
| `.claude/hooks/pretooluse-bom-guard.sh` | `N/A` | Reads no live remote state — inspects local `git diff` output and reads local `.cs` bytes for a BOM. |
|
||||
| `.claude/hooks/pretooluse-worktree-guard.sh` | `N/A` | Reads no live remote state — reads a local `.claude-worktree-owner` marker in the target worktree. |
|
||||
|
||||
### Husky git hooks
|
||||
|
||||
| Site | Class | Note |
|
||||
|---|---|---|
|
||||
| `.husky/pre-push` | `N/A` | Delegates every remote read to `prepush-donewhen.sh`, `prepush-rebase-check.sh` and `prepush-clean-worktree-check.sh`, each of which carries its own row. This file forwards stdin ref lines and runs local `npm run check:api/lint/typecheck/build` against the checked-out tree. |
|
||||
| `.husky/pre-commit` | `N/A` | Reads no live remote state — local lint-staged, `decisions-guard.sh`, a `git diff --cached` scan, and local `dotnet format --verify-no-changes`. |
|
||||
| `.husky/commit-msg` | `N/A` | Reads no live remote state — greps the local commit-message file for a trailer. |
|
||||
|
||||
### Scripts
|
||||
|
||||
| Site | Class | Note |
|
||||
|---|---|---|
|
||||
| `scripts/post-review-verdict.sh` — commit-status write | `PINNED` | Re-reads the PR and compares **both** `.head.sha` and `.base.ref` immediately before the POST, and `die`s (exit 1, no status written) on a mismatch **or on a field it cannot read**. That last clause is new: both comparisons were guarded by `[ -n "$x" ] &&`, so a well-formed 2xx body that merely omitted the field made the check a no-op and the status was posted having confirmed nothing — found by cold review on #778 and regression-tested against the real predecessor, since the redundant `-z` arm alone mutates green. Closes #706 and #632 for this path by read-compare-refuse, not by CAS: Gitea's status API offers no conditional write. Residual: the **comment** is posted *before* the re-read, so a head that moves in between leaves a verdict comment with no status — the comment is not the gate, but the mismatch is confusing and is tracked in **#792**. |
|
||||
| `scripts/pr-changed-files.sh` — paged file enumeration | `UNSAFE-KNOWN` | #707's fix, graded honestly after cold review: `.base.ref`, `.base.sha` and `.head.sha` are captured before paging and re-checked after, and any *observed* movement fails the whole enumeration closed rather than emitting a short list. But before-and-after equality is **ABA-vulnerable** — a `main → scratch → main` retarget during paging can return the same ref and, if nothing merged meanwhile, the same base sha, while the pages in between were diffed against the scratch base. The script's own comment says it narrows rather than erases; this row previously said "any movement fails", which was stronger than the code. Accepted here because the enumerator cannot close it alone, but be exact about what the caller-side fence does and does not cover: `ci.verdict-write-retarget-fence` counts `change_target_branch` events, so it catches the BASE alias and **nothing else**. A HEAD alias is not covered by anything — a force-push `H1 -> H2 -> H1` during pagination leaves the final `.head.sha` comparison equal while the middle pages were enumerated against `H2`, and no counter moves. That residual is real, unfenced, and stated here rather than papered over; closing it needs a monotonic head-mutation fence or enumeration bound to an immutable tree, neither of which exists today — tracked in **#803**, which also carries the three older contracts that still assert more than this row does. |
|
||||
| `scripts/select-queue.sh` — issue list, then per-issue `/dependencies` | `UNSAFE-KNOWN` | The open-issue list (labels, milestone, priority) is snapshotted once; per-candidate dependency reads happen seconds later and never re-read the issue's own labels, so an issue claimed `in-progress` in that gap still appears on the shortlist. Accepted: the script authorizes **no write**. The real gate is the four-way claim check in `process.parallel-session-claim`, which runs after selection and re-reads live state by construction. Tightening this would move a check that must be adversarial into a tool that is advisory. |
|
||||
| `scripts/ci-detect-already-validated.sh` — prior-head combined status | `UNSAFE-KNOWN` | Reads the PR head's status and emits `skip=true`, with nothing re-checking before the consuming job runs. Accepted and narrow: the skip elides only **re-running** test/migrations on a tree already validated; the `build` job still builds and pushes unconditionally, so no image ever ships from unvalidated source. |
|
||||
| `scripts/issue-qualification-audit.sh` — paged issue list, then a report | `UNSAFE-KNOWN` | Pages the open-issue list and reports which issues lack a `priority:` label, so like every paged read here its pages can straddle a change and the report can name a state no single instant held. Graded to match `select-queue.sh` rather than `N/A`: the two run the same shape, and the reason offered for accepting `select-queue.sh` — it authorizes no write — cannot simultaneously be the reason this one is out of the class. Accepted on the same terms: it is advisory, session-end, human-read, and the labels it prompts for are applied by hand afterwards. |
|
||||
| `scripts/security-scan.sh` — `docker pull`, then `docker run` the same tag | `UNSAFE-KNOWN` | Pull and run are two steps over a MUTABLE tag, which is the same shape the registry rows below were graded down for; "resolution and use are one step" overstated it. In practice the second step resolves against the local daemon, which holds the image the pull just placed, so a mid-window retag does not change what runs. Accepted on that, plus the scope: this boots a throwaway container and scans it, authorizing nothing. |
|
||||
| `scripts/migration-smoke.sh` — `docker pull`, then `docker run` the same tag | `UNSAFE-KNOWN` | The same pull-then-run over a mutable tag as `security-scan.sh` above, and graded with it rather than left behind: its row previously said "Same shape" as a note that has since been rewritten to the opposite conclusion, so the backreference had quietly inverted. This one deserves the grade MORE, not less — `security-scan.sh` boots a throwaway container and authorizes nothing, while this is the pre-deploy migration smoke that gates a production stack recreation. Accepted on the same bound (the run resolves against the local daemon holding the image the pull just placed) plus its own stated operator-trust gap: the resolved image id is **reported** for a human rather than compared against a prior read. |
|
||||
| `scripts/hook-fire-log.sh` | `N/A` | Entirely local: reads stdin and writes JSONL under the cache dir; the only `curl` in the file is in a comment. |
|
||||
| `scripts/e2e-local.sh` | `N/A` | No outbound call at all; readiness is a local log grep and a local port probe against a subprocess it started. |
|
||||
| `scripts/e2e-ui.sh` | `N/A` | Launches a local Chromium and runs specs against `http://localhost:$PORT`. |
|
||||
| `scripts/e2e-functional.sh` | `N/A` | Every call targets `$BASE_URL`, defaulting to `http://localhost:8409`. The one non-local-looking address, `192.0.2.1`, is TEST-NET-1 (RFC 5737) — written into the DB as a connection row precisely so it is unroutable, never dialed by the script. |
|
||||
|
||||
| `scripts/ci-detect-docs-only.sh` — `git fetch origin "$base"`, then diffs the fetched tip against HEAD to emit `docs_only`, which gates whether the required test/migrations jobs run their real steps | `UNSAFE-KNOWN` | Read-then-act with no re-check between the fetch and the emitted value, and the decision genuinely gates required CI work. Accepted because the script is deliberately asymmetric: every ambiguous, undeterminable or shallow-checkout case resolves to `docs_only=false` (run everything), and only an exact unanimous all-docs diff yields `true` — so a stale or racing base read can at worst cause an unnecessary full run, never a skipped one (#416). |
|
||||
| `scripts/refresh-shared-checkout.sh` — `git fetch origin main`, then `git merge --ff-only origin/main` and a conditional `npm ci` | `UNSAFE-KNOWN` | Fetch-then-act with no re-check between the fetch and the merge. Accepted because every action is self-refusing or reversible: `--ff-only` fails harmlessly rather than diverging if the ref moved on, the script refuses outright when the tree is not clean `main` or is ahead or mid-rebase, and this is a developer-convenience checkout rather than a release or merge-authorization path — a stale read costs one extra fetch next run, never lost work. |
|
||||
| `scripts/check-review-verdict.sh` | `N/A` | Reads no live remote state itself — classifies a comments JSON payload supplied on stdin; the fetch belongs to the caller's row. |
|
||||
| `scripts/decisions_validate.py` | `N/A` | Reads no live remote state — its `git log`/`show`/`ls-tree`/`merge-base` calls operate on refs the caller already checked out or passed via `--base`/`--head`, never a fetch. |
|
||||
| `scripts/prove-fix.sh` | `N/A` | Reads no live remote state — `git worktree add`/`rev-parse`/`diff-tree` operate on the local repository's own objects. |
|
||||
| `scripts/add-migration.sh` | `N/A` | Reads no live remote state — runs `dotnet ef migrations add` against local project files; implicit NuGet resolution is dependency supply-chain, out of this class per limit 1. |
|
||||
| `scripts/update-openapi.sh` | `N/A` | Reads no live remote state — a local `dotnet build`/`GenerateOpenApiDocuments` then a local python script. |
|
||||
| `scripts/cleanup-code.sh` | `N/A` | `dotnet tool restore` resolves and uses tooling in one step (limit 1, not a check-and-use split); the rest is a local `git status --porcelain` scan. |
|
||||
| `scripts/cleanup-all-code.sh` | `N/A` | Same shape as `cleanup-code.sh` — restore-and-use in one step, no check-then-act over remote state. |
|
||||
| `scripts/build_decisions_catalog.py` | `N/A` | Reads no live remote state — parses local decision records and writes the local catalog. |
|
||||
| `scripts/decisions_lib.py` | `N/A` | Reads no live remote state — pure parser over local decision-record files. |
|
||||
| `scripts/migrate_decisions_split.py` | `N/A` | Reads no live remote state — one-shot local file migration over `docs/decisions/`. |
|
||||
| `scripts/generate-endpoint-index.py` | `N/A` | Reads no live remote state — reads local `v1.json` and writes a local markdown index. |
|
||||
| `scripts/check-doc-narrative.py` | `N/A` | Reads no live remote state. Both modes are local: `--diff` reads `git diff` against a ref the CALLER fetched, `--all` reads `git ls-files`. The `docs-reminder` fetch that supplies the ref carries its own row below. |
|
||||
| `scripts/check-kickoff-guard.sh` | `N/A` | Reads no live remote state — scans a fixed local file list for forbidden phrasing. |
|
||||
| `scripts/check-local-lsp.sh` | `N/A` | Reads no live remote state — probes local PATH binaries and spawns a local MCP server over stdio. |
|
||||
| `scripts/mcp_smoke.py` | `N/A` | Reads no live remote state — spawns a local subprocess and speaks JSON-RPC over stdio pipes, no network socket. |
|
||||
| `scripts/jq-preflight.sh` | `N/A` | Reads no live remote state — runs local `jq --version`. |
|
||||
| `scripts/ci-peak-anon.sh` | `N/A` | Reads no live remote state — samples the runner's local cgroup `memory.stat`/`memory.peak`. |
|
||||
| `scripts/ci-prove-ban-detects.sh` | `N/A` | Reads no live remote state — mutates a local workflow copy and runs pytest against the local checkout. |
|
||||
| `scripts/ci-step-ran.sh` | `N/A` | Reads no live remote state — reads runner-supplied env vars and local marker files it wrote itself. |
|
||||
| `scripts/set-provider.sh` | `N/A` | Reads no live remote state — sets local `dotnet user-secrets` values. |
|
||||
| `scripts/__init__.py` | `N/A` | Empty package marker — executes nothing. |
|
||||
| `scripts/scripted-schedules/entrypoint.py` — `ScriptedScheduleApi.get_context(build_id)`, then `define_content` / `reset_playout` / `build_playout` against the same live server | `UNSAFE-KNOWN` | A genuine read-then-act over live ErsatzTV state, and the row cold review found missing when the population was still non-recursive. The context is fetched, handed to user-supplied script functions that mutate the playout, and re-fetched after a reset with nothing pinning either read — a concurrent build or edit between them is invisible. Accepted because it runs inside a single scripted-schedule build the server itself serialises per playout, and because the API exposes no version or ETag on the context to compare against; the honest bound is that the blast radius is one playout's content, reversible by rebuilding. |
|
||||
| `scripts/macOS/bundle.sh` | `N/A` | Reads no live remote state — moves files and creates symlinks in a local app bundle. |
|
||||
| `scripts/macOS/sign.sh` | `N/A` | `codesign --timestamp` contacts Apple's timestamp server, but resolution and use are one step with no earlier check whose answer is later trusted — the same boundary as limit 1. |
|
||||
| `scripts/macOS/sign-dmg.sh` | `N/A` | Same shape as `sign.sh` — a timestamped `codesign` over a local DMG, no check-then-act over remote state. |
|
||||
|
||||
### Workflows
|
||||
|
||||
| Site | Class | Note |
|
||||
|---|---|---|
|
||||
| `.gitea/workflows/review-verdict.yml` — status read → status POST | `UNSAFE-KNOWN` | The residual this whole class reduces to. Gitea's status API has no ETag, no If-Match and no expected-previous-state, so read and write cannot be made one operation. Narrowed twice rather than claimed closed: a monotonic `change_target_branch` **event-count** fence refuses to write if the count moved (`ci.verdict-write-retarget-fence`; the count is used because the branch *name* is ABA-vulnerable), and a high-water-mark re-read repairs a `success` posted over a human verdict back to `pending`. The file states the residual window explicitly rather than asserting safety. |
|
||||
| `.gitea/workflows/review-verdict.yml` — base-ref checkout | `PINNED` | `ref: ${{ github.event.pull_request.base.sha }}` — a full sha from the fixed event payload, so the PR head cannot supply the workflow definition that judges it. |
|
||||
| `.gitea/workflows/review-verdict.yml` — changed-file enumeration | `UNSAFE-KNOWN` | Delegates to `scripts/pr-changed-files.sh` with the head sha and expected base, and therefore **inherits that row's residual, not a pin** — this row said "inherits that row's pins" while the row it points at was being graded down, which is exactly the stale-cross-reference a multi-round edit produces. Accepted on better terms than the enumerator alone — this is the one caller that also runs the monotonic `change_target_branch` event-count fence (`ci.verdict-write-retarget-fence`) — but only for the BASE axis. The HEAD alias described in the enumerator's row (`H1 -> H2 -> H1` during pagination) is unfenced here too, and this row previously implied the fence covered it. |
|
||||
| `.gitea/workflows/docker-build.yml` — CI toolchain image | `UNSAFE-KNOWN` | Graded down by cold review, correctly: this file's own definition of `PINNED` names an image **digest**, and `ersatztv-ci:<short-sha>` is a mutable **tag**. A registry tag can be repointed after `ci-image-pin` verifies it and before a job pulls it, and a 7-hex short sha is additionally collision-prone. Accepted rather than fixed here because the exposure needs write access to our own LAN registry — i.e. an attacker already inside the trust boundary — and two controls bound it: jobs never consume `:latest`, and `pr-checks.yml`'s `ci-image-pin` fails the build if the tag drifts from the last commit touching `docker/ci/`. Consuming `image@sha256:…` is the real fix and is the natural companion to **#772**, which already covers the availability half of this tag's weakness. |
|
||||
| `.gitea/workflows/docker-build.yml` — release smoke pull | `UNSAFE-KNOWN` | Pulls `${IMAGE}:${SMOKE_SHORT_SHA}`, the tag this same job pushed moments earlier. The first draft called that `PINNED` on the strength of the job's `concurrency` group; cold review showed the group is **per-ref**, so a branch build and a tag build of the same commit sit in DIFFERENT groups and can publish the same `:<short-sha>` — the smoke step can therefore pull the other run's image. Accepted rather than fixed here because the fix is the same one-line change as the row above (pull `image@sha256:…`, propagated from the push step) and belongs with it; until then no registry row in this file claims to be pinned. |
|
||||
| `.gitea/workflows/docker-build.yml` — `api-docs` / `format` base fetch | `UNSAFE-KNOWN` | Fetches the live base tip to diff generated artifacts, with nothing pinning it. Graded up from `N/A` by cold review, which was right that "advisory" was too quick: these are not branch-protection-required contexts, but the merge-consent hook reads Gitea's **combined** status, and a combined state that is not `success` blocks the auto-grant — so a wrong answer here does participate in merge consent. Accepted because the failure direction is benign: a base that advanced mid-job makes a generated artifact look stale and FAILS the job, costing a re-run, rather than passing something it should not. |
|
||||
| `.gitea/workflows/ci-image.yml` — verify the pushed image | `UNSAFE-KNOWN` | Pulls back the `:<sha>` tag it pushed in the immediately preceding step. Same demotion and same reason as the two rows above: a registry tag is not a digest, and anything able to write to the registry can repoint it between the push and the verify, which would make the verification confirm an image other than the one built. Accepted on the same terms — the exposure requires registry write access, i.e. an actor already inside the trust boundary — and the same fix applies. |
|
||||
| `.gitea/workflows/pr-checks.yml` — `prove-fix` | `PINNED` | `base`/`head` are full shas from the event payload, immune to later PR mutation. |
|
||||
| `.gitea/workflows/pr-checks.yml` — `ci-image-pin` | `N/A` | A purely local comparison — `git log` over the checked-out tree against a literal in `docker-build.yml`. It never queries the registry, which is exactly why it cannot bound the retagging the image rows above describe. |
|
||||
| `.gitea/workflows/pr-checks.yml` — `docs-reminder` base fetch | `N/A` | Fetches the live base tip, but the result only selects the text of a non-blocking warning. The rationale has to be narrower than the first draft's "the job cannot fail and never reaches the combined status", which is false — any job's status joins the combined state, and runner or checkout failure can redden it. What is true, and is what earns the `N/A`: the fetch and diff are failure-swallowed, so the remote read cannot change this job's outcome, only the warning's wording. It draws no authorization from what it reads, this file's second `N/A` clause. Grouping it with `decisions-guard` over-demoted it, and the two have different classifications, so per the rule above they get separate rows. Since ersatztv#784 the job runs TWO advisory checks off this fetch (the parity-doc reminder and `scripts/check-doc-narrative.py`); the classification is unchanged because the second is advisory on the same terms — it exits 0 on every path, so a moved base still only changes warning text (its unproven arms are enumerated in `docs.no-session-narrative`). |
|
||||
| `.gitea/workflows/pr-checks.yml` — `decisions-guard` base fetch | `UNSAFE-KNOWN` | Fetches the live base tip and diffs against it, unpinned, and unlike `docs-reminder` this job CAN fail — so its answer reaches the combined status the merge hook reads and therefore participates in consent. Accepted on the same grounds as the `api-docs`/`format` row: a base that moved mid-job produces a spurious FAILURE and a re-run, never a spurious pass, and it re-fetches fresh on every trigger with no state carried between runs. |
|
||||
| `.gitea/workflows/renovate.yml` — `renovate/renovate:43` | `N/A` | Out of this class, in scope for a different one — see the limits below. |
|
||||
| `.gitea/workflows/dependency-scan.yml` — NuGet advisory query | `N/A` | `dotnet list package --vulnerable` queries a live advisory database and fails the job on the report marker in the same step, so there is no check-then-act split. Worth one line anyway: a green here means "no advisories **as of this run**", which is a dated claim about mutable remote data rather than a property of the tree — which is why the scan runs on a weekly schedule instead of only on PRs. |
|
||||
|
||||
## Limits, stated rather than implied
|
||||
|
||||
1. **Unpinned dependencies are a different class and are not graded here.** `actions/checkout@v4`,
|
||||
`docker/build-push-action@v6`, `mysql:8.4` and `renovate/renovate:43` are floating tags, and
|
||||
`renovate.yml` runs its one with a repo-writing token. But resolution and execution are the same
|
||||
step — there is no earlier check whose answer a later action trusts — so they are a
|
||||
supply-chain-pinning concern, not a check-and-use race. Listing them as `N/A` here records that
|
||||
they were examined and classified, not that they are safe.
|
||||
|
||||
2. **The population is files, not call sites.** The test derives which *files* are in scope; it
|
||||
cannot tell that an existing file grew a second, unpinned read. That residue is the
|
||||
sites-in-code limit named in `testing.guard-derives-population-from-source` — it needs
|
||||
find-all-references tooling, tracked in #777 — and it is why the Note column is prose a reviewer
|
||||
reads rather than a field a script checks.
|
||||
|
||||
3. **Scope is hand-written; the population inside it is derived, with no content filter at all.**
|
||||
Scope is every file under `scripts/` (`*.sh`, `*.py`), `.claude/hooks/`, `.husky/` and
|
||||
`.gitea/workflows/` — a reviewed policy choice. Inside it, *every* file gets a row, and a file
|
||||
that reads no remote state earns an explicit `N/A` rather than silently staying out.
|
||||
|
||||
The first version filtered that scope by an outbound-network token list and argued the filter was
|
||||
a scope choice rather than a population filter. Cold review rejected the distinction, and the
|
||||
evidence settled it: the list omitted `git fetch`, which is this repo's most common remote read,
|
||||
so `prepush-rebase-check.sh` — which fetches `origin/main` and derives a **push decision** from
|
||||
it — was structurally invisible to a guard claiming to cover "every executable that reads live
|
||||
remote state", along with three others. The defence offered was that over-inclusion is the safe
|
||||
direction; the filter also *under*-included, which is the direction that costs a blind spot.
|
||||
Enumerating the directories costs more rows and has none.
|
||||
|
||||
4. **Nothing here checks that a `PINNED` claim is true.** The test asserts every in-scope file has a
|
||||
row and that the classifications come from a closed vocabulary. Whether a row is honest stays a
|
||||
review responsibility, and this table is what review reads — the same split, and the same
|
||||
admitted residue, as `docs/guard-inventory.md`.
|
||||
+36
-4
@@ -178,10 +178,42 @@ the #644 follow-up got Class A right and Class B only half right):
|
||||
|
||||
The helper owns both bounds: at most ONE `getLibraryBrowseItems` request per settled query, at most
|
||||
`LIBRARY_PICKER_RESULTS` (25) rows, and no request at all below `LIBRARY_PICKER_MIN_QUERY` (2)
|
||||
characters. Selecting a media-library type must issue **zero** requests. There is no truncation, so
|
||||
there is no truncation hint — the old `Showing the first 100 of 5000 — use search to narrow.` copy
|
||||
is gone from these pickers along with the window it described. Prove the bound with a
|
||||
**request-count assertion against a large (20k-row) fixture**, not by inspection.
|
||||
characters. Selecting a media-library type must issue **zero** requests. The per-kind
|
||||
`LIBRARY_PICKER_RESULTS` cap is the only truncation this class has — there is no whole-type window
|
||||
left to hint at, so the old `Showing the first 100 of 5000 — use search to narrow.` copy is gone
|
||||
from these pickers along with the window it described. Surfacing the per-kind cap is *permitted*
|
||||
wherever it is reachable, and *required* only where bulk selection makes the count actionable — see
|
||||
the `AddItemsDialog` sub-bullet below, which sums the cap across kinds and renders a `Showing N of
|
||||
M matches` hint for exactly that reason. Prove the bound with a **request-count assertion against a
|
||||
large (20k-row) fixture**, not by inspection.
|
||||
|
||||
- **`SearchPicker` is the single-select SHAPE, not the rule itself.** A MULTI-select picker
|
||||
(`CollectionsScreen`'s `AddItemsDialog` — checkbox rows, many items added at once, fanned out
|
||||
over several kinds) cannot render `SearchPicker` and must not be forced to. It satisfies this
|
||||
section by taking the same *constraints* the helper enforces for single-select — the gate on
|
||||
`LIBRARY_PICKER_MIN_QUERY`, `titleContainsQuery` the typed text, `LIBRARY_PICKER_RESULTS` per
|
||||
kind — via `searchLibraryBrowseItems` (`web/src/api/libraryBrowse.ts`), a sibling of
|
||||
`searchLibraryPickerOptions` that returns full `LibraryBrowseItem` rows plus `totalCount`
|
||||
instead of `{id, name}`, so the bound lives in the helper rather than the caller (#685 review
|
||||
finding 2). There is no post-fetch `slice`, but the per-kind cap can still truncate the real
|
||||
match count — this is a bulk multi-select add, where "add the 40 matching episodes" is a
|
||||
first-class use, so `AddItemsDialog` sums each kind's `totalCount` and renders a `Showing N of
|
||||
M matches` hint once it exceeds the rendered rows (finding 4 — an earlier revision of this
|
||||
bullet called the truncation nothing left to hint at). **The gate's home is the shared HELPER,
|
||||
not the screen — however single-sink the screen's own function looks.** #685 got this wrong
|
||||
twice in a row, and the second time is the instructive one: the check sat inside `runSearch`,
|
||||
which genuinely IS the one sink both entry paths route through, so it read as correct. It was
|
||||
still a duplicate of the helper's gate, and the two masked each other: as of `4be3f247d` —
|
||||
which had no unit tests on the helper — deleting EITHER copy left the whole suite green, so the
|
||||
min-query boundary test pinned nothing. Removing the screen's copy is what made the helper's
|
||||
gate load-bearing. **The invariant, not the count: every gate must have at least one test that
|
||||
reddens when that gate ALONE is removed.** A guard you cannot redden is not a guard, and "it's
|
||||
the single sink" is not evidence that it is the only one. **Outstanding on this screen**: `AddItemsDialog` still lacks the monotonic `seqRef`
|
||||
stale-response guard and `useIsMountedRef()` — the same class of guard "Debounced typeaheads"
|
||||
below mandates there, applied to a debounced-while-typing fetch; `AddItemsDialog` is an explicit
|
||||
Search-button submission, not a typeahead, so that mandate doesn't reach it directly, but the
|
||||
same race (a superseded search settling after a newer one) can still occur here — tracked in
|
||||
**ersatztv#740**, not yet fixed here.
|
||||
|
||||
- **Compile typed text; never forward raw Lucene.** Send `titleContainsQuery(text)` →
|
||||
`title:*<escaped>*`. The index's default field does not match bare title words (`Alpha` finds
|
||||
|
||||
@@ -0,0 +1,59 @@
|
||||
# Ruff configuration for this repo's Python surface (all of it lives under `scripts/`).
|
||||
#
|
||||
# WHY THIS FILE EXISTS (ersatztv#780). Without a committed config, ruff falls back to whatever
|
||||
# `~/.config/ruff/ruff.toml` the operator's machine happens to have — so a second machine lints this
|
||||
# repo differently, or not at all. That is the environment-divergence class #643/#647/#648 (a shell
|
||||
# gate whose behaviour was a function of an untested interpreter version) and #512 (a test that
|
||||
# passed on a fast laptop and flaked on a starved CI VM). The settings below are pinned HERE so the
|
||||
# lint verdict is a property of the repo, not of the machine.
|
||||
#
|
||||
# It is enforced by the `script-tests` job (`Script lint and tests (ruff + pytest)`) in
|
||||
# .gitea/workflows/pr-checks.yml. A config nobody runs is the same divergence one step later.
|
||||
#
|
||||
# That job does NOT invoke `ruff check .`: it passes an explicit population from `git ls-files` with
|
||||
# `--no-force-exclude`. An `exclude` added to this file silently empties a discovery-based run into a
|
||||
# GREEN one — a top-level `exclude` empties both commands, one under `[lint]` empties `check`, one
|
||||
# under `[format]` (where an appended line lands, by TOML rules) empties `format --check`. Adding
|
||||
# `exclude` here will therefore not do what you expect, which is the point. The measured matrix is in
|
||||
# `ci.python-lint-ruff-config-committed`.
|
||||
#
|
||||
# `pyright` is deliberately NOT gated: its only findings here are `reportMissingImports` for
|
||||
# `etv_client` in scripts/scripted-schedules/entrypoint.py, which resolves only inside that script's
|
||||
# deploy environment, and gating it would put a node toolchain on the git-only `small` lane for zero
|
||||
# real findings. Revisit if this repo grows a typed Python surface.
|
||||
|
||||
target-version = "py311"
|
||||
line-length = 120
|
||||
|
||||
[lint]
|
||||
select = [
|
||||
"E", # pycodestyle errors
|
||||
"W", # pycodestyle warnings
|
||||
"F", # pyflakes
|
||||
"I", # isort
|
||||
"B", # flake8-bugbear
|
||||
"UP", # pyupgrade
|
||||
"SIM", # flake8-simplify
|
||||
"S", # flake8-bandit (security)
|
||||
# RUF100 is load-bearing, not tidiness: every `# noqa` below is an assertion that a real finding
|
||||
# is being suppressed for a stated reason, and without this a suppression that suppresses nothing
|
||||
# stays in the file reading as one. #780 did exactly that mid-branch — a `# noqa: UP031` on a site
|
||||
# the same branch had already fixed in code — and found two more already in the tree: one whose
|
||||
# rule had stopped firing, one for a rule this config never enables.
|
||||
"RUF100",
|
||||
]
|
||||
ignore = [
|
||||
"S603", # subprocess call - check for execution of untrusted input (too noisy for scripts)
|
||||
"S607", # starting a process with a partial executable path
|
||||
]
|
||||
|
||||
[lint.per-file-ignores]
|
||||
# scripts/tests asserts, so S101 would fire on every test. S105 is deliberately NOT exempted here:
|
||||
# the eight sites that trip it (`env["ETV_GITEA_TOKEN"] = "stub"`) carry a per-site `# noqa: S105`
|
||||
# instead, so a real credential pasted into a fixture next year still reddens the gate. A directory
|
||||
# blanket would have given up hardcoded-credential coverage over the largest Python surface in the
|
||||
# repo, permanently, to suppress eight known lines.
|
||||
"scripts/tests/**" = ["S101"]
|
||||
|
||||
[format]
|
||||
quote-style = "double"
|
||||
Executable
+260
@@ -0,0 +1,260 @@
|
||||
#!/usr/bin/env python3
|
||||
"""ersatztv#784 — ADVISORY nudge for `docs.no-session-narrative`.
|
||||
|
||||
A doc records the END STATE; the path to it belongs in the commit message, not the artifact.
|
||||
|
||||
THIS NEVER FAILS. Every path returns exit 0 — including a bad argument, an unresolvable base ref,
|
||||
an unreadable file and an unhandled exception. That is a design constraint, not an oversight: a
|
||||
narrative detector is a string predicate over prose, and `docs/defect-shapes-773.md` §4 plus
|
||||
`testing.guard-derives-population-from-source` both argue that class must not be load-bearing (the
|
||||
withdrawn `test_review_verdict_vocabulary_parity.py` — six review rounds, then deleted — is the
|
||||
empirical case). Do not convert this into a gate; the decision record says no in as many words.
|
||||
|
||||
WHY PYTHON AND NOT SHELL. The first implementation hand-parsed `git diff -U0` output in bash by
|
||||
matching line prefixes, and cold review demonstrated four separate defects in that one parser: the
|
||||
`\\ No newline at end of file` marker was counted as content, an added line whose own text began
|
||||
`++ ` was eaten by the `+++ ` header arm, `core.quotePath` hid non-ASCII paths, and `read` dropped a
|
||||
final unterminated line. Those are four instances of one mistake — deciding what a diff line IS from
|
||||
its prefix alone, with no hunk state. Patching them one at a time is the shape this repo has
|
||||
recorded as never converging, so the mechanism was replaced rather than the sites.
|
||||
|
||||
check-doc-narrative.py --diff <base-ref> scan lines ADDED against <base-ref> (the CI mode)
|
||||
check-doc-narrative.py --all scan the whole tracked corpus (deliberate sweep)
|
||||
|
||||
Scanning only ADDED lines in CI is what keeps the existing corpus of legitimate history out of the
|
||||
output; `--all` deliberately reports all of it, for a human to apply the who-benefits test to.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
# The never-fails invariant must not depend on the ambient locale. Both the summary line and the
|
||||
# last-resort handler below carry non-ASCII text, so under ascii/latin-1 stdio the very code meant to
|
||||
# guarantee exit 0 is what raises. Degrade unencodable characters instead of failing on them.
|
||||
try:
|
||||
sys.stdout.reconfigure(errors="replace") # type: ignore[union-attr]
|
||||
sys.stderr.reconfigure(errors="replace") # type: ignore[union-attr]
|
||||
except Exception: # noqa: S110 — a stdout that cannot be reconfigured is not a reason to fail
|
||||
pass # deliberate: this is the never-fails invariant's own setup, so it cannot itself raise
|
||||
|
||||
# `docs/decisions/**` is exempt WHOLESALE — a decision record narrating how a rule was got wrong is
|
||||
# carrying the rationale it exists to carry, so a detector that flagged it would fight the
|
||||
# convention it serves.
|
||||
EXEMPT_PREFIXES = ("docs/decisions/",)
|
||||
|
||||
# Session-narrative phrasings. Deliberately narrow: each is first person or names a revision of THIS
|
||||
# artifact. Broad words that also appear in legitimate dated history ("previously", "was wrong") are
|
||||
# absent on purpose — a false positive on a carved-out case is what makes an advisory check stop
|
||||
# being read.
|
||||
PATTERNS = re.compile(
|
||||
r"an earlier draft"
|
||||
r"|earlier drafts"
|
||||
r"|the (?:first|previous|original) (?:version|draft) of (?:this|the)"
|
||||
r"|my first attempt"
|
||||
r"|I (?:initially|first|originally|then) (?:thought|assumed|tried|wrote|found)"
|
||||
r"|we (?:then|initially) (?:found|thought|realis|realiz)"
|
||||
r"|it turned out that"
|
||||
r"|earlier today"
|
||||
r"|as of just now"
|
||||
r"|currently investigating",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
WARNING = (
|
||||
"{path}:{line} reads as session narrative — a reader coming cold never saw the earlier draft. "
|
||||
"Answer the review finding in the COMMIT MESSAGE and let only the corrected claim enter the "
|
||||
"doc (docs.no-session-narrative). Keep it only if a reader would ACT differently knowing it "
|
||||
"(dated measurement, stated snapshot boundary, tested-and-rejected result, a trap and its "
|
||||
"consequence). Line: {text}"
|
||||
)
|
||||
|
||||
|
||||
def is_scanned_path(path: str) -> bool:
|
||||
"""The population: `docs/**/*.md` minus `docs/decisions/**`, plus root-level `*.md`.
|
||||
|
||||
Stated positively and in one place so the record's `mechanics:` can quote it exactly. Skills,
|
||||
`web/`, and other nested markdown outside `docs/` are deliberately NOT in scope.
|
||||
"""
|
||||
if not path.endswith(".md"):
|
||||
return False
|
||||
if any(path.startswith(p) for p in EXEMPT_PREFIXES):
|
||||
return False
|
||||
return path.startswith("docs/") or "/" not in path
|
||||
|
||||
|
||||
# Git's OUTPUT FORMAT is configurable, and this script reads paths and line numbers out of that
|
||||
# format. Three separate knobs were each demonstrated turning a real hit into `scanned 0 file(s)` —
|
||||
# `core.quotePath` hiding non-ASCII paths, `diff.dstPrefix` rewriting the header, `color.diff=always`
|
||||
# injecting ANSI escapes. Pinning them one at a time is refuting variants, not clearing the channel,
|
||||
# so the channel is closed at both ends: the user's and the system's config files are taken out of
|
||||
# the picture entirely (which also covers knobs nobody has thought of yet), and the handful that a
|
||||
# REPO-local config could still set are pinned explicitly on the command line, where they win.
|
||||
GIT_ENV_OVERRIDES = {
|
||||
"GIT_CONFIG_GLOBAL": os.devnull,
|
||||
"GIT_CONFIG_SYSTEM": os.devnull,
|
||||
"GIT_CONFIG_NOSYSTEM": "1",
|
||||
}
|
||||
# `core.quotePath=false` is witnessed by a test. The env overrides above are NOT, and cannot be: they
|
||||
# exist for the knob nobody has named yet, which is exactly what three rounds of naming one knob at a
|
||||
# time argued for. Every knob that IS named is pinned on the command line, where it also beats a
|
||||
# repo-local config, and has a row in FORMAT_KNOBS in the test file.
|
||||
GIT_CONFIG_PINS = ("-c", "core.quotePath=false")
|
||||
|
||||
|
||||
def git(*args: str) -> tuple[int, str]:
|
||||
"""Run git with its output format pinned. Returns (returncode, stdout) and never raises."""
|
||||
try:
|
||||
p = subprocess.run(
|
||||
["git", *GIT_CONFIG_PINS, *args],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
errors="replace",
|
||||
env={**os.environ, **GIT_ENV_OVERRIDES},
|
||||
)
|
||||
except OSError as exc: # git missing, or not a repo we can exec in
|
||||
return 1, f"{exc}"
|
||||
if p.returncode != 0:
|
||||
sys.stderr.write(p.stderr)
|
||||
return p.returncode, p.stdout
|
||||
|
||||
|
||||
def scan_line(path: str, lineno: int, text: str, out: list[str]) -> None:
|
||||
if PATTERNS.search(text):
|
||||
detail = WARNING.format(path=path, line=lineno, text=text[:160])
|
||||
out.append(f"::warning file={path}::{detail}")
|
||||
|
||||
|
||||
def added_lines(diff: str):
|
||||
"""Yield (path, lineno, text) for every ADDED line in a unified diff.
|
||||
|
||||
A line's meaning comes from HUNK STATE, not from its prefix: `+++ ` is a header only before the
|
||||
first `@@` of a file, and inside a hunk it is content whose own text starts `++ `. That
|
||||
distinction is the whole reason this is not a prefix match.
|
||||
"""
|
||||
path = None
|
||||
in_hunk = False
|
||||
lineno = 0
|
||||
for raw in diff.split("\n"):
|
||||
if raw.startswith("diff --git "):
|
||||
path, in_hunk = None, False
|
||||
elif raw.startswith("@@"):
|
||||
m = re.match(r"@@ -\d+(?:,\d+)? \+(\d+)(?:,\d+)? @@", raw)
|
||||
if not m:
|
||||
in_hunk = False
|
||||
continue
|
||||
in_hunk = True
|
||||
lineno = int(m.group(1)) - 1
|
||||
elif not in_hunk:
|
||||
if raw.startswith("+++ "):
|
||||
p = raw[4:]
|
||||
# `/dev/null` on the new side means the file was DELETED. Stated honestly: this arm
|
||||
# is DEFENSIVE, not load-bearing — a deletion contributes no `+` lines, so nothing is
|
||||
# yielded for it either way, and removing this arm reddens no test. It is kept because
|
||||
# `path` should never name a file the added lines do not belong to. A
|
||||
# `--diff-filter=d` on the git call was removed rather than kept beside it: a second
|
||||
# mechanism nobody can witness failing is how a duplicate guard hides its twin.
|
||||
path = None if p == "/dev/null" else (p[2:] if p.startswith("b/") else p)
|
||||
elif raw.startswith("+"):
|
||||
lineno += 1
|
||||
if path is not None:
|
||||
yield path, lineno, raw[1:]
|
||||
elif raw.startswith("-") or raw.startswith("\\"):
|
||||
pass # a removed line, or the no-trailing-newline marker: neither advances the new file
|
||||
else:
|
||||
lineno += 1 # context (absent at -U0, but harmless and correct if -U grows)
|
||||
|
||||
|
||||
def run_diff(base: str, out: list[str]) -> int:
|
||||
rc, diff = git(
|
||||
"diff",
|
||||
"-U0",
|
||||
# Pinned, not decorative: `diff.renames=false` in a developer's gitconfig turns a `git mv`
|
||||
# into a whole-file add and re-flags every pre-existing line. Same channel as the prefixes.
|
||||
"--find-renames",
|
||||
# Pin the header shape the path is parsed out of. `diff.noprefix`, `diff.srcPrefix` and
|
||||
# `diff.dstPrefix` each rewrite it from a developer's gitconfig, and `diff.external` replaces
|
||||
# the output entirely — a configured prefix silently produced a scanned-0-files clean run.
|
||||
"--src-prefix=a/",
|
||||
"--dst-prefix=b/",
|
||||
"--no-ext-diff",
|
||||
"--no-color",
|
||||
f"{base}...HEAD",
|
||||
)
|
||||
if rc != 0:
|
||||
print(
|
||||
f"doc-narrative: could not diff against '{base}' — SCANNED NOTHING. "
|
||||
"This is reported rather than swallowed: a silent zero-file scan is indistinguishable "
|
||||
"from a clean one, which is the failure `ci.required-job-step-execution-markers` exists for."
|
||||
)
|
||||
return -1
|
||||
scanned = set()
|
||||
for path, lineno, text in added_lines(diff):
|
||||
if not is_scanned_path(path):
|
||||
continue
|
||||
scanned.add(path)
|
||||
scan_line(path, lineno, text, out)
|
||||
return len(scanned)
|
||||
|
||||
|
||||
def run_all(out: list[str]) -> int:
|
||||
# Population from `git ls-files`, never a filesystem walk — an untracked scratch file is not
|
||||
# part of the corpus (#778).
|
||||
rc, listing = git("ls-files", "-z", "--", "*.md")
|
||||
if rc != 0:
|
||||
print("doc-narrative: could not list tracked files — SCANNED NOTHING.")
|
||||
return -1
|
||||
scanned = 0
|
||||
for path in listing.split("\0"):
|
||||
if not path or not is_scanned_path(path):
|
||||
continue
|
||||
try:
|
||||
with open(path, encoding="utf-8", errors="replace") as fh:
|
||||
text = fh.read()
|
||||
except OSError as exc:
|
||||
# A tracked-but-deleted doc is an ordinary working state, not a reason to fail.
|
||||
print(f"doc-narrative: skipped {path} ({exc.strerror}).")
|
||||
continue
|
||||
scanned += 1
|
||||
# splitlines() keeps a final unterminated line, which `read`-per-line dropped.
|
||||
for i, line in enumerate(text.splitlines(), start=1):
|
||||
scan_line(path, i, line, out)
|
||||
return scanned
|
||||
|
||||
|
||||
def main(argv: list[str]) -> int:
|
||||
mode = argv[1] if len(argv) > 1 else "--all"
|
||||
out: list[str] = []
|
||||
|
||||
if mode == "--diff":
|
||||
if len(argv) < 3 or not argv[2]:
|
||||
print("doc-narrative: --diff needs a base ref — SCANNED NOTHING. (advisory; not a failure)")
|
||||
return 0
|
||||
scanned = run_diff(argv[2], out)
|
||||
elif mode == "--all":
|
||||
scanned = run_all(out)
|
||||
else:
|
||||
print(f"doc-narrative: unknown mode '{mode}'. usage: {argv[0]} [--all | --diff <base-ref>]")
|
||||
return 0
|
||||
|
||||
for line in out:
|
||||
print(line)
|
||||
if scanned < 0:
|
||||
return 0
|
||||
print(
|
||||
f"doc-narrative: scanned {scanned} file(s); {len(out)} advisory warning(s). "
|
||||
"NON-BLOCKING — this check never fails a run."
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
sys.exit(main(sys.argv))
|
||||
# The never-fails constraint outranks a clean traceback, so this catch is deliberately blind.
|
||||
except Exception as exc:
|
||||
print(f"doc-narrative: internal error ({exc!r}) — SCANNED NOTHING. Advisory; not a failure.")
|
||||
sys.exit(0)
|
||||
Executable
+131
@@ -0,0 +1,131 @@
|
||||
#!/usr/bin/env bash
|
||||
# Verify the local code-intelligence toolchain (ersatztv#777).
|
||||
#
|
||||
# Operator-run, NOT a CI gate: every dependency it checks is a developer-machine
|
||||
# install (Homebrew dotnet, a global csharp-ls, web/node_modules), so there is no
|
||||
# runner on which a red here would mean anything. It exists because the #777 root
|
||||
# causes were both environment divergence — a config that silently pointed at a
|
||||
# path this machine does not have — and that class is invisible until something
|
||||
# looks. See docs/local-lsp-tooling.md.
|
||||
#
|
||||
# Exit 0 = every check ran AND passed. Exit 1 = at least one FAIL *or* SKIP —
|
||||
# a skipped check is a non-result, not a success.
|
||||
|
||||
set -uo pipefail
|
||||
|
||||
REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
failures=0
|
||||
checks=0
|
||||
skipped=0
|
||||
|
||||
pass() { checks=$((checks + 1)); printf ' PASS %s\n' "$1"; }
|
||||
fail() { checks=$((checks + 1)); failures=$((failures + 1)); printf ' FAIL %s\n' "$1"; }
|
||||
# A skip is NOT a pass. Counting it as one would let the summary read "all checks
|
||||
# passed" while the most important check never ran.
|
||||
skip() { checks=$((checks + 1)); skipped=$((skipped + 1)); printf ' SKIP %s\n' "$1"; }
|
||||
note() { printf ' %s\n' "$1"; }
|
||||
|
||||
echo "Local LSP toolchain check — $REPO_ROOT"
|
||||
echo
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
echo "csharp-ls (the LSP tool's C# server)"
|
||||
|
||||
if ! command -v csharp-ls >/dev/null 2>&1; then
|
||||
fail "csharp-ls is not on PATH"
|
||||
note "install: dotnet tool install --global csharp-ls"
|
||||
else
|
||||
pass "csharp-ls on PATH ($(command -v csharp-ls))"
|
||||
fi
|
||||
|
||||
# MSBuildLocator resolves the SDK next to the dotnet host it finds, and needs a
|
||||
# sibling host/fxr. Homebrew's bin/dotnet has none — its real root is libexec.
|
||||
dotnet_root="${DOTNET_ROOT:-}"
|
||||
if [ -z "$dotnet_root" ]; then
|
||||
if [ -n "${DOTNET_HOST_PATH:-}" ]; then
|
||||
dotnet_root="$(dirname "$DOTNET_HOST_PATH")"
|
||||
elif command -v dotnet >/dev/null 2>&1; then
|
||||
dotnet_root="$(dirname "$(readlink -f "$(command -v dotnet)" 2>/dev/null || command -v dotnet)")"
|
||||
fi
|
||||
fi
|
||||
|
||||
if [ -z "$dotnet_root" ]; then
|
||||
fail "no dotnet host could be resolved (DOTNET_ROOT, DOTNET_HOST_PATH, PATH all empty)"
|
||||
elif compgen -G "$dotnet_root/host/fxr/*/libhostfxr.*" >/dev/null 2>&1; then
|
||||
pass "dotnet root has host/fxr ($dotnet_root)"
|
||||
else
|
||||
fail "dotnet root has NO host/fxr — csharp-ls cannot initialize ($dotnet_root)"
|
||||
note "this is the #777 failure: set DOTNET_ROOT to the root that owns host/fxr,"
|
||||
note "e.g. /opt/homebrew/opt/dotnet/libexec on a Homebrew install."
|
||||
note "Claude Code picks it up from .claude/settings.local.json -> env.DOTNET_ROOT"
|
||||
note "(local, not the tracked settings.json — the value is machine-specific)."
|
||||
fi
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
echo
|
||||
echo "typescript-language-server (the LSP tool's TS server)"
|
||||
|
||||
if ! command -v typescript-language-server >/dev/null 2>&1; then
|
||||
fail "typescript-language-server is not on PATH"
|
||||
note "install: npm install -g typescript-language-server"
|
||||
else
|
||||
pass "typescript-language-server on PATH"
|
||||
fi
|
||||
|
||||
# v5 has no --tsserver-path and the plugin cannot pass initializationOptions, so
|
||||
# the ONLY lever is making `typescript` resolvable from the workspace root, which
|
||||
# for Claude Code is the repo root — not web/.
|
||||
if [ -e "$REPO_ROOT/node_modules/typescript/lib/tsserver.js" ]; then
|
||||
pass "typescript resolvable from the repo root"
|
||||
else
|
||||
fail "typescript NOT resolvable from the repo root — the TS server will refuse to start"
|
||||
note "the package lives in web/node_modules; link it at the root:"
|
||||
note " mkdir -p '$REPO_ROOT/node_modules'"
|
||||
note " ln -sfn '$REPO_ROOT/web/node_modules/typescript' '$REPO_ROOT/node_modules/typescript'"
|
||||
fi
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
echo
|
||||
echo "csharp-lsp MCP server (the C# path subagents can actually reach)"
|
||||
|
||||
mcp_json="$REPO_ROOT/.mcp.json"
|
||||
if [ ! -f "$mcp_json" ]; then
|
||||
fail ".mcp.json not present (it is gitignored — see docs/local-lsp-tooling.md to recreate it)"
|
||||
elif [ "${SKIP_MCP_SMOKE:-0}" = "1" ]; then
|
||||
skip "csharp-lsp MCP server (SKIP_MCP_SMOKE=1) — NOT verified"
|
||||
elif ! command -v perl >/dev/null 2>&1 || ! command -v python3 >/dev/null 2>&1; then
|
||||
fail "cannot run the MCP smoke test: perl and python3 are both required"
|
||||
else
|
||||
# Actually START the server and list its tools. Checking `[ -x command ]` instead
|
||||
# would be vacuous: a DIRECTORY satisfies -x (`[ -x /bin ]` is true), so that
|
||||
# predicate can report a pass for a server that cannot run at all.
|
||||
#
|
||||
# `exec @ARGV or die` is load-bearing. Without the `or die`, perl exits 0 when it
|
||||
# cannot exec the command, the `if` reads that as success, and this branch prints
|
||||
# a PASS having run nothing — the same false green one level up.
|
||||
note "starting the MCP server (this takes ~30-60s on a cold build)..."
|
||||
if smoke_out="$(perl -e 'alarm shift; exec @ARGV or die "exec failed: $!\n"' 240 \
|
||||
python3 "$REPO_ROOT/scripts/mcp_smoke.py" "$mcp_json" csharp-lsp 200 \
|
||||
--expect-server csharp-lsp-mcp \
|
||||
--expect-tool csharp_set_workspace --expect-tool csharp_references 2>&1)"; then
|
||||
pass "csharp-lsp MCP server starts and serves tools — ${smoke_out#OK: }"
|
||||
else
|
||||
fail "csharp-lsp MCP server did not come up"
|
||||
note "${smoke_out:-(no output — the smoke test itself was killed)}"
|
||||
note "#777's failure was an entry naming a dotnet install that no longer existed;"
|
||||
note "a net8.0-targeted vendored clone on an SDK-10-only host fails the same way."
|
||||
fi
|
||||
fi
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
echo
|
||||
passed=$(( checks - failures - skipped ))
|
||||
if [ "$failures" -eq 0 ] && [ "$skipped" -eq 0 ]; then
|
||||
echo "All $checks checks passed."
|
||||
elif [ "$failures" -eq 0 ]; then
|
||||
echo "$passed passed, $skipped SKIPPED of $checks — a skipped check is not a passed one."
|
||||
else
|
||||
echo "$failures of $checks checks FAILED ($skipped skipped) — see docs/local-lsp-tooling.md."
|
||||
fi
|
||||
# A skip is a non-result, so it is not success: exit non-zero unless everything ran.
|
||||
exit $(( (failures > 0 || skipped > 0) ? 1 : 0 ))
|
||||
Executable
+104
@@ -0,0 +1,104 @@
|
||||
#!/usr/bin/env bash
|
||||
# Prove — at runtime, every run — that the delimiter ban actually DETECTS a delimiter (ersatztv#767).
|
||||
#
|
||||
# WHY THIS EXISTS AS A SHELL STEP RATHER THAN A TEST. The `scan` job's guarantee used to be a pytest
|
||||
# that asserted the ban command works. Review then disarmed the whole gate with ONE new file at the
|
||||
# repo root — a `pytest.ini` carrying `addopts = -k "not delimiter_banned"`, or a `conftest.py` with
|
||||
# `pytest_collection_modifyitems` — which deselects the ban test AND the tests guarding it, leaving
|
||||
# every job green while a delimiter sits in `build`'s `Smoke` body and the image publishes. That tier
|
||||
# sits above every `env:` a test can reconstruct, and above the tests themselves: anything living
|
||||
# inside pytest can be deselected by pytest's own configuration. So the positive control lives here.
|
||||
#
|
||||
# IT RUNS IN THE REAL CHECKOUT, NOT A COPY, and that is the whole point. An isolated copy does not
|
||||
# inherit the repo-root `pytest.ini`/`conftest.py`/`pyproject.toml` that a disarm would live in, so a
|
||||
# copy-based proof reports the ban healthy while the job's actual invocation is deselected — measured,
|
||||
# and it is why the first version of this script was wrong. Same cwd, same rootdir, same config and
|
||||
# the same BAN INVOCATION as the run being vouched for — not the same command line, since the
|
||||
# workflow step also runs the wiring tests and this runs only the ban file. Only the workflow file
|
||||
# differs, and only for a moment.
|
||||
#
|
||||
# The workflow file is restored by an EXIT trap. That covers a normal exit and SIGTERM (measured:
|
||||
# `git status` clean afterwards); it does NOT cover SIGKILL — a cancelled or OOM-killed job leaves
|
||||
# the checkout poisoned. Do not upgrade this to "cannot leave a poisoned tree behind": it can.
|
||||
# It stays harmless for a reason worth stating rather than assuming. The workspace that could be
|
||||
# reused is THIS job's own, on the persistent `small` lane (`scan` has no `container:` — see the
|
||||
# carve-out in `ci.required-job-step-execution-markers`), not `build`'s, which checks out fresh in a
|
||||
# separate job. A leftover poisoned workflow there makes the NEXT `scan` red rather than quiet, and
|
||||
# `actions/checkout` restores it anyway. Fail-closed in both directions.
|
||||
#
|
||||
# The failure direction is the safe one throughout: if this script cannot do its work (no python3, a
|
||||
# moved anchor, an unwritable tree) it exits non-zero and the release is blocked, never silently
|
||||
# skipped — `set -euo pipefail` extends that to the unexpected cases.
|
||||
#
|
||||
# AND IT ONLY VOUCHES FOR THE EXACT EXPECTED FAILURE. Three outcomes are distinguished below: pytest
|
||||
# passing (the ban is not enforcing), pytest failing for a NON-test reason (collection error, nothing
|
||||
# collected — cannot prove anything), and pytest failing on some OTHER test while the ban test itself
|
||||
# was deselected. Only the ban test's `[build]` parametrisation failing counts as proof. Both weaker
|
||||
# readings were live bugs in earlier drafts of this file: "non-zero means it noticed" reported OK on a
|
||||
# collection error, and a later draft still fell through to OK when an unrelated test reddened.
|
||||
#
|
||||
# NOTE ON THE OPENER: this file must never contain the literal two-character expression opener, both
|
||||
# because `scan`'s own run bodies are asserted delimiter-free and because a file containing it is the
|
||||
# very hazard under test. It is CONSTRUCTED from its parts below.
|
||||
set -euo pipefail
|
||||
|
||||
BAN_TEST="scripts/tests/test_ci_dropped_step_guard.py"
|
||||
WORKFLOW=".gitea/workflows/docker-build.yml"
|
||||
# The line the poison is injected above, inside `build`'s `Smoke + IPTV E2E` body. Kept in one place
|
||||
# so that rewriting that step fails loudly here instead of silently poisoning nothing.
|
||||
ANCHOR='IMG="${IMAGE}:${SMOKE_SHORT_SHA}"'
|
||||
|
||||
cd "${GITHUB_WORKSPACE:-$(pwd)}"
|
||||
|
||||
backup="$(mktemp)"
|
||||
cp "$WORKFLOW" "$backup"
|
||||
restore() { cp "$backup" "$WORKFLOW"; rm -f "$backup"; }
|
||||
trap restore EXIT
|
||||
|
||||
python3 - "$WORKFLOW" "$ANCHOR" <<'PY'
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
path, anchor = Path(sys.argv[1]), sys.argv[2]
|
||||
text = path.read_text()
|
||||
if anchor not in text:
|
||||
sys.exit(
|
||||
"::error::ci-prove-ban-detects.sh could not find its anchor in the workflow. The Smoke step "
|
||||
"was rewritten without updating this script, so the proof would poison nothing and pass "
|
||||
"vacuously. Refusing."
|
||||
)
|
||||
# Built from parts on purpose: the literal opener must not appear in this repo's own scripts.
|
||||
opener = chr(36) + "{" + "{"
|
||||
path.write_text(text.replace(anchor, f"# {opener} steps.meta.outputs.short }}}}\n {anchor}", 1))
|
||||
PY
|
||||
|
||||
echo "Proving the delimiter ban detects an injected opener (ersatztv#767)…"
|
||||
output="$(mktemp)"
|
||||
trap 'restore; rm -f "$output"' EXIT
|
||||
set +e
|
||||
PYTHONPATH=. python3 -m pytest "$BAN_TEST" -q >"$output" 2>&1
|
||||
detected=$?
|
||||
set -e
|
||||
|
||||
# THREE OUTCOMES, EXHAUSTIVELY, and only the last one is a pass. "Non-zero means it noticed" is not
|
||||
# good enough — that was this script's own first bug, and an earlier draft of this very block still
|
||||
# fell through to OK when pytest exited 1 for a DIFFERENT test's failure while the ban test was
|
||||
# deselected. Requiring the exact expected failure is what makes this a proof rather than a liveness
|
||||
# check. (`[build]` because the ban is parametrised per job, and `build` is the one whose dropped
|
||||
# `Smoke` publishes an unsmoked image.)
|
||||
EXPECTED='test_the_delimiter_banned_jobs_have_NO_expression_delimiter_in_any_run_body\[build\]'
|
||||
|
||||
if [ "$detected" -eq 0 ]; then
|
||||
echo "::error::THE DELIMITER BAN IS NOT ENFORCING. ${BAN_TEST} PASSED on a tree whose build/Smoke body contains an expression delimiter, so the release-path gate would let an unsmoked candidate publish. Causes seen before: a repo-root pytest.ini/conftest.py/pyproject.toml deselecting it, a neutered opener regex, 'build' dropped from DELIMITER_BAN_JOBS, or the test renamed. This check runs in the real checkout and outside pytest on purpose — anything inside pytest can be deselected by pytest's own configuration." >&2
|
||||
sed 's/^/ /' "$output" >&2
|
||||
exit 1
|
||||
elif [ "$detected" -ne 1 ]; then
|
||||
echo "::error::ci-prove-ban-detects.sh could not prove anything: pytest exited ${detected} on the poisoned tree, which is not a test failure (2=collection error, 3=internal, 4=usage, 5=nothing collected). The ban may well be fine, but this check cannot vouch for it, so it refuses rather than reporting a pass it did not earn." >&2
|
||||
sed 's/^/ /' "$output" >&2
|
||||
exit 1
|
||||
elif ! grep -qE "$EXPECTED" "$output"; then
|
||||
echo "::error::ci-prove-ban-detects.sh saw a test failure on the poisoned tree, but NOT the expected one. The ban test's build parametrisation did not fail, so something else did — the ban itself may be deselected or renamed while an unrelated test reddens. Refusing to read that as proof." >&2
|
||||
sed 's/^/ /' "$output" >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "OK: the ban detected the injected delimiter in build/Smoke (pytest exit 1, build parametrisation failed) — the gate is enforcing."
|
||||
Executable
+255
@@ -0,0 +1,255 @@
|
||||
#!/usr/bin/env bash
|
||||
# Per-step execution markers for the two REQUIRED docker-build.yml jobs (ersatztv#756).
|
||||
#
|
||||
# WHY THIS EXISTS. A `run:` body the runner declines to interpolate is DROPPED, and the job still
|
||||
# concludes `success` (ersatztv#751, `ci.workflow-run-body-no-expressions`). In
|
||||
# `review-verdict.yml` that is fail-CLOSED — the required `review-verdict/h10` is simply absent and
|
||||
# the merge is blocked. In `docker-build.yml` it is fail-OPEN: `Build & test (.NET)` and
|
||||
# `EF migration integrity (SQLite + MySql)` are the other two required contexts on `main`, so a
|
||||
# dropped step there sends a required check GREEN having done no work. #751 guarded the safe
|
||||
# direction because that is where the live bug was, not because these were checked.
|
||||
#
|
||||
# WHY PER STEP, NOT PER JOB, which is what #756 proposed. A marker written by the job's FIRST step
|
||||
# only proves the job started. The dangerous drop is not step 1 — it is `Test`, or the migration
|
||||
# replay: the job runs everything around them, reports green, and nothing ran that anyone cared
|
||||
# about. A guard that cannot see the fail-open case it was built for is the "guard that never
|
||||
# executed" failure one level up. So every consequential step marks itself and a trailing guard
|
||||
# asserts the whole expected SET.
|
||||
#
|
||||
# THAT GUARD CARRIES NO `if:` — unlike the #751 one, which uses `if: always()` because its job has a
|
||||
# single real step. These jobs have a dozen, and a genuine early failure legitimately skips every
|
||||
# later step, so `always()` would print a false "these steps never executed" on top of every ordinary
|
||||
# red build. The default `success()` is the wanted condition: the guard is skipped only when an
|
||||
# earlier step FAILED, which already fails the job, so guard-skipped implies job-red and every green
|
||||
# path runs the guard.
|
||||
#
|
||||
# WHY A SCRIPT AND NOT AN INLINE BODY, unlike the #751 guard. Two reasons, and the second is the
|
||||
# load-bearing one:
|
||||
#
|
||||
# * The path literal exists ONCE. The #751 guard carries it twice (write + assert) and its tests
|
||||
# spend real effort proving the two copies agree, because a divergence reddens every run and
|
||||
# then gets deleted as broken. Here they cannot diverge.
|
||||
# * A one-line `run: scripts/ci-step-ran.sh …` CANNOT CONTAIN AN EXPRESSION DELIMITER, so the
|
||||
# mechanism this guards against cannot drop the guard itself. #751's own record names this as
|
||||
# the stronger construction ("the body would have had to move into scripts/, where a one-line
|
||||
# run: makes the class unreachable") and settled for inline only because the measurement showed
|
||||
# it was not required there.
|
||||
#
|
||||
# WHY A SCRIPT IS ACCEPTABLE HERE THOUGH IT WOULD NOT BE IN review-verdict.yml. That workflow
|
||||
# checks out the PR's BASE precisely so a PR cannot supply the code that judges it. `docker-build.yml`
|
||||
# is head-resolved by design — a PR already supplies every test this job runs — so calling a script
|
||||
# from the head adds no authority a PR did not already have. This is a CORRECTNESS gate against
|
||||
# silent no-ops, not a security gate against a hostile PR; that job belongs to `review-verdict/h10`.
|
||||
# Do not copy this reasoning back into the gate workflow.
|
||||
#
|
||||
# THE MARKER FILE IS KEYED ON THE RUN, and BE PRECISE ABOUT WHY — the obvious justification is a
|
||||
# #751 measurement that does NOT transfer to these jobs, and saying so is the point. #751 measured
|
||||
# `RUNNER_TEMP` to be `/tmp` and called it "not a private per-job directory"; that was taken on
|
||||
# `review-verdict.yml`, which runs WITHOUT a `container:`. `test` and `migrations` run INSIDE the CI
|
||||
# toolchain image, so their `/tmp` is the job container's own and starts empty. That follows from
|
||||
# `container:`, NOT from a measurement: the build-lane probe confirmed only that `RUNNER_TEMP` is
|
||||
# `/tmp` here (the marker landed at `/tmp/etv-ci-steps-ran-test-1910-1`) — it says nothing about the
|
||||
# directory being private or empty, and an earlier draft of this comment cited it as though it did.
|
||||
# The fresh container is what actually rules out a stale marker here; the keying is defence in depth.
|
||||
#
|
||||
# It is kept because container-per-job is a property of how the lane is configured today, not a
|
||||
# guarantee, and a STALE marker is the one failure that makes this guard PASS on a run whose step was
|
||||
# dropped — a silent success, i.e. the exact thing being removed. Cheap insurance against a lane
|
||||
# change nobody would think to re-check this against.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
usage() {
|
||||
cat >&2 <<'EOF'
|
||||
usage:
|
||||
ci-step-ran.sh mark <key>
|
||||
Record that this step began executing. Call it as the step's FIRST act, before
|
||||
anything in the body can fail.
|
||||
|
||||
ci-step-ran.sh assert --always <key>... [--gated <key>...]
|
||||
Fail unless every expected key was marked. --always keys are always required.
|
||||
--gated keys are required only when the job's skip gates did NOT fire, read from
|
||||
ETV_DOCS_ONLY / ETV_REVALIDATE_SKIP so this mirrors the steps' own `if:`.
|
||||
EOF
|
||||
exit 2
|
||||
}
|
||||
|
||||
# NO SILENT FALLBACK FOR THE RUN IDENTITY — found by cold review. The first version defaulted to
|
||||
# `nojob`/`norunid`/`1`, and those are REUSABLE: with `GITHUB_RUN_ID` unset, every run on the host
|
||||
# would share ONE marker file, so a leftover from any earlier run would satisfy the guard on a run
|
||||
# whose step was dropped. A silent PASS — the exact failure the keying exists to remove, reintroduced
|
||||
# by the code meant to implement it.
|
||||
#
|
||||
# THE TWO HALVES ARE TREATED DIFFERENTLY, ON EVIDENCE, because the blast radii differ and this is a
|
||||
# REQUIRED check — a wrong refusal deadlocks every merge, so strictness is not free:
|
||||
#
|
||||
# * `GITHUB_JOB` and `GITHUB_RUN_ID` are MEASURED present on this runner (#756's build-lane probe
|
||||
# wrote `/tmp/etv-ci-steps-ran-test-1910-1`; `test` is the job id and 1910 is the real API run
|
||||
# id). Absence would mean the runner changed under us, so refusing is safe AND correct.
|
||||
# * `GITHUB_RUN_ATTEMPT` is measured present TOO, as of ersatztv#756's own PR run — but note how,
|
||||
# because the first two attempts to settle it were both bad. Grepping a job log for the variable
|
||||
# NAME proves nothing (logs do not dump the environment). Inferring it from the ABSENCE of this
|
||||
# script's "not set" warning proves nothing either, because that warning goes to stderr and
|
||||
# whether step stderr reaches a job log here was itself never established. So the script was made
|
||||
# to REPORT its resolved identity on stdout, where capture is not in question, and the answer was
|
||||
# then simply read off run 1916: `Marker identity: job=test run=1916 attempt=1 (from the runner)`
|
||||
# and the same for `migrations`. Both required jobs, on the lane that matters.
|
||||
#
|
||||
# That measurement is what promoted it from warn-and-default to REQUIRED, which is why the residual
|
||||
# this comment used to describe — a rerun inheriting attempt 1's markers — no longer exists FOR THE
|
||||
# CONTAINER JOBS. If a future runner stops exporting any of the three, every job reddens with a
|
||||
# message naming the variable; that is loud, instantly diagnosable, and the correct direction for a
|
||||
# required check.
|
||||
#
|
||||
# EXCEPT ON A LANE WITH NO `container:` (ersatztv#767). The `scan` job runs on `small` with no
|
||||
# container, so RUNNER_TEMP is the shared host /tmp and the keying below is the ONLY thing separating
|
||||
# runs, not defence in depth on top of a fresh filesystem. There the rerun residual DOES still exist:
|
||||
# a single-job rerun that does not increment GITHUB_RUN_ATTEMPT would find the previous attempt's
|
||||
# marker file. See the carve-out in `ci.required-job-step-execution-markers`; do not read the
|
||||
# paragraph above as covering that job.
|
||||
marker_path() {
|
||||
local missing=""
|
||||
[ -n "${GITHUB_JOB:-}" ] || missing="$missing GITHUB_JOB"
|
||||
[ -n "${GITHUB_RUN_ID:-}" ] || missing="$missing GITHUB_RUN_ID"
|
||||
[ -n "${GITHUB_RUN_ATTEMPT:-}" ] || missing="$missing GITHUB_RUN_ATTEMPT"
|
||||
if [ -n "$missing" ]; then
|
||||
# NOTHING IS PRINTED TO STDOUT HERE, and that is load-bearing rather than style: this
|
||||
# function's stdout IS its return value (it is always called inside `$( )`), so a notice
|
||||
# printed here is captured INTO the path. An earlier revision did exactly that and both
|
||||
# sub-commands then failed on a nonexistent directory. Caught by
|
||||
# test_a_degraded_run_IDENTITY_*, which is why that test asserts on the exit status and on
|
||||
# the absence of any marker file rather than only on the message.
|
||||
echo "::error::ci-step-ran.sh cannot identify this run —${missing} not set. The marker path would fall back to a name other runs also use, and a stale marker would make the dropped-step guard PASS on a run whose step never executed (ersatztv#756). Refusing rather than degrading to a reusable name." >&2
|
||||
exit 3
|
||||
fi
|
||||
printf '%s/etv-ci-steps-ran-%s-%s-%s' \
|
||||
"${RUNNER_TEMP:-${GITHUB_WORKSPACE:-/tmp}}" \
|
||||
"$GITHUB_JOB" "$GITHUB_RUN_ID" "$GITHUB_RUN_ATTEMPT"
|
||||
}
|
||||
|
||||
cmd_mark() {
|
||||
[ "$#" -eq 1 ] && [ -n "$1" ] || usage
|
||||
# Appended, never truncated: every step in the job shares one file, and a `>` here would erase
|
||||
# its predecessors and make the guard red on every run.
|
||||
#
|
||||
# A failure to write is NOT swallowed. The step is running under `bash -e`, so a non-zero here
|
||||
# fails the step and reddens the job — which is the same direction the guard would take a moment
|
||||
# later, but with a message pointing at the real cause instead of at a missing marker.
|
||||
local target
|
||||
# NOT `>> "$(marker_path)"`: the refusal above `exit`s a SUBSHELL there, and bash discards a
|
||||
# command substitution's exit status when it is only part of a redirection — the write would go
|
||||
# to an empty path and the error would read as a redirection failure rather than the real cause.
|
||||
target="$(marker_path)" || exit $?
|
||||
printf '%s\n' "$1" >> "$target"
|
||||
}
|
||||
|
||||
cmd_assert() {
|
||||
local -a always=() gated=()
|
||||
local bucket=""
|
||||
while [ "$#" -gt 0 ]; do
|
||||
case "$1" in
|
||||
--always) bucket=always ;;
|
||||
--gated) bucket=gated ;;
|
||||
-*) usage ;;
|
||||
*)
|
||||
case "$bucket" in
|
||||
always) always+=("$1") ;;
|
||||
gated) gated+=("$1") ;;
|
||||
*) usage ;;
|
||||
esac ;;
|
||||
esac
|
||||
shift
|
||||
done
|
||||
# ANTI-VACUITY, at runtime rather than only in the test suite. An `assert` called with no
|
||||
# expectations passes unconditionally and reports "every expected step executed" — a guard that
|
||||
# proves nothing while looking like it proved everything. Refuse instead.
|
||||
if [ "${#always[@]}" -eq 0 ] && [ "${#gated[@]}" -eq 0 ]; then
|
||||
echo "::error::ci-step-ran.sh assert was called with no expected keys, so it would pass unconditionally. This is a workflow bug, not a build failure." >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
# The skip gates, mirroring the `if:` every gated step carries:
|
||||
# steps.detect.outputs.docs_only != 'true' && steps.revalidate.outputs.skip != 'true'
|
||||
# Anything other than the exact string `true` means the step was expected to run — including the
|
||||
# EMPTY string, which is what these read as when the detect step itself was dropped. That
|
||||
# direction is deliberate: a dropped detect step must widen what is required, never narrow it.
|
||||
local skipped=no
|
||||
if [ "${ETV_DOCS_ONLY:-}" = "true" ] || [ "${ETV_REVALIDATE_SKIP:-}" = "true" ]; then
|
||||
skipped=yes
|
||||
fi
|
||||
|
||||
local marker attempt_used
|
||||
# `|| exit $?` because `set -e` does NOT fire on a failing command substitution in an assignment;
|
||||
# without it a degraded identity would leave `marker` empty and every key would read as missing —
|
||||
# fail-closed by luck, with a misleading message.
|
||||
marker="$(marker_path)" || exit $?
|
||||
# Read the attempt back OFF THE RESOLVED PATH rather than from the environment. It reports what
|
||||
# the path was actually keyed on, so a future change to how the path is built cannot silently
|
||||
# disagree with the line that documents it.
|
||||
attempt_used="${marker##*-}"
|
||||
# `${arr[@]+"${arr[@]}"}` rather than a bare `"${arr[@]}"`: under `set -u` bash 3.2 (the system
|
||||
# bash on the Macs this suite also runs on) treats expanding an EMPTY array as an unbound
|
||||
# variable and aborts. The CI image ships bash 5, where it is fine — which is exactly the kind of
|
||||
# difference that makes a guard pass locally and die on the runner, or the reverse.
|
||||
local -a expected=(${always[@]+"${always[@]}"})
|
||||
if [ "$skipped" = no ]; then
|
||||
expected+=(${gated[@]+"${gated[@]}"})
|
||||
else
|
||||
echo "Skip gate fired (docs_only='${ETV_DOCS_ONLY:-}', already_validated='${ETV_REVALIDATE_SKIP:-}') — the gated steps were not expected to run."
|
||||
fi
|
||||
|
||||
# RE-CHECKED AFTER GATING, not only on argv — found by cold review, which reproduced it:
|
||||
# `ETV_DOCS_ONLY=true … assert --always --gated foo` printed "All 0 expected step(s) executed"
|
||||
# and exited 0. The argv check above cannot see that, because the set is emptied by the gate, not
|
||||
# by the caller. Unreachable with today's argv (both jobs pass `--always detect revalidate`), but
|
||||
# it directly contradicted the comment above it, and a guard that reports proving everything
|
||||
# while proving nothing is the failure this whole file exists to remove.
|
||||
if [ "${#expected[@]}" -eq 0 ]; then
|
||||
echo "::error::ci-step-ran.sh assert ended up with NO expected keys after the skip gate, so it would pass unconditionally. This is a workflow bug, not a build failure." >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
local -a missing=()
|
||||
local key
|
||||
for key in "${expected[@]}"; do
|
||||
# `grep -qxF` over a FILE, never a pipeline: `grep -q` exits at its first match and would
|
||||
# SIGPIPE a producer, which under `set -o pipefail` inverts the result for large inputs
|
||||
# (ersatztv#698). Reading the file directly has no producer to kill. `-x` so a key cannot be
|
||||
# satisfied by another key that contains it, `-F` so a key is never read as a pattern.
|
||||
if ! grep -qxF "$key" "$marker" 2>/dev/null; then
|
||||
missing+=("$key")
|
||||
fi
|
||||
done
|
||||
|
||||
if [ "${#missing[@]}" -gt 0 ]; then
|
||||
echo "::error::These steps of job '${GITHUB_JOB:-?}' never executed: ${missing[*]}. The runner DROPPED them (an interpolation failure over a run: body does this and still reports the job GREEN — ersatztv#751/#756) or their \`if:\` no longer matches the guard's expectations. This job is a REQUIRED check, so a green here would mean a required context passed having done no work. Failing the job so it is visible."
|
||||
if [ -f "$marker" ]; then
|
||||
echo "Marker file ${marker} recorded:"
|
||||
sed 's/^/ /' "$marker"
|
||||
else
|
||||
echo "There is no marker file at ${marker} at all — not one step of this job executed."
|
||||
fi
|
||||
exit 1
|
||||
fi
|
||||
# The resolved identity, on stdout, every run. This is what turns "is GITHUB_RUN_ATTEMPT
|
||||
# exported here?" from an inference into something a reader just looks up — and it is why the
|
||||
# variable is still WARN-and-default rather than REFUSE: `GITHUB_JOB` and `GITHUB_RUN_ID` have
|
||||
# positive evidence (the probe's marker filename), this one does not yet, and refusing on an
|
||||
# unestablished variable would redden a REQUIRED check. Promote it once a run has printed
|
||||
# `attempt=<n> (from the runner)`.
|
||||
# Kept after the promotion, though all three components are now required and the line can no
|
||||
# longer report anything but the runner's own values. It is the standing evidence: this is the
|
||||
# line that settled whether GITHUB_RUN_ATTEMPT is exported, and it is what a future reader checks
|
||||
# first if the keying is ever doubted again.
|
||||
echo "Marker identity: job=${GITHUB_JOB} run=${GITHUB_RUN_ID} attempt=${attempt_used} (from the runner)"
|
||||
echo "All ${#expected[@]} expected step(s) executed: ${expected[*]}"
|
||||
}
|
||||
|
||||
[ "$#" -ge 1 ] || usage
|
||||
sub="$1"
|
||||
shift
|
||||
case "$sub" in
|
||||
mark) cmd_mark "$@" ;;
|
||||
assert) cmd_assert "$@" ;;
|
||||
*) usage ;;
|
||||
esac
|
||||
@@ -19,7 +19,7 @@ from datetime import date
|
||||
from pathlib import Path
|
||||
from typing import NamedTuple
|
||||
|
||||
import scripts.decisions_lib as dl # noqa: E402 (run with PYTHONPATH=. or as module)
|
||||
import scripts.decisions_lib as dl # (run with PYTHONPATH=. or as module)
|
||||
|
||||
SKIP_HEADINGS = dl.SKIP_HEADINGS # single source of truth
|
||||
# `signals` is required alongside the lifecycle fields: the `**Signals:**` line (plus `key:`) is what
|
||||
@@ -469,12 +469,7 @@ def _is_stripped_index(path: Path, archive_dir: Path, recs: list) -> bool:
|
||||
file there is exempt only if it actually LOOKS like a stripped index: exactly one keyless
|
||||
record whose heading is one of the known generated ones (`dl.SKIP_HEADINGS`).
|
||||
"""
|
||||
return (
|
||||
path.parent == archive_dir
|
||||
and len(recs) == 1
|
||||
and not recs[0].key
|
||||
and recs[0].heading in dl.SKIP_HEADINGS
|
||||
)
|
||||
return path.parent == archive_dir and len(recs) == 1 and not recs[0].key and recs[0].heading in dl.SKIP_HEADINGS
|
||||
|
||||
|
||||
def record_wing_faults(records_dir: Path | None = None, archive_dir: Path | None = None) -> list[str]:
|
||||
|
||||
@@ -62,9 +62,7 @@ def render(spec: dict) -> str:
|
||||
"`scripts/update-openapi.sh`.*"
|
||||
)
|
||||
lines.append("")
|
||||
lines.append(
|
||||
f"{len(spec.get('paths', {}))} endpoints, {len(operations)} operations."
|
||||
)
|
||||
lines.append(f"{len(spec.get('paths', {}))} endpoints, {len(operations)} operations.")
|
||||
lines.append("")
|
||||
|
||||
for tag in tags:
|
||||
@@ -73,9 +71,7 @@ def render(spec: dict) -> str:
|
||||
lines.append("| Method | Path | Operation | Summary |")
|
||||
lines.append("|---|---|---|---|")
|
||||
for op in sorted(grouped[tag], key=lambda o: (o["path"], o["method"])):
|
||||
lines.append(
|
||||
f"| {op['method']} | `{op['path']}` | {op['operationId']} | {op['summary']} |"
|
||||
)
|
||||
lines.append(f"| {op['method']} | `{op['path']}` | {op['operationId']} | {op['summary']} |")
|
||||
lines.append("")
|
||||
|
||||
return "\n".join(lines).rstrip("\n") + "\n"
|
||||
|
||||
Executable
+557
@@ -0,0 +1,557 @@
|
||||
#!/usr/bin/env bash
|
||||
# Hook fire log — the WRITE side and the READ side, deliberately in ONE file (ersatztv#776).
|
||||
#
|
||||
# WHY THIS EXISTS. In this harness version only `Stop` hooks emit a structured transcript record
|
||||
# (`stop_hook_summary`/`hookInfos`). `PreToolUse` and `PostToolUse` hooks leave no durable execution
|
||||
# trace — which is every guard this repo actually relies on. #773 §5.4 could therefore only count
|
||||
# *filename mentions in settings dumps*, i.e. inference. A guard that is neither proven nor
|
||||
# observable is an assertion about the past, and a silently broken hook looks identical to a working
|
||||
# one indefinitely. #756's standing lesson: make the system REPORT it rather than infer it.
|
||||
#
|
||||
# WHY ONE FILE. The reader and the writer share the record format. Two files means the format can
|
||||
# drift and the report can quietly describe a shape nothing writes any more — the same argument that
|
||||
# put `scripts/ci-step-ran.sh` in one script instead of an inline workflow body (#756).
|
||||
#
|
||||
# WHY IT CAPTURES STDOUT RATHER THAN BEING TOLD THE DECISION. Every hook here exits 0 always; the
|
||||
# decision is communicated by *printing* `hookSpecificOutput.permissionDecision` (PreToolUse) or
|
||||
# `decision` (Stop). `pretooluse-merge-consent.sh` alone reaches that print from ~40 call sites via
|
||||
# its `decide` helper. Asking each site to also set a variable would (a) be 40 edits in the most
|
||||
# load-bearing guard in the repo and (b) record what the author *meant*, which is the inference this
|
||||
# issue exists to abolish. Capturing the bytes the hook actually emits records what the HARNESS
|
||||
# sees. It cannot drift from the decision because it IS the decision.
|
||||
#
|
||||
# WHY IT CAPTURES STDIN. 8 of the 9 hooks already open with `input=$(cat)` — a full blocking slurp —
|
||||
# so reading stdin once here and replaying it via `exec 0<` is not a new risk, it is the read they
|
||||
# already perform, moved earlier. It buys the tool name and session id for the log, and it makes
|
||||
# `pretooluse-agent-ram.sh` (which reads no stdin at all today) observable on the same terms as the
|
||||
# rest instead of being a hole in the table.
|
||||
#
|
||||
# FAIL-OPEN, DELIBERATELY AND IN THIS DIRECTION ONLY. This file is observability, not a guard. If
|
||||
# anything here fails — no temp dir, unwritable log, missing `date` — the hook must behave EXACTLY as
|
||||
# it did before instrumentation. A logging bug that denies a merge, or that swallows a guard's deny
|
||||
# JSON, would be far worse than the blindness it is fixing. Every function returns 0, the stdout
|
||||
# replay is the first act of the exit path, and the original exit code is re-raised explicitly.
|
||||
#
|
||||
# USAGE (write side) — the first two lines of every hook, before anything reads stdin:
|
||||
# . "${CLAUDE_PROJECT_DIR:-$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd)}/scripts/hook-fire-log.sh"
|
||||
# etv_hook_fire_begin <hook-name> [label]
|
||||
#
|
||||
# USAGE (read side):
|
||||
# scripts/hook-fire-log.sh report # this session (or every session, see below)
|
||||
# scripts/hook-fire-log.sh report --all # every session in the log dir
|
||||
# scripts/hook-fire-log.sh report --json
|
||||
# scripts/hook-fire-log.sh path # where the current session logs
|
||||
#
|
||||
# `scripts/tests/test_hook_fire_log.py` is the guard: it derives the hook population from the
|
||||
# filesystem (never a list — `testing.guard-derives-population-from-source`) and fails if any hook
|
||||
# is uninstrumented, and it proves stdin, stdout and the exit code survive the wrapper.
|
||||
|
||||
# ---------------------------------------------------------------------------------------------
|
||||
# Shared: where the log lives, and how a record is written
|
||||
# ---------------------------------------------------------------------------------------------
|
||||
|
||||
etv_hook_fire_log_dir() {
|
||||
printf '%s' "${ETV_HOOK_FIRE_LOG_DIR:-${HOME:-/tmp}/.cache/ersatztv/hook-fire}"
|
||||
}
|
||||
|
||||
etv_hook_fire_session() {
|
||||
# The harness does not export a session id to hooks in this version, so the payload's
|
||||
# `.session_id` is the real source and the env var is only a fallback. Recorded rather than
|
||||
# assumed: several hooks already read `.session_id` from stdin for exactly this reason.
|
||||
printf '%s' "${ETV_HOOK_FIRE_SESSION:-${CLAUDE_SESSION_ID:-unknown-session}}"
|
||||
}
|
||||
|
||||
etv_hook_fire_log_file() {
|
||||
# SCRUBBED, like every other use of this value. The session id is extracted from the payload by a
|
||||
# `sed` that only excludes `"`, so a `/` or `..` in it would otherwise steer the write outside the
|
||||
# log dir. Harness-generated UUIDs make that unreachable today, but a file whose stated thesis is
|
||||
# "restrict the value space so there is no escaping bug to have" should not exempt the one use
|
||||
# that becomes a path.
|
||||
printf '%s/%s.jsonl' "$(etv_hook_fire_log_dir)" "$(etv_hook_fire_scrub_component "$(etv_hook_fire_session)")"
|
||||
}
|
||||
|
||||
# Sanitise a value to a safe JSON scalar charset. This is why no `jq` is needed on the write side:
|
||||
# with the value space restricted there is nothing to escape, so there is no escaping bug to have.
|
||||
# A quote, backslash or newline in a hook name or tool name is not a case worth supporting — it is a
|
||||
# case worth flattening, loudly, to `_`.
|
||||
etv_hook_fire_scrub() {
|
||||
printf '%s' "${1:-}" | tr -c 'A-Za-z0-9._/:+@=-' '_' | cut -c1-200
|
||||
}
|
||||
|
||||
# A STRICTER scrub for the one value that becomes a PATH. The record scrub above deliberately keeps
|
||||
# `/` and `.` — tool names like `mcp__gitea__x` and event paths read better with them — but those
|
||||
# are exactly the two characters that turn a session id into `../../escaped`. Passing a value
|
||||
# through a scrub is not the same as passing it through the RIGHT scrub, and the first version of
|
||||
# this fix reused the record scrubber and left the traversal wide open while reading as fixed.
|
||||
etv_hook_fire_scrub_component() {
|
||||
printf '%s' "${1:-}" | tr -c 'A-Za-z0-9_-' '_' | cut -c1-120
|
||||
}
|
||||
|
||||
# Append one record. Never fails; never writes a partial line (built whole, appended once).
|
||||
etv_hook_fire_record() {
|
||||
[ "${ETV_HOOK_FIRE_DISABLE:-0}" = "1" ] && return 0
|
||||
|
||||
local file line
|
||||
file="${ETV_HOOK_FIRE_LOG_FILE:-$(etv_hook_fire_log_file)}"
|
||||
mkdir -p "$(dirname "$file")" 2>/dev/null || return 0
|
||||
|
||||
line=$(printf '{"ts":"%s","session":"%s","pid":"%s","hook":"%s","label":"%s","event":"%s","tool":"%s","phase":"%s","code":"%s","decision":"%s"}' \
|
||||
"$(date -u '+%Y-%m-%dT%H:%M:%SZ' 2>/dev/null || printf 'unknown')" \
|
||||
"$(etv_hook_fire_scrub "$(etv_hook_fire_session)")" \
|
||||
"$(etv_hook_fire_scrub "${ETV_HOOK_FIRE_PID:-$$}")" \
|
||||
"$(etv_hook_fire_scrub "${1:-}")" \
|
||||
"$(etv_hook_fire_scrub "${2:-}")" \
|
||||
"$(etv_hook_fire_scrub "${3:-}")" \
|
||||
"$(etv_hook_fire_scrub "${4:-}")" \
|
||||
"$(etv_hook_fire_scrub "${5:-}")" \
|
||||
"$(etv_hook_fire_scrub "${6:-}")" \
|
||||
"$(etv_hook_fire_scrub "${7:-}")" 2>/dev/null) || return 0
|
||||
|
||||
# ORDER MATTERS: `2>` BEFORE `>>`. Redirections are applied left to right, so
|
||||
# `printf ... >> "$file" 2>/dev/null` opens the file FIRST and bash reports a failure to open it
|
||||
# on the stderr still in force — the hook prints `Operation not permitted` at the harness. An
|
||||
# existing-but-unwritable log file is the reachable case; the fail-open test missed it by using a
|
||||
# path that dies at `mkdir` instead. Redirecting stderr first covers the open failure too.
|
||||
printf '%s\n' "$line" 2>/dev/null >> "$file" || true
|
||||
return 0
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------------------------
|
||||
# Write side: what a hook calls
|
||||
# ---------------------------------------------------------------------------------------------
|
||||
|
||||
# Classify what the hook printed. The harness reads these two shapes and nothing else:
|
||||
# PreToolUse : {"hookSpecificOutput":{"permissionDecision":"allow|deny|ask", ...}}
|
||||
# Stop : {"decision":"block", "reason": ...}
|
||||
# Anything else printed on stdout is surfaced to the user but decides nothing.
|
||||
etv_hook_fire_classify() {
|
||||
local out="${1:-}" code="${2:-0}" mode="${3:-capture}" d=""
|
||||
# BYTE-ORIENTED, not locale-aware. One 0xE9 byte in a `permissionDecisionReason` made `sed` print
|
||||
# `illegal byte sequence` to the harness AND fail to extract, filing a real `deny` as `output`.
|
||||
#
|
||||
# `local -x`, NOT `local`. A plain `local` sets a shell variable without the export attribute, so
|
||||
# the child `sed`/`tr` never sees it — the fix was INERT and read as applied. It looked correct
|
||||
# only because this author's shell sets `LANG` alone; with an inherited `LC_CTYPE` (macOS
|
||||
# Terminal, ssh `SendEnv LC_*`, sudo `env_keep`) `LC_CTYPE` outranks the exported `LANG=C` and the
|
||||
# symptom returns in full.
|
||||
local -x LC_ALL=C LANG=C
|
||||
|
||||
# A `stream`-mode hook is a git hook: git reads its exit code and nothing else, so that is the
|
||||
# whole of its decision. Reporting `no-op` here because no JSON was captured would be an
|
||||
# inference, and inference is what this file exists to replace.
|
||||
if [ "$mode" = "stream" ]; then
|
||||
if [ "$code" = "0" ]; then printf 'pass'; else printf 'blocked'; fi
|
||||
return 0
|
||||
fi
|
||||
|
||||
if [ -n "$out" ]; then
|
||||
# `[^"]*`, not `[A-Za-z-]*`: a restricted class means an odd value fails to EXTRACT and is
|
||||
# filed as generic `output`, so "every non-canonical value is recorded as unrecognized" was
|
||||
# true only for values the class happened to admit. Extract anything, then judge it below.
|
||||
d=$(printf '%s' "$out" | sed -n 's/.*"permissionDecision"[[:space:]]*:[[:space:]]*"\([^"]*\)".*/\1/p' | head -n1)
|
||||
# A PRESENT-but-empty value is not the same as an absent key, and treating them alike let
|
||||
# `{"permissionDecision":""}` fall through to `output` — a malformed decision laundered into
|
||||
# "the hook just printed something".
|
||||
if [ -z "$d" ] && printf '%s' "$out" | grep -q '"permissionDecision"[[:space:]]*:'; then
|
||||
d="unrecognized"
|
||||
fi
|
||||
[ -z "$d" ] && d=$(printf '%s' "$out" | sed -n 's/.*"decision"[[:space:]]*:[[:space:]]*"\([^"]*\)".*/\1/p' | head -n1)
|
||||
|
||||
# A NON-CANONICAL VALUE IS RECORDED AS SUCH, not normalised into a valid one. An earlier version
|
||||
# lowercased whatever it found, so `"permissionDecision":"Deny"` was filed as a clean `deny` —
|
||||
# manufacturing a decision the harness may never have honoured. The documented values are
|
||||
# lowercase; anything else is a hook bug, and the log should say so rather than launder it.
|
||||
case "$d" in
|
||||
allow|deny|ask|block|"") ;;
|
||||
*) d="unrecognized" ;;
|
||||
esac
|
||||
# `additionalContext` with no decision (design-sync-reminder's `start` path) informs, it does
|
||||
# not gate. Naming it distinctly keeps it out of the allow/deny counts. Matched WITH its quotes
|
||||
# and colon — a bare substring test also fired on human prose that merely used the word.
|
||||
if [ -z "$d" ] && printf '%s' "$out" | grep -q '"additionalContext"[[:space:]]*:'; then d="context"; fi
|
||||
[ -z "$d" ] && d="output"
|
||||
fi
|
||||
|
||||
# Exit 2 is the harness's block channel and it DOMINATES: the call is blocked whatever the JSON
|
||||
# said. Recording a printed `allow` here would report a permit for an invocation that was refused,
|
||||
# which is the one direction a log of security decisions must never be wrong in.
|
||||
if [ "$code" = "2" ]; then printf 'deny-exit2'; return 0; fi
|
||||
|
||||
[ -z "$d" ] && d="no-op"
|
||||
|
||||
# A non-zero status does NOT annotate a parsed decision. It used to overwrite one (`deny` + exit 1
|
||||
# recorded as `error`), and the first correction over-swung the other way and appended `+error` —
|
||||
# inventing a composite state the harness does not report. The status already has its own field in
|
||||
# the record, so the decision field states the decision and nothing else. `error` means only "it
|
||||
# failed and said nothing classifiable".
|
||||
# `output` too, not just `no-op`: a hook that printed a diagnostic and then FAILED was being
|
||||
# filed as `output`, so the report showed `output=1` and the failure vanished from the histogram.
|
||||
# A parsed decision is still left alone — the status has its own field.
|
||||
case "$code:$d" in 0:*) ;; *:no-op|*:output) d="error" ;; esac
|
||||
|
||||
printf '%s' "$d"
|
||||
}
|
||||
|
||||
# etv_hook_fire_begin <hook-name> [label] [stdout-mode]
|
||||
#
|
||||
# `stdout-mode` is `capture` (default) or `stream`, and the split is not a tuning knob — the two
|
||||
# kinds of hook in this repo decide by different channels:
|
||||
#
|
||||
# Claude Code hooks (pretooluse-*, posttooluse-*, design-sync-reminder) always exit 0 and
|
||||
# communicate by PRINTING JSON. Their decision is only observable by capturing stdout, and their
|
||||
# output is a single line consumed by the harness after exit, so buffering costs nothing.
|
||||
#
|
||||
# Git hooks (prepush-*, decisions-guard) decide by EXIT CODE, and their stdout is progress text a
|
||||
# human is watching in real time. Capturing it would hold a slow pre-push hook's output back until
|
||||
# the end, turning a working progress display into an apparent hang. They pass `stream`, and their
|
||||
# decision is read from the exit code, which is what git reads too.
|
||||
etv_hook_fire_begin() {
|
||||
ETV_HOOK_FIRE_NAME="${1:-unknown-hook}"
|
||||
ETV_HOOK_FIRE_LABEL="${2:-}"
|
||||
ETV_HOOK_FIRE_MODE="${3:-capture}"
|
||||
ETV_HOOK_FIRE_PID="$$"
|
||||
ETV_HOOK_FIRE_STDIN_TMP=""
|
||||
ETV_HOOK_FIRE_STDOUT_TMP=""
|
||||
# RESET, never merely default. Inherited from the environment (exported by a parent, or a second
|
||||
# `begin` in one shell) a stale `1` made the first flush return immediately: stdout stayed
|
||||
# redirected and no exit record was ever written.
|
||||
ETV_HOOK_FIRE_FLUSHED=0
|
||||
|
||||
[ "${ETV_HOOK_FIRE_DISABLE:-0}" = "1" ] && return 0
|
||||
|
||||
ETV_HOOK_FIRE_LOG_FILE="$(etv_hook_fire_log_file)"
|
||||
mkdir -p "$(dirname "$ETV_HOOK_FIRE_LOG_FILE")" 2>/dev/null || { ETV_HOOK_FIRE_DISABLE=1; return 0; }
|
||||
|
||||
# --- stdin: slurp, replay, and read the payload's identifying fields ------------------------
|
||||
#
|
||||
# NEVER on a terminal. A `git commit` run interactively hands its hooks a TTY on fd 0, and `cat`
|
||||
# would block there forever — instrumentation hanging the commit it was added to observe. Claude
|
||||
# Code always writes the JSON payload and closes the pipe, which is why the 8 hooks that already
|
||||
# open with `input=$(cat)` are safe today; that guarantee does not extend to the git hooks, so the
|
||||
# capture is conditioned on stdin not being a tty rather than on which hook is calling.
|
||||
local sin payload="" event="" tool="" sess=""
|
||||
if [ ! -t 0 ]; then
|
||||
sin=$(mktemp "${TMPDIR:-/tmp}/etv-hook-stdin.XXXXXX" 2>/dev/null) || sin=""
|
||||
else
|
||||
sin=""
|
||||
fi
|
||||
if [ -n "$sin" ]; then
|
||||
ETV_HOOK_FIRE_STDIN_TMP="$sin"
|
||||
cat 2>/dev/null > "$sin" || true
|
||||
# Replay: even a partial capture is closer to the truth than the drained pipe the hook would
|
||||
# otherwise inherit.
|
||||
#
|
||||
# NO `2>/dev/null` ON THIS LINE, EVER. `exec` with redirections and no command applies them to
|
||||
# the shell PERMANENTLY, so `exec 0<"$sin" 2>/dev/null` does not suppress errors from this one
|
||||
# redirection — it sends the HOOK'S ENTIRE STDERR to /dev/null for the rest of its life. That
|
||||
# silenced every husky hook's user-facing output, which is stderr: the H6 "push to main BLOCKED"
|
||||
# message, the BOM guard's remediation text, `husky - commit message missing Co-Authored-By`.
|
||||
# The guards still blocked, and the human was told nothing about why.
|
||||
# Readability is tested instead of relying on redirection-failure suppression.
|
||||
if [ -r "$sin" ]; then exec 0<"$sin" || true; fi
|
||||
payload=1
|
||||
fi
|
||||
|
||||
if [ -n "$payload" ]; then
|
||||
# Read the fields from the FILE, byte-oriented, with no size cap. These were extracted from a
|
||||
# `head -c 65536` prefix, so a payload whose `tool_response` pushed `session_id` past 64 KB
|
||||
# filed its records under `unknown-session` with empty event and tool — and a report keyed on
|
||||
# the real session then showed those fires as NEVER HAVING HAPPENED. A truncating read is a
|
||||
# sampling error, and this one manufactured exactly the vacuity #776 exists to abolish.
|
||||
local -x LC_ALL=C LANG=C
|
||||
# FIRST occurrence, via `grep -o`. A `sed` substitution with a leading `.*` is GREEDY, and
|
||||
# payloads are one long line, so it selected the LAST match: a nested
|
||||
# `{"session_id":"...","tool_name":"..."}` inside a `tool_response` outranked the top-level one
|
||||
# and the whole invocation filed under the wrong session. Removing the 64 KB cap is what armed
|
||||
# it — the cap had been accidentally protecting the right answer, which is the kind of load a
|
||||
# bound can be silently carrying.
|
||||
#
|
||||
# The scan is bounded again at 256 KB, but now the bound is safe rather than load-bearing:
|
||||
# identity fields are at the head of the payload, and first-match means a later duplicate cannot
|
||||
# win. Unbounded cost 0.5s per scan on a 20 MB payload, three scans per fire.
|
||||
# A BOUNDED FAST PATH WITH AN UNBOUNDED FALLBACK. A plain cap is a truncating read, and a
|
||||
# truncating read is a sampling error: a payload whose `tool_response` pushes `session_id` past
|
||||
# the cap returns nothing, the record files under `unknown-session`, and that fire reads as
|
||||
# NEVER HAVING HAPPENED — the false vacuity this whole change exists to remove, reintroduced by
|
||||
# the bound added to make it fast. So the cap is an optimisation only: if the prefix yields
|
||||
# nothing, the whole payload is scanned. Identity fields sit at the head in practice, so the
|
||||
# fallback is rare; correctness no longer depends on that being true.
|
||||
#
|
||||
# `etv_hook_fire__field`, not `_etv_field`: a function defined inside another is still GLOBAL in
|
||||
# bash, so a short generic name leaks into the hook's namespace and can collide with something
|
||||
# the hook defines. It is unset after use.
|
||||
etv_hook_fire__field() {
|
||||
local v
|
||||
v=$(head -c 262144 "$sin" 2>/dev/null \
|
||||
| grep -o "\"$1\"[[:space:]]*:[[:space:]]*\"[^\"]*\"" 2>/dev/null \
|
||||
| head -n1 | sed 's/.*:[[:space:]]*"//; s/"$//' 2>/dev/null)
|
||||
if [ -z "$v" ]; then
|
||||
v=$(grep -o "\"$1\"[[:space:]]*:[[:space:]]*\"[^\"]*\"" "$sin" 2>/dev/null \
|
||||
| head -n1 | sed 's/.*:[[:space:]]*"//; s/"$//' 2>/dev/null)
|
||||
fi
|
||||
printf '%s' "$v"
|
||||
}
|
||||
event=$(etv_hook_fire__field hook_event_name)
|
||||
tool=$(etv_hook_fire__field tool_name)
|
||||
sess=$(etv_hook_fire__field session_id)
|
||||
unset -f etv_hook_fire__field 2>/dev/null || true
|
||||
fi
|
||||
|
||||
# The payload's session id is authoritative when the env var is absent, which it is in this
|
||||
# harness version. Re-point the log file at it before the first record is written.
|
||||
if [ -n "$sess" ] && [ -z "${CLAUDE_SESSION_ID:-}" ] && [ -z "${ETV_HOOK_FIRE_SESSION:-}" ]; then
|
||||
ETV_HOOK_FIRE_SESSION="$sess"
|
||||
ETV_HOOK_FIRE_LOG_FILE="$(etv_hook_fire_log_file)"
|
||||
mkdir -p "$(dirname "$ETV_HOOK_FIRE_LOG_FILE")" 2>/dev/null || true
|
||||
fi
|
||||
|
||||
ETV_HOOK_FIRE_EVENT="$event"
|
||||
ETV_HOOK_FIRE_TOOL="$tool"
|
||||
|
||||
etv_hook_fire_record "$ETV_HOOK_FIRE_NAME" "$ETV_HOOK_FIRE_LABEL" "$event" "$tool" "fire" "" ""
|
||||
|
||||
# --- stdout: divert, so the exit path can read the decision the hook emitted ----------------
|
||||
local sout=""
|
||||
if [ "$ETV_HOOK_FIRE_MODE" = "capture" ]; then
|
||||
sout=$(mktemp "${TMPDIR:-/tmp}/etv-hook-stdout.XXXXXX" 2>/dev/null) || sout=""
|
||||
fi
|
||||
if [ -n "$sout" ]; then
|
||||
ETV_HOOK_FIRE_STDOUT_TMP="$sout"
|
||||
exec 3>&1 || { ETV_HOOK_FIRE_STDOUT_TMP=""; rm -f "$sout" 2>/dev/null; return 0; }
|
||||
exec 1>"$sout" || { exec 1>&3 3>&-; ETV_HOOK_FIRE_STDOUT_TMP=""; rm -f "$sout" 2>/dev/null; return 0; }
|
||||
# A READ FD held open on the same file, so the replay does not depend on the PATH still
|
||||
# resolving at exit. Replaying with `cat "$path"` loses everything if anything unlinks the file
|
||||
# mid-run — a $TMPDIR reaper, a hook that clears its own scratch — because the write fd stays
|
||||
# valid on the unlinked inode while the name is gone: the hook prints its `deny` into a file
|
||||
# nothing can reopen. An fd survives unlink, which turns that from silent guard-disarming into
|
||||
# a non-event. Pinned by `test_output_SURVIVES_a_vanished_stdout_tempfile`.
|
||||
exec 4<"$sout" || true
|
||||
fi
|
||||
|
||||
# SIGNALS ARE DELIBERATELY NOT TRAPPED — a withdrawal, recorded because the next reader will
|
||||
# otherwise re-add this. A `trap ... TERM` was added so a hook killed by the harness timeout would
|
||||
# not lose output it had already printed (measured then at 52 bytes before, 0 after). It produced
|
||||
# three defects in three rounds and cost more than it bought:
|
||||
#
|
||||
# 1. the handler ended in `exit "$?"`, so a SIGTERMed `prepush-donewhen.sh` reported 0 and git
|
||||
# PUSHED TO MAIN — a signal became consent;
|
||||
# 2. `local sig=...` clobbered `$?`, so a killed guard was RECORDED as having passed;
|
||||
# 3. and the one that settles it: bash does not run a trap until the current foreground command
|
||||
# finishes, so a hook mid-`curl` took 30s to die where it had taken 1s. A TERM-then-KILL
|
||||
# supervisor therefore gets no flush AT ALL, plus a 29s stall — strictly worse than the
|
||||
# untrapped behaviour on the very path the trap existed for.
|
||||
#
|
||||
# WHAT IS LOST, enumerated rather than waved past: a hook killed by a signal loses its `exit`
|
||||
# record, loses stdout it had already written, and leaks its two temp files. The first is a real
|
||||
# gap in the log and is why `report` counts `fire` records, not `exit` records. The second is
|
||||
# near-unreachable — every capture-mode hook prints its decision and exits immediately, a
|
||||
# sub-millisecond window. The third is bounded by $TMPDIR cleanup.
|
||||
#
|
||||
# The invariant that replaces the rescue is stronger and is what the tests now assert: under a
|
||||
# signal an instrumented hook behaves EXACTLY as an uninstrumented one.
|
||||
trap 'etv_hook_fire_end "$?"' EXIT
|
||||
return 0
|
||||
}
|
||||
|
||||
# Flush and record, WITHOUT deciding how the process ends. `ETV_HOOK_FIRE_FLUSHED` keeps it
|
||||
# idempotent. That guard is retained after the signal traps were withdrawn, because it is cheap and
|
||||
# because the failure it prevents — two contradictory `exit` records for one invocation, a real
|
||||
# `deny` followed by a phantom `no-op` — is silent, and re-entrancy would return the moment anyone
|
||||
# adds a second caller.
|
||||
etv_hook_fire_flush() {
|
||||
local code="${1:-0}" out=""
|
||||
# The `tr -d '\000'` below is a child process too, and it sat outside every locale declaration:
|
||||
# `tr: Illegal byte sequence` reached the harness and truncated the classification copy.
|
||||
local -x LC_ALL=C LANG=C
|
||||
[ "${ETV_HOOK_FIRE_FLUSHED:-0}" = "1" ] && return 0
|
||||
ETV_HOOK_FIRE_FLUSHED=1
|
||||
|
||||
# RESTORE FIRST, UNCONDITIONALLY — then replay if there is anything to replay.
|
||||
#
|
||||
# These were one conditional, `[ -n "$TMP" ] && [ -f "$TMP" ]`, and that coupling was the bug:
|
||||
# `-f` asks "is there output to replay", but the fd restore must happen whenever the redirect was
|
||||
# ESTABLISHED, which is a different fact. Any route that makes `-f` false while the redirect is
|
||||
# live — the temp file unlinked by a $TMPDIR reaper mid-run (the fd stays valid, so the hook
|
||||
# writes happily into an unlinked inode), a $TMPDIR where mktemp yields a non-regular file — left
|
||||
# fd 1 still pointing at the temp target with the restore skipped, so nothing later could rescue
|
||||
# the bytes. A guard's `deny` was silently discarded and the hook still exited 0.
|
||||
#
|
||||
# Split, the worst case degrades from "the guard is disarmed" to "the log is short", which is the
|
||||
# correct failure direction for observability. Pinned by
|
||||
# `test_output_SURVIVES_a_vanished_stdout_tempfile`.
|
||||
if [ -n "${ETV_HOOK_FIRE_STDOUT_TMP:-}" ]; then
|
||||
# Same rule as the stdin replay: a trailing `2>/dev/null` here would permanently silence stderr
|
||||
# rather than suppress an error from this redirection. fd 3 is known open on this path.
|
||||
exec 1>&3 3>&- || true
|
||||
fi
|
||||
if [ -n "${ETV_HOOK_FIRE_STDOUT_TMP:-}" ]; then
|
||||
# ONE read, used for BOTH the replay and the classification, so the two cannot disagree. They
|
||||
# did: the classifier read fd 4 while the replay preferred the file, so a hook that used fd 4
|
||||
# itself replayed its `deny` correctly to the harness and filed it in the log as `no-op` — the
|
||||
# log quietly contradicting the decision it exists to record.
|
||||
#
|
||||
# The `printf X` / `%X` dance preserves trailing newlines, which `$(...)` strips. Without it the
|
||||
# rescue path delivered 52 bytes where the hook wrote 53, and every JSON parser downstream
|
||||
# accepts the short form without complaint. Pinned by `test_stdout_is_replayed_BYTE_EXACT`.
|
||||
#
|
||||
# NO `2>/dev/null` ON THE `exec` — see the stdin comment. An earlier version of THIS line had
|
||||
# it, eight lines below the comment forbidding it, which is why the rule is now restated at
|
||||
# every `exec` rather than once.
|
||||
if [ -r "${ETV_HOOK_FIRE_STDOUT_TMP:-}" ]; then
|
||||
# Common path: stream the FILE straight through. A shell variable cannot hold a NUL byte, so
|
||||
# replaying via `$(...)` silently drops them and warns on stderr; `cat` is byte-exact for any
|
||||
# content. Classification reads the SAME file, so the two cannot disagree — reading them from
|
||||
# different sources is what made the harness see `deny` while the log recorded `no-op`.
|
||||
cat "$ETV_HOOK_FIRE_STDOUT_TMP" 2>/dev/null || true
|
||||
# `tr -d '\000'` before the substitution: bash cannot hold a NUL in a variable and warns
|
||||
# about it ON STDERR, which the harness sees — an instrumentation message leaking into a
|
||||
# guard's output channel. Classification does not care about NULs; the replay above is
|
||||
# byte-exact regardless, because it streams the file rather than a variable.
|
||||
out=$( { head -c 65536 "$ETV_HOOK_FIRE_STDOUT_TMP" 2>/dev/null | tr -d '\000'; printf 'X'; } )
|
||||
out="${out%X}"
|
||||
else
|
||||
# Rescue path: the name is gone but the fd still reads the unlinked inode. This one goes
|
||||
# through a variable and is therefore NUL-lossy — stated rather than hidden, because the
|
||||
# alternative is losing the output entirely.
|
||||
out=$( { cat 2>/dev/null <&4 | tr -d '\000'; printf 'X'; } )
|
||||
out="${out%X}"
|
||||
[ -n "$out" ] && printf '%s' "$out"
|
||||
fi
|
||||
exec 4<&- || true
|
||||
fi
|
||||
|
||||
etv_hook_fire_record \
|
||||
"${ETV_HOOK_FIRE_NAME:-unknown-hook}" "${ETV_HOOK_FIRE_LABEL:-}" \
|
||||
"${ETV_HOOK_FIRE_EVENT:-}" "${ETV_HOOK_FIRE_TOOL:-}" \
|
||||
"exit" "$code" "$(etv_hook_fire_classify "$out" "$code" "${ETV_HOOK_FIRE_MODE:-capture}")" || true
|
||||
|
||||
rm -f "${ETV_HOOK_FIRE_STDIN_TMP:-}" "${ETV_HOOK_FIRE_STDOUT_TMP:-}" 2>/dev/null || true
|
||||
ETV_HOOK_FIRE_STDOUT_TMP=""
|
||||
return 0
|
||||
}
|
||||
|
||||
etv_hook_fire_end() {
|
||||
local code="${1:-0}"
|
||||
etv_hook_fire_flush "$code"
|
||||
# Re-raise the hook's own status explicitly rather than relying on the trap preserving it. Bash
|
||||
# does not re-enter an EXIT trap, so this is not recursive.
|
||||
exit "$code"
|
||||
}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------------------------
|
||||
# Read side: the report
|
||||
# ---------------------------------------------------------------------------------------------
|
||||
|
||||
# The hook population, DERIVED from the filesystem, never listed
|
||||
# (`testing.guard-derives-population-from-source`). A hook added tomorrow appears in the report as
|
||||
# NEVER-FIRED the moment it exists, rather than being invisible because nobody updated an array.
|
||||
etv_hook_fire_population() {
|
||||
local root="${1:-}" f
|
||||
[ -d "$root/.claude/hooks" ] || return 0
|
||||
for f in "$root"/.claude/hooks/*.sh; do
|
||||
[ -f "$f" ] || continue
|
||||
basename "$f" .sh
|
||||
done
|
||||
}
|
||||
|
||||
etv_hook_fire_repo_root() {
|
||||
if [ -n "${CLAUDE_PROJECT_DIR:-}" ] && [ -d "$CLAUDE_PROJECT_DIR/.claude/hooks" ]; then
|
||||
printf '%s' "$CLAUDE_PROJECT_DIR"
|
||||
return 0
|
||||
fi
|
||||
( cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." >/dev/null 2>&1 && pwd )
|
||||
}
|
||||
|
||||
etv_hook_fire_report() {
|
||||
local all=0 as_json=0 dir root files hook
|
||||
while [ $# -gt 0 ]; do
|
||||
case "$1" in
|
||||
--all) all=1 ;;
|
||||
--json) as_json=1 ;;
|
||||
--dir) shift; ETV_HOOK_FIRE_LOG_DIR="${1:-}" ;;
|
||||
--session) shift; ETV_HOOK_FIRE_SESSION="${1:-}" ;;
|
||||
*) printf 'hook-fire-log: unknown report option %s\n' "$1" >&2; return 2 ;;
|
||||
esac
|
||||
shift
|
||||
done
|
||||
|
||||
dir="$(etv_hook_fire_log_dir)"
|
||||
root="$(etv_hook_fire_repo_root)"
|
||||
|
||||
if [ "$all" = "1" ]; then
|
||||
files=$(find "$dir" -name '*.jsonl' -type f 2>/dev/null | sort)
|
||||
else
|
||||
files="$(etv_hook_fire_log_file)"
|
||||
[ -f "$files" ] || files=""
|
||||
fi
|
||||
|
||||
if [ -z "$files" ]; then
|
||||
printf 'hook-fire-log: no records under %s%s\n' "$dir" \
|
||||
"$([ "$all" = "1" ] || printf ' for session %s (try --all)' "$(etv_hook_fire_session)")" >&2
|
||||
fi
|
||||
|
||||
# ANTI-VACUITY. A report over an empty population would print a clean table of nothing and read
|
||||
# as "all hooks accounted for" — the exact failure this file exists to end.
|
||||
local pop
|
||||
pop=$(etv_hook_fire_population "$root")
|
||||
if [ -z "$pop" ]; then
|
||||
printf 'hook-fire-log: found NO hook scripts under %s/.claude/hooks — refusing to report, because a report over an empty population reads as full coverage.\n' "$root" >&2
|
||||
return 2
|
||||
fi
|
||||
|
||||
local total_fires=0 rows=""
|
||||
while IFS= read -r hook; do
|
||||
[ -n "$hook" ] || continue
|
||||
local fires decisions
|
||||
fires=0
|
||||
decisions=""
|
||||
if [ -n "$files" ]; then
|
||||
fires=$(cat $files 2>/dev/null | grep -c "\"hook\":\"$hook\",.*\"phase\":\"fire\"" || true)
|
||||
decisions=$(cat $files 2>/dev/null \
|
||||
| grep "\"hook\":\"$hook\",.*\"phase\":\"exit\"" \
|
||||
| sed -n 's/.*"decision":"\([^"]*\)".*/\1/p' \
|
||||
| sort | uniq -c | sort -rn \
|
||||
| awk '{printf "%s=%s ", $2, $1}')
|
||||
fi
|
||||
[ -z "$fires" ] && fires=0
|
||||
total_fires=$((total_fires + fires))
|
||||
if [ "$as_json" = "1" ]; then
|
||||
rows="${rows}{\"hook\":\"$hook\",\"fires\":$fires,\"decisions\":\"$(printf '%s' "$decisions" | tr -d '"')\"}\n"
|
||||
else
|
||||
rows="$(printf '%s%-34s %6s %s\n' "$rows" "$hook" "$fires" "${decisions:-—}")"$'\n'
|
||||
fi
|
||||
done <<EOF
|
||||
$pop
|
||||
EOF
|
||||
|
||||
if [ "$as_json" = "1" ]; then
|
||||
printf '{"log_dir":"%s","total_fires":%s,"hooks":[%s]}\n' "$dir" "$total_fires" \
|
||||
"$(printf '%b' "$rows" | sed '/^$/d' | paste -sd, -)"
|
||||
return 0
|
||||
fi
|
||||
|
||||
printf 'Hook fire log — %s\n' "$dir"
|
||||
printf '%-34s %6s %s\n' 'HOOK' 'FIRES' 'DECISIONS'
|
||||
printf '%s' "$rows"
|
||||
printf '\n%s hook scripts on disk, %s recorded fires.\n' "$(printf '%s\n' "$pop" | wc -l | tr -d ' ')" "$total_fires"
|
||||
printf 'A hook showing 0 has NOT been observed firing. That is a finding to investigate (broken\nwiring vs genuinely never matched in this window), not a blank to ignore.\n'
|
||||
return 0
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------------------------
|
||||
# CLI (only when executed, never when sourced)
|
||||
# ---------------------------------------------------------------------------------------------
|
||||
|
||||
if [ "${BASH_SOURCE[0]}" = "$0" ]; then
|
||||
set -uo pipefail
|
||||
case "${1:-report}" in
|
||||
report) shift 2>/dev/null || true; etv_hook_fire_report "$@" ;;
|
||||
path) etv_hook_fire_log_file; printf '\n' ;;
|
||||
record) shift; etv_hook_fire_record "$@" ;;
|
||||
*)
|
||||
printf 'usage: %s [report [--all|--json|--dir D|--session S] | path]\n' "$0" >&2
|
||||
exit 2
|
||||
;;
|
||||
esac
|
||||
fi
|
||||
@@ -0,0 +1,307 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Bounded MCP smoke test for a server declared in an .mcp.json (ersatztv#777).
|
||||
|
||||
Written because the caller's original check only asked `[ -x command ]`, which a
|
||||
DIRECTORY satisfies (`[ -x /bin ]` is true), so it could report a pass for a server
|
||||
that cannot run. Review then found the first version of THIS file had the same
|
||||
weakness one level in: it accepted any response carrying the right id, so a server
|
||||
that was not the configured one — or that answered with a malformed body — passed.
|
||||
Hence the shape and identity checks below: "it answered" is not "it answered
|
||||
correctly", and a smoke test that cannot tell them apart is decoration.
|
||||
|
||||
Usage:
|
||||
mcp_smoke.py <.mcp.json> <server> [timeout] [--expect-server NAME]
|
||||
[--expect-tool NAME]...
|
||||
|
||||
Exit 0 only when the server answered `initialize` and `tools/list` with
|
||||
well-formed bodies, matched `--expect-server` if given, and exposed every
|
||||
`--expect-tool`. Failures exit non-zero with a diagnostic naming the stage.
|
||||
Codes group failures by STAGE (config=3-5, command=6, project=7, spawn=8,
|
||||
protocol=9-11, malformed=12, identity=13, tools=14) — several distinct causes
|
||||
deliberately share a stage code, so read the message, not the number.
|
||||
|
||||
ACCEPTED LIMIT: a server that writes a gigabyte with no newline can still exhaust
|
||||
memory before the timeout fires. Guarding that needs the very frame-capping reader
|
||||
whose caps caused three defects in an earlier round, and the input here is our OWN
|
||||
configured server on a developer machine — not an adversary. Stated rather than
|
||||
silently unhandled.
|
||||
|
||||
Deliberately NOT bounded by message/line caps. A first attempt added them and
|
||||
they were the defect: an over-long line had its suffix re-parsed as a fresh
|
||||
message (a false green), and a cap reached before the awaited reply reported
|
||||
"server did not start". What IS bounded is the set of retained DECODED responses
|
||||
— only a reply to the request in flight is kept, notifications are dropped as
|
||||
they arrive — and wall-clock, via the caller's timeout. The raw line buffer is
|
||||
explicitly NOT bounded; that is the accepted limit stated above.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import contextlib
|
||||
import json
|
||||
import os
|
||||
import secrets
|
||||
import shutil
|
||||
import signal
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
|
||||
|
||||
def fail(msg: str, code: int) -> int:
|
||||
print(f"FAIL: {msg}")
|
||||
return code
|
||||
|
||||
|
||||
def main() -> int:
|
||||
argv = sys.argv[1:]
|
||||
expect_server: str | None = None
|
||||
expect_tools: list[str] = []
|
||||
positional: list[str] = []
|
||||
i = 0
|
||||
while i < len(argv):
|
||||
if argv[i] == "--expect-server" and i + 1 < len(argv):
|
||||
expect_server = argv[i + 1]
|
||||
i += 2
|
||||
elif argv[i] == "--expect-tool" and i + 1 < len(argv):
|
||||
expect_tools.append(argv[i + 1])
|
||||
i += 2
|
||||
else:
|
||||
positional.append(argv[i])
|
||||
i += 1
|
||||
|
||||
if len(positional) < 2:
|
||||
return fail(
|
||||
"usage: mcp_smoke.py <.mcp.json> <server> [timeout] [--expect-server NAME] [--expect-tool NAME]...", 2
|
||||
)
|
||||
cfg_path, server = positional[0], positional[1]
|
||||
if len(positional) > 2:
|
||||
try:
|
||||
budget = int(positional[2])
|
||||
except ValueError:
|
||||
return fail(f"timeout must be an integer, got {positional[2]!r}", 2)
|
||||
if budget <= 0:
|
||||
return fail(f"timeout must be positive, got {budget}", 2)
|
||||
else:
|
||||
budget = 180
|
||||
|
||||
try:
|
||||
with open(cfg_path, encoding="utf-8") as fh:
|
||||
doc = json.load(fh)
|
||||
except FileNotFoundError:
|
||||
return fail(f"{cfg_path} does not exist", 3)
|
||||
except json.JSONDecodeError as exc:
|
||||
return fail(f"{cfg_path} is not valid JSON: {exc}", 4)
|
||||
except OSError as exc:
|
||||
return fail(f"{cfg_path} could not be read: {exc}", 4)
|
||||
|
||||
servers = doc.get("mcpServers")
|
||||
if not isinstance(servers, dict):
|
||||
return fail(f"{cfg_path} has no 'mcpServers' object", 5)
|
||||
cfg = servers.get(server)
|
||||
if not isinstance(cfg, dict):
|
||||
return fail(f"{cfg_path} has no '{server}' server entry", 5)
|
||||
|
||||
command = cfg.get("command")
|
||||
args = cfg.get("args") or []
|
||||
if not isinstance(command, str) or not command:
|
||||
return fail(f"'{server}' has no string 'command'", 5)
|
||||
if not isinstance(args, list) or not all(isinstance(a, str) for a in args):
|
||||
return fail(f"'{server}' has a non-string-list 'args'", 5)
|
||||
|
||||
# A directory is executable, so `-x` alone is vacuous. A bare command name is
|
||||
# legitimate and resolves via PATH (e.g. "gitea-mcp-server"), so resolve first
|
||||
# and only then insist on a regular file.
|
||||
resolved = command if os.path.sep in command else shutil.which(command)
|
||||
if resolved is None:
|
||||
return fail(f"command not found on PATH: {command}", 6)
|
||||
if not os.path.isfile(resolved):
|
||||
return fail(f"command is not a regular file: {resolved}", 6)
|
||||
if not os.access(resolved, os.X_OK):
|
||||
return fail(f"command is not executable: {resolved}", 6)
|
||||
|
||||
# The server runs with the config's directory as cwd, so a relative --project
|
||||
# must be validated against THAT, not against the caller's cwd.
|
||||
workdir = os.path.dirname(os.path.abspath(cfg_path)) or os.getcwd()
|
||||
for idx, a in enumerate(args):
|
||||
target = None
|
||||
if a == "--project" and idx + 1 < len(args):
|
||||
target = args[idx + 1]
|
||||
elif a.startswith("--project="):
|
||||
target = a.split("=", 1)[1]
|
||||
if target:
|
||||
probe = target if os.path.isabs(target) else os.path.join(workdir, target)
|
||||
if not os.path.exists(probe):
|
||||
return fail(f"--project path does not exist: {probe}", 7)
|
||||
|
||||
env = dict(os.environ)
|
||||
extra = cfg.get("env") or {}
|
||||
if isinstance(extra, dict):
|
||||
env.update({k: v for k, v in extra.items() if isinstance(v, str)})
|
||||
|
||||
try:
|
||||
proc = subprocess.Popen(
|
||||
[resolved, *args],
|
||||
stdin=subprocess.PIPE,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.DEVNULL,
|
||||
env=env,
|
||||
cwd=workdir,
|
||||
start_new_session=True, # own process group, so children die with us
|
||||
)
|
||||
except OSError as exc:
|
||||
return fail(f"could not start server: {exc}", 8)
|
||||
|
||||
try:
|
||||
pgid = os.getpgid(proc.pid)
|
||||
except OSError:
|
||||
pgid = None
|
||||
|
||||
# Retain ONLY the reply to the request currently in flight. Keeping every integer
|
||||
# id let a server pre-answer id 2 before it was asked, and `wait_for(2)` then
|
||||
# accepted an answer to a question never posed — a false green. It also let a
|
||||
# duplicate id overwrite an earlier reply, and let `responses` grow without bound.
|
||||
# One pending id at a time fixes all three.
|
||||
lock = threading.Lock()
|
||||
pending: int | None = None
|
||||
responses: dict[int, dict] = {}
|
||||
drained = threading.Event()
|
||||
|
||||
def reader() -> None:
|
||||
try:
|
||||
for raw in proc.stdout: # type: ignore[union-attr]
|
||||
line = raw.decode(errors="replace").strip()
|
||||
if not line.startswith("{"):
|
||||
continue
|
||||
try:
|
||||
msg = json.loads(line)
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
rid = msg.get("id")
|
||||
if not isinstance(rid, int):
|
||||
continue # a notification: nothing to retain
|
||||
with lock:
|
||||
# unsolicited, or a second answer to an already-answered id
|
||||
if rid != pending or rid in responses:
|
||||
continue
|
||||
responses[rid] = msg
|
||||
finally:
|
||||
drained.set()
|
||||
|
||||
threading.Thread(target=reader, daemon=True).start()
|
||||
|
||||
def send(payload: dict) -> bool:
|
||||
"""False when the pipe is gone — an instantly-exiting server is the #777
|
||||
failure mode, so it must produce a diagnostic, not a BrokenPipeError."""
|
||||
try:
|
||||
proc.stdin.write((json.dumps(payload) + "\n").encode()) # type: ignore[union-attr]
|
||||
proc.stdin.flush() # type: ignore[union-attr]
|
||||
return True
|
||||
except (BrokenPipeError, OSError, ValueError):
|
||||
return False
|
||||
|
||||
def expect(req_id: int, payload: dict, deadline: float) -> dict | None:
|
||||
"""Register the id BEFORE sending, so a reply cannot arrive unregistered."""
|
||||
nonlocal pending
|
||||
with lock:
|
||||
pending = req_id
|
||||
if not send(payload):
|
||||
return None
|
||||
return wait_for(req_id, deadline)
|
||||
|
||||
def wait_for(req_id: int, deadline: float) -> dict | None:
|
||||
while time.time() < deadline:
|
||||
if req_id in responses:
|
||||
return responses[req_id]
|
||||
# Only conclude "no answer" once the process is gone AND stdout is fully
|
||||
# drained; otherwise a reply already in the pipe is reported as a no-show.
|
||||
if proc.poll() is not None and drained.wait(timeout=2):
|
||||
return responses.get(req_id)
|
||||
time.sleep(0.25)
|
||||
return responses.get(req_id)
|
||||
|
||||
def cleanup() -> None:
|
||||
# `dotnet run` execs a CHILD (csharp-lsp-mcp), so the leader exiting on
|
||||
# SIGTERM says nothing about the descendant. Always follow up with SIGKILL to
|
||||
# the saved group: a stale server surviving a probe is exactly the litter
|
||||
# this session found at start-up.
|
||||
if pgid is not None:
|
||||
for sig in (signal.SIGTERM, signal.SIGKILL):
|
||||
try:
|
||||
os.killpg(pgid, sig)
|
||||
except OSError:
|
||||
break # no group members left
|
||||
with contextlib.suppress(subprocess.TimeoutExpired):
|
||||
proc.wait(timeout=5)
|
||||
time.sleep(0.2)
|
||||
else:
|
||||
for sig in (signal.SIGTERM, signal.SIGKILL):
|
||||
try:
|
||||
proc.send_signal(sig)
|
||||
proc.wait(timeout=5)
|
||||
break
|
||||
except (OSError, subprocess.TimeoutExpired):
|
||||
continue
|
||||
|
||||
deadline = time.time() + budget
|
||||
try:
|
||||
# Random ids close the residual pre-answer race: holding a lock across send()
|
||||
# cannot reject a frame the server emitted BEFORE the request, but a server
|
||||
# cannot pre-answer an id it cannot guess.
|
||||
id_init = secrets.randbelow(2**31 - 1000) + 1000
|
||||
id_tools = secrets.randbelow(2**31 - 1000) + 1000
|
||||
while id_tools == id_init:
|
||||
id_tools = secrets.randbelow(2**31 - 1000) + 1000
|
||||
init = expect(
|
||||
id_init,
|
||||
{
|
||||
"jsonrpc": "2.0",
|
||||
"id": id_init,
|
||||
"method": "initialize",
|
||||
"params": {
|
||||
"protocolVersion": "2024-11-05",
|
||||
"capabilities": {},
|
||||
"clientInfo": {"name": "mcp-smoke", "version": "0"},
|
||||
},
|
||||
},
|
||||
deadline,
|
||||
)
|
||||
if init is None:
|
||||
return fail(f"no 'initialize' response within {budget}s (server did not start)", 9)
|
||||
if "error" in init:
|
||||
return fail(f"initialize returned an error: {json.dumps(init['error'])[:300]}", 9)
|
||||
result = init.get("result")
|
||||
if not isinstance(result, dict):
|
||||
return fail("initialize response has no 'result' object (malformed)", 12)
|
||||
info = result.get("serverInfo")
|
||||
if not isinstance(info, dict) or not isinstance(info.get("name"), str):
|
||||
return fail("initialize result has no 'serverInfo.name' string (malformed)", 12)
|
||||
actual = info["name"]
|
||||
if expect_server is not None and actual != expect_server:
|
||||
return fail(f"wrong server: expected '{expect_server}', got '{actual}'", 13)
|
||||
|
||||
send({"jsonrpc": "2.0", "method": "notifications/initialized", "params": {}})
|
||||
listed = expect(id_tools, {"jsonrpc": "2.0", "id": id_tools, "method": "tools/list", "params": {}}, deadline)
|
||||
if listed is None:
|
||||
return fail(f"no 'tools/list' response within {budget}s", 10)
|
||||
lresult = listed.get("result")
|
||||
if not isinstance(lresult, dict):
|
||||
return fail("tools/list response has no 'result' object (malformed)", 12)
|
||||
tools = lresult.get("tools")
|
||||
if not isinstance(tools, list):
|
||||
return fail("tools/list 'tools' is not a list (malformed)", 12)
|
||||
names = {t.get("name") for t in tools if isinstance(t, dict) and isinstance(t.get("name"), str)}
|
||||
if not names:
|
||||
return fail("server started but exposes zero well-formed tools", 11)
|
||||
missing = [t for t in expect_tools if t not in names]
|
||||
if missing:
|
||||
return fail(f"server '{actual}' is missing expected tool(s): {', '.join(missing)}", 14)
|
||||
print(f"OK: {server} -> {actual} {info.get('version', '')}, {len(names)} tools")
|
||||
return 0
|
||||
finally:
|
||||
cleanup()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -51,10 +51,21 @@ note="$*"
|
||||
|
||||
printf '%s' "$pr" | grep -qE '^[0-9]+$' || die "pull-request number must be numeric, got '$pr'"
|
||||
|
||||
# Normalize to upper-case and classify. The positive set MUST stay in sync with the hook's
|
||||
# condition-(c) regex in .claude/hooks/pretooluse-merge-consent.sh (mergeable|approved|lgtm) —
|
||||
# a word this script treats as positive but the hook does not would let the server-side status go
|
||||
# green while the hook still denies, which reads as an unexplained deny.
|
||||
# Normalize to upper-case and classify. The positive set MUST stay in sync with the READ side's
|
||||
# POS_RE/NEG_RE in scripts/check-review-verdict.sh (mergeable|approved|lgtm) — a word this script
|
||||
# treats as positive but the classifier does not would let the server-side status go green while
|
||||
# the merge-consent hook still denies, which reads as an unexplained deny.
|
||||
#
|
||||
# That sync is NOT asserted anywhere, and you should not assume otherwise. ersatztv#774 tried: a
|
||||
# test extracted both vocabularies from their own shell source and compared them. Six cold-review
|
||||
# rounds each found another shell construction that either escaped the extractor or made it fail on
|
||||
# a correct tree, so it was withdrawn rather than patched a seventh time. Writing a shell parser as
|
||||
# a regex does not converge.
|
||||
#
|
||||
# The fix is to stop having two copies — one declarative vocabulary both scripts read — tracked in
|
||||
# ersatztv#788. Until that lands, THIS COMMENT IS THE ONLY THING holding the two lists together, and
|
||||
# comments drift: this one already had, naming .claude/hooks/pretooluse-merge-consent.sh as the home
|
||||
# of the regex. The hook carries no copy; it delegates to check-review-verdict.sh.
|
||||
verdict=$(printf '%s' "$verdict_raw" | tr '[:lower:]' '[:upper:]')
|
||||
case "$verdict" in
|
||||
MERGEABLE|APPROVED|LGTM) state="success" ;;
|
||||
@@ -131,15 +142,36 @@ printf 'posted comment: Review-verdict: %s @ %s\n' "$verdict" "$short"
|
||||
# introduced by the refactor, so the refusal is now explicit rather than a side effect of `set -e`.
|
||||
prjson_now=$(api_get "repos/$owner/$repo/pulls/$pr") \
|
||||
|| die "could not re-read PR #$pr to confirm the head and base had not moved while posting — no status was written. Re-run once Gitea is reachable."
|
||||
# `[ -n "$x" ] && [ "$x" != "$want" ]` was a fail-OPEN on BOTH of the checks below (ersatztv#778).
|
||||
# A 2xx body that merely LOST the field — `{"head":{},"base":{}}` — yields an empty value, so the
|
||||
# `-n` conjunct is false, the comparison never runs, and the status is posted having confirmed
|
||||
# NOTHING about either the head or the base. The re-read exists precisely to refuse when it cannot
|
||||
# confirm, so a field it cannot read must die exactly like a field that moved. The transport failure
|
||||
# one line up is already fatal; this closes the same hole one level down, which is where it keeps
|
||||
# reappearing in this repo.
|
||||
#
|
||||
# WHICH LINE CARRIES THE SAFETY, stated because it is not the one it looks like: dropping the `-n`
|
||||
# conjunct is the fix. The unconditional `!=` below already rejects an empty value, so the explicit
|
||||
# `-z` arms are REDUNDANT for the safety property and exist only to give the operator an accurate
|
||||
# message ("carried no head sha" rather than "moved to ''"). Disarming a `-z` arm alone therefore
|
||||
# leaves the suite green — the two overlap, and a mutation proof aimed at it would be vacuous. The
|
||||
# proof in `test_a_reread_that_LOSES_a_field_refuses_instead_of_posting` is taken against the real
|
||||
# predecessor (the `-n` conjunct restored), which is what actually goes red.
|
||||
sha_now=$(printf '%s' "$prjson_now" | jq -r '.head.sha // ""')
|
||||
if [ -n "$sha_now" ] && [ "$sha_now" != "$sha" ]; then
|
||||
if [ -z "$sha_now" ]; then
|
||||
die "re-read PR #$pr but its response carried no head sha, so it is UNPROVEN that the head is still $short — no status was written. Re-run once Gitea returns a well-formed PR body."
|
||||
fi
|
||||
if [ "$sha_now" != "$sha" ]; then
|
||||
die "head moved from $short to ${sha_now:0:7} while posting — that commit is UNREVIEWED, so no status was written. Re-review the new head and run this again."
|
||||
fi
|
||||
# The same TOCTOU window applies to the base (ersatztv#632): a retarget between the read above and
|
||||
# the status write below would bind the verdict to a base that is no longer the PR's, and the head
|
||||
# sha check would not notice because retargeting does not move the head.
|
||||
base_now=$(printf '%s' "$prjson_now" | jq -r '.base.ref // ""')
|
||||
if [ -n "$base_now" ] && [ "$base_now" != "$base_ref" ]; then
|
||||
if [ -z "$base_now" ]; then
|
||||
die "re-read PR #$pr but its response carried no base ref, so it is UNPROVEN that the base is still '$base_ref' — no status was written. Re-run once Gitea returns a well-formed PR body."
|
||||
fi
|
||||
if [ "$base_now" != "$base_ref" ]; then
|
||||
die "base branch changed from '$base_ref' to '$base_now' while posting — the diff you reviewed is not the diff this PR now merges, so no status was written. Re-review against the new base and run this again."
|
||||
fi
|
||||
|
||||
|
||||
Executable
+303
@@ -0,0 +1,303 @@
|
||||
#!/usr/bin/env bash
|
||||
# prove-fix.sh — witness a fix's test failing BEFORE the fix (ersatztv#794).
|
||||
#
|
||||
# THE RULE THIS ENFORCES. `testing.guard-ships-with-mutation-proof` says a guard is not
|
||||
# tested because a test involving it passes; it ships with a proof it can fail. The same
|
||||
# argument applies to every bug fix, and nothing enforced it. #776 and #793 each took six
|
||||
# review rounds, and in both the recurring mechanism was one thing: a fix's test was
|
||||
# written to confirm the fix, not to discriminate against its absence.
|
||||
#
|
||||
# HOW. Run the named tests at the commit (CONTROL — must be GREEN), then again in a
|
||||
# SEPARATE fresh worktree with the commit's non-test files reverted (must be RED). The
|
||||
# control is what makes the second run mean anything: a test that is already failing
|
||||
# proves nothing by failing again.
|
||||
#
|
||||
# ONLY pytest exit 1 COUNTS AS RED, and that is the whole safety argument. "Non-zero" is
|
||||
# not "the test failed": 2 is an interrupted collection, 3 an internal error, 4 a usage
|
||||
# error, 5 nothing collected, and a killed run gives 143. Cold review measured a SIGTERM
|
||||
# being read as red in an early DRAFT — cancellation masquerading as evidence. On the first
|
||||
# COMMITTED version the reproducible false PROVEN is the marker case (a failed `cd` giving
|
||||
# the subshell status 1), not the signal one; see the decision record.
|
||||
# `--continue-on-collection-errors` is passed so a genuine collection failure
|
||||
# (what happens when the fix ADDED the module the test imports) is reported as a test
|
||||
# error and exits 1 instead of vanishing into the ambiguous 2/3 band. Measured: pass 0,
|
||||
# fail 1, collection-error 2 (1 with the flag), SIGTERM 143.
|
||||
#
|
||||
# WHAT THIS DOES NOT DO, stated because a prover that overclaims is the defect it exists
|
||||
# to catch: it checks a test CAN go red, not that it asserts the RIGHT property. #776's
|
||||
# `test_output_survives_a_SIGTERM` would have passed this check while still never reading
|
||||
# `returncode` — the one thing its fix changed. That judgement stays with review.
|
||||
#
|
||||
# Usage:
|
||||
# prove-fix.sh [--repo DIR] <commit> [test-selector]
|
||||
#
|
||||
# The selector may be omitted when the commit carries a `Proves: <selector>` trailer.
|
||||
# There is deliberately no heuristic fallback: guessing from the touched test files
|
||||
# silently does nothing when a fix edits an existing test, and a prover that quietly
|
||||
# proves nothing is worse than one that refuses.
|
||||
#
|
||||
# Exit codes:
|
||||
# 0 PROVEN — green with the fix, red (pytest exit 1) without it
|
||||
# 1 UNPROVEN — the tests passed without the fix; they do not discriminate
|
||||
# 2 usage / bad arguments
|
||||
# 3 no selector (no argument and no `Proves:` trailer)
|
||||
# 4 nothing to revert (no non-test files) — a docs/CI-only commit; opt out explicitly
|
||||
# 5 environment/git/selector failure, or a run whose exit code is not interpretable
|
||||
# 6 control failed — the selector does not even pass WITH the fix
|
||||
|
||||
set -uo pipefail
|
||||
|
||||
REPO="$PWD"
|
||||
if [ "${1:-}" = "--repo" ]; then
|
||||
REPO="${2:-}"; shift 2
|
||||
fi
|
||||
COMMIT="${1:-}"
|
||||
SELECTOR="${2:-}"
|
||||
|
||||
TMP=""
|
||||
TMP_ENUM=""
|
||||
# ONE cleanup, ONE EXIT trap. An earlier version installed a SECOND `trap ... EXIT` later in
|
||||
# the script to remove TMP_ENUM; `on_signal` then ran `trap - EXIT`, disarming it, so a
|
||||
# signalled run leaked that directory. Folding both removals in here removes the ordering
|
||||
# subtlety rather than adding a third trap to compensate for it.
|
||||
cleanup() {
|
||||
if [ -n "$TMP" ]; then
|
||||
for d in "$TMP/wt-control" "$TMP/wt-reverted"; do
|
||||
[ -d "$d" ] && git -C "$REPO" worktree remove --force "$d" >/dev/null 2>&1
|
||||
done
|
||||
rm -rf "$TMP" >/dev/null 2>&1
|
||||
fi
|
||||
[ -n "$TMP_ENUM" ] && rm -rf "$TMP_ENUM" >/dev/null 2>&1
|
||||
return 0
|
||||
}
|
||||
# A signal must not look like a verdict: clean up, then exit non-zero EXPLICITLY. Without
|
||||
# that explicit exit the handler falls through and the previous status stands, which is
|
||||
# how a cancelled run printed PROVEN in the draft.
|
||||
on_signal() { cleanup; trap - EXIT; printf 'prove-fix: interrupted by signal\n' >&2; exit 5; }
|
||||
trap cleanup EXIT
|
||||
trap on_signal INT TERM
|
||||
|
||||
die() { printf '%s\n' "$1" >&2; exit "$2"; }
|
||||
|
||||
[ -n "$COMMIT" ] || die "usage: prove-fix.sh [--repo DIR] <commit> [test-selector]" 2
|
||||
git -C "$REPO" rev-parse --git-dir >/dev/null 2>&1 || die "not a git repository: $REPO" 5
|
||||
command -v python3 >/dev/null 2>&1 || die "python3 is required to run the tests" 5
|
||||
|
||||
git -C "$REPO" rev-parse --verify --quiet "$COMMIT^{commit}" >/dev/null \
|
||||
|| die "no such commit: $COMMIT" 5
|
||||
SHA="$(git -C "$REPO" rev-parse "$COMMIT")"
|
||||
|
||||
# A merge commit has several parents, so "the code before this change" is ambiguous.
|
||||
# Refuse rather than silently taking the first parent: a `Proves:` trailer on a merge is a
|
||||
# claim this script cannot evaluate, and quietly evaluating a different one would be the
|
||||
# overclaim it exists to prevent.
|
||||
PARENTS="$(git -C "$REPO" rev-list --parents -n 1 "$SHA")" || die "rev-list failed for $SHA" 5
|
||||
NPARENT=$(( $(printf '%s' "$PARENTS" | wc -w) - 1 ))
|
||||
[ "$NPARENT" -le 1 ] || die \
|
||||
"cannot prove a MERGE commit ($NPARENT parents): $SHA
|
||||
'The code before this change' is ambiguous across parents. Put the Proves: trailer on the
|
||||
commit that carries the fix." 5
|
||||
|
||||
PARENT="$(git -C "$REPO" rev-parse --verify --quiet "${SHA}^" || true)"
|
||||
[ -n "$PARENT" ] || die "cannot prove a root commit (no parent to revert to): $SHA" 5
|
||||
|
||||
# ---------------------------------------------------------------- selector
|
||||
if [ -z "$SELECTOR" ]; then
|
||||
RAW_TRAILERS="$(git -C "$REPO" show -s --format='%(trailers:key=Proves,valueonly)' "$SHA")"
|
||||
# More than one `Proves:` means the commit claims two proofs and only the first would be
|
||||
# checked — the rest would read as covered while never running.
|
||||
if [ "$(printf '%s\n' "$RAW_TRAILERS" | grep -c .)" -gt 1 ]; then
|
||||
die "commit carries more than one 'Proves:' trailer; only one is checked, so the others
|
||||
would read as proven without ever running. Use a single selector." 3
|
||||
fi
|
||||
SELECTOR="$(printf '%s\n' "$RAW_TRAILERS" | head -1)"
|
||||
# Trim surrounding whitespace ONLY. Not `xargs`: it applies shell-ish quote parsing, so
|
||||
# a legitimate parametrised node id like `test_x[can't]` makes it report an unterminated
|
||||
# quote and yield an empty selector — silently dropping a real claim.
|
||||
SELECTOR="${SELECTOR#"${SELECTOR%%[![:space:]]*}"}"
|
||||
SELECTOR="${SELECTOR%"${SELECTOR##*[![:space:]]}"}"
|
||||
fi
|
||||
# Split ONCE, deliberately, into an array — then always expand as "${SEL[@]}". Leaving the
|
||||
# scalar unquoted at the call site would also apply PATHNAME EXPANSION, so a node id
|
||||
# containing a glob character could select different tests than the trailer names.
|
||||
read -r -a SEL <<< "$SELECTOR"
|
||||
|
||||
[ -n "$SELECTOR" ] || die \
|
||||
"no test selector: pass one, or give the commit a 'Proves: <selector>' trailer.
|
||||
Refusing to guess from the touched test files — that silently proves nothing when a fix
|
||||
edits an existing test, which is the failure mode this script exists to prevent." 3
|
||||
|
||||
# ---------------------------------------------------------------- classify
|
||||
TMP_ENUM="$(mktemp -d "${TMPDIR:-/tmp}/prove-fix-enum.XXXXXX")" || die "mktemp failed" 5
|
||||
# NUL-delimited --name-status: git tells us whether each path was Added, Modified,
|
||||
# Deleted or Renamed. Deriving added-ness from git's own status letter beats probing the
|
||||
# parent blob and interpreting the error text — `cat-file -e parent:new.py` fails with a
|
||||
# non-empty message for a legitimately-added file, so "non-empty stderr means git broke"
|
||||
# both mis-classifies that case and would leave real git failures indistinguishable.
|
||||
# -z also protects paths containing newlines/quotes, which git otherwise quotes and which
|
||||
# would then silently stay un-reverted while we claim to have reverted them.
|
||||
# Capture to a file with a CHECKED status first. Inside `done < <(git ...)` the producer's
|
||||
# exit status is unavailable, so a git failure that had already emitted one complete record
|
||||
# would pass the non-empty check and revert only PART of the commit — overstating "without
|
||||
# the fix" and manufacturing a proof from an incomplete revert.
|
||||
NS="$TMP_ENUM/name-status"
|
||||
git -C "$REPO" diff-tree --no-commit-id -r -z --name-status "$SHA" >"$NS" \
|
||||
|| die "git diff-tree failed for $SHA — cannot enumerate what to revert" 5
|
||||
|
||||
CHANGED=(); STATUS=()
|
||||
while IFS= read -r -d '' st; do
|
||||
IFS= read -r -d '' path || die "truncated --name-status stream after status '$st'" 5
|
||||
case "$st" in
|
||||
R*|C*) IFS= read -r -d '' newpath \
|
||||
|| die "truncated rename/copy record after '$st' '$path'" 5
|
||||
CHANGED+=("$newpath"); STATUS+=("A") # the new path is absent in the parent
|
||||
CHANGED+=("$path"); STATUS+=("D") ;; # the old path is present in it
|
||||
*) CHANGED+=("$path"); STATUS+=("${st:0:1}") ;;
|
||||
esac
|
||||
done <"$NS"
|
||||
[ "${#CHANGED[@]}" -gt 0 ] || die "commit touches no files, or enumeration failed: $SHA" 5
|
||||
|
||||
is_test_path() {
|
||||
case "$1" in
|
||||
scripts/tests/*|*/scripts/tests/*) return 0 ;;
|
||||
*[Tt]ests/*) return 0 ;;
|
||||
*.test.ts|*.test.tsx|*.test.js) return 0 ;;
|
||||
*_test.py|test_*.py) return 0 ;;
|
||||
*) return 1 ;;
|
||||
esac
|
||||
}
|
||||
|
||||
NON_TEST=(); NON_TEST_ST=(); TEST_FILES=()
|
||||
for i in "${!CHANGED[@]}"; do
|
||||
f="${CHANGED[$i]}"
|
||||
if is_test_path "$f"; then
|
||||
TEST_FILES+=("$f")
|
||||
else
|
||||
NON_TEST+=("$f"); NON_TEST_ST+=("${STATUS[$i]}")
|
||||
fi
|
||||
done
|
||||
|
||||
if [ "${#NON_TEST[@]}" -eq 0 ]; then
|
||||
die "nothing to revert: $SHA changes only test files.
|
||||
A commit with no code side cannot be proven this way. If it is a docs/CI-only fix, opt out
|
||||
explicitly with a reason rather than letting this pass silently." 4
|
||||
fi
|
||||
|
||||
printf 'commit %s\n' "$SHA"
|
||||
printf 'selector %s\n' "$SELECTOR"
|
||||
printf 'reverting %d non-test file(s), keeping %d test file(s)\n' \
|
||||
"${#NON_TEST[@]}" "${#TEST_FILES[@]}"
|
||||
|
||||
TMP="$(mktemp -d "${TMPDIR:-/tmp}/prove-fix.XXXXXX")" || die "mktemp failed" 5
|
||||
mkdir -p "$TMP/tmp-control" "$TMP/tmp-reverted" || die "could not create phase temp dirs" 5
|
||||
|
||||
# Each phase gets its OWN worktree and TMPDIR, and pytest's cache is disabled. Sharing one
|
||||
# worktree lets state written during the control run decide the second run: a test that
|
||||
# creates a marker and fails when it already exists would "go red" with the fix still in
|
||||
# place — a false PROVEN manufactured entirely by the harness.
|
||||
#
|
||||
# PYTEST_RC IS READ FROM A MARKER, NOT FROM THE SUBSHELL. `( cd X && pytest ) ; rc=$?`
|
||||
# returns the SUBSHELL's status, and a failed `cd` or a failed redirection also yields 1 —
|
||||
# which the "only exit 1 is red" rule would then accept as a witnessed test failure with
|
||||
# pytest never having run. Cold review found exactly that. The marker file is written only
|
||||
# after pytest RETURNS, so its absence means "pytest did not complete" and can never be
|
||||
# mistaken for a verdict.
|
||||
PYTEST_RC=""
|
||||
run_phase() { # $1 = worktree, $2 = label; sets PYTEST_RC or returns non-zero
|
||||
rm -f "$TMP/rc-$2"
|
||||
(
|
||||
cd "$1" || exit 91
|
||||
exec >"$TMP/out-$2.txt" 2>&1 || exit 92
|
||||
PYTHONPATH=. TMPDIR="$TMP/tmp-$2" PYTEST_ADDOPTS= \
|
||||
python3 -m pytest "${SEL[@]}" -q --continue-on-collection-errors -p no:cacheprovider
|
||||
printf '%s' "$?" > "$TMP/rc-$2"
|
||||
)
|
||||
if [ ! -s "$TMP/rc-$2" ]; then
|
||||
return 1 # pytest never completed — harness failure, NOT a test result
|
||||
fi
|
||||
PYTEST_RC="$(cat "$TMP/rc-$2")"
|
||||
return 0
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------- control
|
||||
git -C "$REPO" worktree add --detach "$TMP/wt-control" "$SHA" >/dev/null 2>&1 \
|
||||
|| die "could not create the control worktree at $SHA" 5
|
||||
|
||||
echo "--- control: running the selector WITH the fix ---"
|
||||
if ! run_phase "$TMP/wt-control" control; then
|
||||
sed -n '1,25p' "$TMP/out-control.txt" 2>/dev/null >&2
|
||||
die "the control run did not complete: pytest produced no exit status.
|
||||
This is a harness failure (bad cd, unwritable log, missing interpreter), not a test
|
||||
result, and must never be reported as a verdict." 5
|
||||
fi
|
||||
CONTROL_RC="$PYTEST_RC"
|
||||
echo "--- control pytest exit: $CONTROL_RC ---"
|
||||
|
||||
if [ "$CONTROL_RC" -ne 0 ]; then
|
||||
sed -n '1,25p' "$TMP/out-control.txt" >&2
|
||||
case "$CONTROL_RC" in
|
||||
4|5) die "selector ran NO tests (pytest exit $CONTROL_RC): $SELECTOR
|
||||
Exit 4 means the path does not exist; 5 means nothing was collected. Fix the selector." 5 ;;
|
||||
*) die "control FAILED: the selector does not pass WITH the fix (pytest exit $CONTROL_RC).
|
||||
A test that is already red proves nothing by being red after a revert. Fix the test or the
|
||||
selector first." 6 ;;
|
||||
esac
|
||||
fi
|
||||
|
||||
# ---------------------------------------------------------------- reverted
|
||||
git -C "$REPO" worktree add --detach "$TMP/wt-reverted" "$SHA" >/dev/null 2>&1 \
|
||||
|| die "could not create the reverted worktree at $SHA" 5
|
||||
|
||||
for i in "${!NON_TEST[@]}"; do
|
||||
f="${NON_TEST[$i]}"; st="${NON_TEST_ST[$i]}"
|
||||
case "$st" in
|
||||
A) # added by this commit: it has no parent version, so remove it
|
||||
rm -f "$TMP/wt-reverted/$f" || die "could not remove added file $f" 5 ;;
|
||||
M|D|T)
|
||||
git -C "$TMP/wt-reverted" checkout "$PARENT" -- "$f" 2>/dev/null \
|
||||
|| die "could not revert $f to $PARENT (status $st)" 5 ;;
|
||||
*) die "unhandled git status '$st' for $f — refusing to guess how to revert it" 5 ;;
|
||||
esac
|
||||
done
|
||||
|
||||
echo "--- running the selector WITHOUT the fix ---"
|
||||
if ! run_phase "$TMP/wt-reverted" reverted; then
|
||||
sed -n '1,25p' "$TMP/out-reverted.txt" 2>/dev/null >&2
|
||||
die "the reverted run did not complete: pytest produced no exit status.
|
||||
This is a harness failure, not a witnessed red. Accepting it would let a broken cd or an
|
||||
unwritable log masquerade as proof — the defect this script exists to catch." 5
|
||||
fi
|
||||
RC="$PYTEST_RC"
|
||||
# A non-numeric status makes BOTH `[ "$RC" -eq 0 ]` and `[ "$RC" -ne 1 ]` return 2, so
|
||||
# control would fall through to PROVEN — a fail-OPEN default in the one place the whole
|
||||
# safety argument rests. Unreachable today (the marker's only writer is `printf '%s' "$?"`),
|
||||
# but every other branch here is deliberately fail-closed and this one should be too.
|
||||
case "$RC" in
|
||||
''|*[!0-9]*) die "unreadable pytest status '$RC' — refusing to guess a verdict" 5 ;;
|
||||
esac
|
||||
echo "--- pytest exit: $RC ---"
|
||||
|
||||
# ---------------------------------------------------------------- verdict
|
||||
if [ "$RC" -eq 0 ]; then
|
||||
printf '\nUNPROVEN %s\n' "$SHA"
|
||||
printf '%s\n' "The named tests PASS without the fix, so they do not discriminate against"
|
||||
printf '%s\n' "its absence. Strengthen the test until reverting the fix reddens it."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ "$RC" -ne 1 ]; then
|
||||
sed -n '1,25p' "$TMP/out-reverted.txt" >&2
|
||||
die "cannot interpret pytest exit $RC as a test failure.
|
||||
Only exit 1 counts as red. 2 is an interrupted collection, 3 an internal error, 4 a usage
|
||||
error, 5 nothing collected, 130/143 a signal — none is evidence that the test
|
||||
discriminates, and treating them as red lets a cancelled or broken run prove a fix." 5
|
||||
fi
|
||||
|
||||
# Print WHICH tests failed. This script's own docstring says review must judge whether the
|
||||
# RIGHT property failed; hiding the report would make that judgement impossible.
|
||||
printf '\n--- failures observed without the fix (first 25 lines) ---\n'
|
||||
sed -n '1,25p' "$TMP/out-reverted.txt"
|
||||
|
||||
printf '\nPROVEN %s\n' "$SHA"
|
||||
printf '%s\n' "Green with the fix (control), red without it (pytest exit 1)."
|
||||
exit 0
|
||||
@@ -3,18 +3,18 @@
|
||||
import argparse
|
||||
import importlib
|
||||
import sys
|
||||
|
||||
from uuid import UUID
|
||||
|
||||
import etv_client
|
||||
from etv_client.api import ScriptedScheduleApi
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Run an ETV scripted schedule")
|
||||
parser.add_argument('host', help="The ETV host (e.g., http://localhost:8409)")
|
||||
parser.add_argument('build_id', type=UUID, help="The build ID for the playout")
|
||||
parser.add_argument('mode', choices=['reset', 'continue'], help="The playout build mode")
|
||||
parser.add_argument('script_name', help="The name of the script module to use (e.g., one)")
|
||||
parser.add_argument("host", help="The ETV host (e.g., http://localhost:8409)")
|
||||
parser.add_argument("build_id", type=UUID, help="The build ID for the playout")
|
||||
parser.add_argument("mode", choices=["reset", "continue"], help="The playout build mode")
|
||||
parser.add_argument("script_name", help="The name of the script module to use (e.g., one)")
|
||||
|
||||
known_args, unknown_args = parser.parse_known_args()
|
||||
|
||||
@@ -28,9 +28,9 @@ def main():
|
||||
|
||||
with etv_client.ApiClient(configuration) as api_client:
|
||||
try:
|
||||
define_content = getattr(script_module, 'define_content')
|
||||
reset_playout = getattr(script_module, 'reset_playout')
|
||||
build_playout = getattr(script_module, 'build_playout')
|
||||
define_content = script_module.define_content
|
||||
reset_playout = script_module.reset_playout
|
||||
build_playout = script_module.build_playout
|
||||
|
||||
api_instance = ScriptedScheduleApi(api_client)
|
||||
|
||||
@@ -48,6 +48,6 @@ def main():
|
||||
except AttributeError as e:
|
||||
print(f"Error: the '{known_args.script_name}' script is missing a required function. {e}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
||||
|
||||
@@ -0,0 +1,27 @@
|
||||
"""Keep the test suite out of the PRODUCTION hook-fire log (ersatztv#776).
|
||||
|
||||
Four suites here drive real hooks as subprocesses — `test_merge_consent_exemption.py`,
|
||||
`test_merge_consent_base_change.py`, `test_prepush_rebase_check_tag_exemption.py`,
|
||||
`test_pr_changed_files.py` — and they predate the instrumentation. Once every hook records its own
|
||||
execution, running `pytest scripts/tests` wrote its synthetic invocations into
|
||||
`$HOME/.cache/ersatztv/hook-fire/`: 96 `pretooluse-merge-consent` fires including two `deny`s, and
|
||||
12 `blocked` decisions from `prepush-rebase-check`, none of them a real session.
|
||||
|
||||
That is not untidiness, it is the defect the whole change exists to remove. `scripts/hook-fire-log.sh
|
||||
report` is meant to answer "what did the harness actually do", and a log carrying test artifacts
|
||||
answers a different question while looking identical — inference reintroduced one layer up. The
|
||||
record has no field distinguishing a test invocation from a real one, and adding one would only move
|
||||
the problem, so the fix is that tests never write to the real log at all.
|
||||
|
||||
Autouse and session-independent: a new suite that drives a hook is isolated by existing, without
|
||||
having to know this file is here.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def isolate_hook_fire_log(tmp_path_factory, monkeypatch):
|
||||
monkeypatch.setenv("ETV_HOOK_FIRE_LOG_DIR", str(tmp_path_factory.mktemp("hook-fire-log")))
|
||||
@@ -0,0 +1,326 @@
|
||||
"""Machinery for the clause-level mutation harness (ersatztv#790).
|
||||
|
||||
`docs/guard-inventory.md` grades each guard's proof `MUTATION`, `BEHAVIOUR-ONLY` or `NONE`, and
|
||||
`MUTATION` means "a clause-level mutation was executed and this named test was witnessed red". A
|
||||
witnessing performed once, by hand, decays the moment anyone edits the guard, and a grade nothing
|
||||
re-checks can simply be wrong.
|
||||
|
||||
This module turns each such row from an assertion into a check: apply the guard's **declared**
|
||||
clause mutation to an isolated copy of the repo and require the row's own named test to go RED.
|
||||
|
||||
WHAT IS DELIBERATELY NOT DONE. The mutation is declared per guard in `mutation_manifest.py`, never
|
||||
inferred. A harness that guessed which clause of a 90-line hook is *the* guard would manufacture
|
||||
exactly the confident-but-empty coverage this exists to prevent — the reason
|
||||
`testing.guard-ships-with-mutation-proof` rejects a generic runner. Guessing is also unnecessary:
|
||||
most of the proof tests already name their clause in source (the BOM test's `= "efbbbf" ]; then`,
|
||||
`UNSET_CLAUSE`, `prove-fix.sh`'s `if [ "$RC" -eq 0 ]; then`), and the manifest reuses that same
|
||||
string rather than inventing a second one.
|
||||
|
||||
THE SANDBOX IS A REAL GIT REPOSITORY, not a directory of copied files. Several guards derive their
|
||||
population from `git ls-files` and one drives `git worktree add`, so a plain copy would send them
|
||||
down their degraded paths and every mutation would "redden" for a reason having nothing to do with
|
||||
the clause. Its contents are the TRACKED files with WORKING-TREE content — `git ls-files -s`, not a
|
||||
filesystem walk (`testing.guard-derives-population-from-source`, and the reason #778's guard was red
|
||||
on every developer checkout and green in CI: `.husky/_/` is generated by `npm ci` and untracked).
|
||||
|
||||
Two index entries are not regular files and are handled explicitly rather than by an exception:
|
||||
the `.claude/skills/jellyfin` symlink is recreated as a symlink (it dangles outside `~/ersatztv`,
|
||||
which is inherent to the cross-repo symlink pattern and not this harness's problem), and the
|
||||
`ErsatzTV-macOS` gitlink is SKIPPED — no guard reads the submodule, and materialising one would
|
||||
cost a fetch per run.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[2]
|
||||
|
||||
# Long enough that a slow shared runner is not mistaken for a hang, short enough that a genuinely
|
||||
# stuck inner pytest fails the job rather than burning the whole CI budget. The full set of proof
|
||||
# tests runs in ~7s locally.
|
||||
PYTEST_TIMEOUT = 300
|
||||
|
||||
# Every git call here is local and confined to the sandbox; anything slower is stuck, not slow.
|
||||
GIT_TIMEOUT = 120
|
||||
|
||||
# The pristine commit of each sandbox, held OUT OF THE REPOSITORY the proof tests drive. A ref inside
|
||||
# it would be one more thing a proof can move: `git branch -f` fails on a checked-out branch, a
|
||||
# global `init.defaultBranch` can collide with the name, and any `git update-ref`/`git checkout -B` a
|
||||
# proof runs could retarget it. An object id kept here cannot be reached from inside the sandbox at
|
||||
# all, and `git reset --hard <oid>` needs no ref to exist.
|
||||
_BASELINES: dict[str, str] = {}
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Mutation:
|
||||
"""One declared clause mutation and the test that must notice it.
|
||||
|
||||
`guard` is the `Guard` column of `docs/guard-inventory.md` — the thing being graded.
|
||||
`target` is the file actually edited. They are usually the same; where they differ, `why` says
|
||||
why, and `test_a_cross_file_mutation_states_why` requires it.
|
||||
`clause` must occur EXACTLY ONCE in `target`: a mutation that lands on an unintended second site
|
||||
proves something about a clause nobody declared.
|
||||
`expect` is a substring the FAILING run's output must contain, and it is what stops exit code 1
|
||||
from being the whole verdict. Pytest reports an ordinary exception the same way it reports a
|
||||
failed assertion, so a mutation that merely CRASHES the proof test — an emptied population
|
||||
reaching an `IndexError`, a syntax error, an unrelated parametrisation — would otherwise be
|
||||
accepted as "the guard noticed". Naming the diagnostic the mutation is supposed to produce makes
|
||||
each row's evidence specific: a red for a different reason fails here and has to be re-declared.
|
||||
|
||||
`granularity` is `CLAUSE` or `DETECTOR`, and it is the honest half of this harness. #790 opened
|
||||
on the observation that neutering `pin_population_faults` wholesale is "coarser than disarming
|
||||
one clause at a time — coarse enough that a single surviving clause would not be noticed". That
|
||||
is true, and it is also not always avoidable: a detector that accumulates faults from several
|
||||
independent arms answers on ANY of them, so disarming one arm leaves its proof test green and
|
||||
the only mutation that reddens is the whole detector. Recording which grade each guard actually
|
||||
admits turns that from an unstated weakness into a measured property.
|
||||
|
||||
A `DETECTOR` entry does not merely SAY a finer mutation was tried; it carries that mutation in
|
||||
`survived_clause`/`survived_replacement`, and `test_every_SURVIVING_clause_mutation_still_does`
|
||||
re-runs it and requires the proof test to stay GREEN. The justification for the coarse grade is
|
||||
therefore executed on every run, exactly like the grade it justifies — a prose claim would decay
|
||||
the same way the hand-run witnessing this whole harness replaces did.
|
||||
"""
|
||||
|
||||
CLAUSE = "CLAUSE"
|
||||
DETECTOR = "DETECTOR"
|
||||
|
||||
guard: str
|
||||
target: str
|
||||
clause: str
|
||||
replacement: str
|
||||
proof: str
|
||||
granularity: str
|
||||
expect: str
|
||||
why: str
|
||||
survived_clause: str = ""
|
||||
survived_replacement: str = ""
|
||||
|
||||
@property
|
||||
def node_id(self) -> str:
|
||||
"""The inventory records proof refs as `file.py::test`; pytest wants a path."""
|
||||
return f"scripts/tests/{self.proof}"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Verdict:
|
||||
ok: bool
|
||||
reason: str
|
||||
|
||||
|
||||
def _clean_env(**extra: str) -> dict[str, str]:
|
||||
"""The environment every subprocess here runs in, with git's ambient state REMOVED.
|
||||
|
||||
Exported `GIT_*` variables override `-C` and `cwd`. `GIT_DIR`, `GIT_WORK_TREE`,
|
||||
`GIT_INDEX_FILE`, `GIT_COMMON_DIR` and `GIT_OBJECT_DIRECTORY` each redirect part of a repository;
|
||||
`GIT_CONFIG_COUNT`/`GIT_CONFIG_KEY_n`/`GIT_CONFIG_VALUE_n` inject arbitrary settings, `core.worktree`
|
||||
among them. Any of those reaching this module's `git init`/`add`/`commit`/`reset --hard` points
|
||||
them at the REAL repository, and the "sandbox" would then write through the tree it exists to
|
||||
stay out of. A git hook exports several of them, and this suite runs from one.
|
||||
|
||||
So this is a DENY-BY-DEFAULT boundary rather than a list of the variables anyone has thought of:
|
||||
every `GIT_*` is dropped and only the identity this module sets itself is put back. Enumerating
|
||||
the dangerous ones is how the first version of this function shipped covering three of them.
|
||||
"""
|
||||
env = {k: v for k, v in os.environ.items() if not k.startswith("GIT_")}
|
||||
env.update(extra)
|
||||
return env
|
||||
|
||||
|
||||
def _git(cwd: Path, *args: str) -> subprocess.CompletedProcess:
|
||||
# `-c` rather than the ambient configuration, because the sandbox must not inherit the
|
||||
# developer's machine: a global `core.hooksPath` would fire this repo's husky hooks against a
|
||||
# throwaway tree, and `commit.gpgsign` would block the commit on a signing key CI does not have —
|
||||
# indefinitely, at a pinentry prompt, which no pytest timeout is watching.
|
||||
return subprocess.run(
|
||||
# `core.worktree` is pinned along with the rest: a proof that plants one in the sandbox's own
|
||||
# config would otherwise redirect `reset --hard` and `clean -qffdx` at a tree outside it.
|
||||
[
|
||||
"git",
|
||||
"-c",
|
||||
"core.hooksPath=/dev/null",
|
||||
"-c",
|
||||
"commit.gpgsign=false",
|
||||
"-c",
|
||||
f"core.worktree={cwd}",
|
||||
*args,
|
||||
],
|
||||
cwd=str(cwd),
|
||||
check=True,
|
||||
capture_output=True,
|
||||
timeout=GIT_TIMEOUT,
|
||||
env=_clean_env(
|
||||
GIT_AUTHOR_NAME="mutation-harness",
|
||||
GIT_AUTHOR_EMAIL="harness@example.invalid",
|
||||
GIT_COMMITTER_NAME="mutation-harness",
|
||||
GIT_COMMITTER_EMAIL="harness@example.invalid",
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def build_sandbox(dest: Path, root: Path = REPO_ROOT) -> Path:
|
||||
"""Materialise `root`'s tracked files at `dest` and make it a git repository."""
|
||||
entries = subprocess.run(
|
||||
["git", "-C", str(root), "ls-files", "-s", "-z"],
|
||||
capture_output=True,
|
||||
check=True,
|
||||
timeout=GIT_TIMEOUT,
|
||||
env=_clean_env(),
|
||||
).stdout.decode()
|
||||
|
||||
copied = 0
|
||||
for entry in entries.split("\0"):
|
||||
if not entry:
|
||||
continue
|
||||
meta, path = entry.split("\t", 1)
|
||||
mode = meta.split()[0]
|
||||
if mode == "160000": # gitlink — see the module docstring
|
||||
continue
|
||||
src = root / path
|
||||
dst = dest / path
|
||||
dst.parent.mkdir(parents=True, exist_ok=True)
|
||||
if src.is_symlink():
|
||||
os.symlink(os.readlink(src), dst)
|
||||
else:
|
||||
shutil.copy2(src, dst)
|
||||
copied += 1
|
||||
|
||||
if copied == 0:
|
||||
raise RuntimeError(
|
||||
"the sandbox population is EMPTY — `git ls-files` returned nothing, so every mutation "
|
||||
"below would run against an empty tree and report success. Anti-vacuity, not paranoia."
|
||||
)
|
||||
|
||||
_git(dest, "init", "-q", ".")
|
||||
# `-f` because some tracked files are also gitignored; without it they would be dropped from the
|
||||
# sandbox's index and a guard deriving its population from `git ls-files` would see less than the
|
||||
# real repo does.
|
||||
_git(dest, "add", "-A", "-f", ".")
|
||||
_git(dest, "commit", "-qm", "mutation-harness sandbox")
|
||||
_BASELINES[str(dest.resolve())] = _git(dest, "rev-parse", "HEAD").stdout.decode().strip()
|
||||
return dest
|
||||
|
||||
|
||||
def reset_sandbox(sandbox: Path) -> None:
|
||||
"""Return the sandbox to its committed state between mutations.
|
||||
|
||||
The proof tests write into `tmp_path`, but a guard driven through its real entry point can leave
|
||||
artifacts in the tree it is pointed at, and one mutation's residue reaching the next would make
|
||||
the second result a function of the first's.
|
||||
"""
|
||||
# RESET TO THE RECORDED BASELINE COMMIT, never to bare HEAD. `git reset --hard` with no argument
|
||||
# resets to whatever HEAD currently is — so a proof test that COMMITS inside the sandbox moves
|
||||
# HEAD onto a commit containing the mutant, and every later "reset" would then faithfully restore
|
||||
# it. The `finally` in `verify_mutation` puts the file back, but nothing would put HEAD back, and
|
||||
# the contamination would surface as an unrelated red several mutations later.
|
||||
#
|
||||
# `-ff` rather than `-f` because a single `-f` refuses to delete a nested git repository, which is
|
||||
# precisely what a proof driving `git init` or `git worktree add` into the sandbox leaves behind.
|
||||
baseline = _BASELINES.get(str(sandbox.resolve()))
|
||||
if baseline is None:
|
||||
raise RuntimeError(
|
||||
f"no recorded baseline for {sandbox} — it was not built by build_sandbox, so there is "
|
||||
"nothing to reset TO and a reset here would pin whatever state the tree is in now"
|
||||
)
|
||||
_git(sandbox, "reset", "-q", "--hard", baseline)
|
||||
_git(sandbox, "clean", "-qffdx")
|
||||
|
||||
|
||||
def run_pytest(sandbox: Path, node_ids: list[str]) -> subprocess.CompletedProcess:
|
||||
return subprocess.run(
|
||||
[
|
||||
sys.executable,
|
||||
"-m",
|
||||
"pytest",
|
||||
"-q",
|
||||
"--no-header",
|
||||
"--tb=short", # the assertion MESSAGE, which `expect` is matched against
|
||||
"-p",
|
||||
"no:cacheprovider", # keeps `git status` in the sandbox clean between mutations
|
||||
*node_ids,
|
||||
],
|
||||
cwd=str(sandbox),
|
||||
capture_output=True,
|
||||
text=True,
|
||||
env=_clean_env(PYTHONPATH="."),
|
||||
timeout=PYTEST_TIMEOUT,
|
||||
)
|
||||
|
||||
|
||||
# Only exit code 1 means "a test ran and failed", and it is the only status accepted here. Everything
|
||||
# else is rejected, which matters most for the two ways a proof ref goes stale — measured, because
|
||||
# they are easy to get the wrong way round: with an explicit `file.py::function` node id, a missing
|
||||
# FILE and a missing FUNCTION both exit 4 ("ERROR: not found"), while 5 needs a successful collection
|
||||
# that selected nothing — a deselection. Reading either as a guard going red is how a harness reports
|
||||
# coverage it does not have.
|
||||
_PYTEST_RED_MEANINGS = {
|
||||
0: "the named test still PASSED with the clause mutated, so the clause is not load-bearing for it",
|
||||
2: "the inner pytest was interrupted",
|
||||
3: "the inner pytest hit an internal error",
|
||||
4: "the inner pytest could not resolve the node id — the proof ref names a file or a test that does not exist",
|
||||
5: "the inner pytest collected successfully but selected NOTHING — the proof ref was deselected",
|
||||
}
|
||||
|
||||
|
||||
def verify_mutation(sandbox: Path, mutation: Mutation) -> Verdict:
|
||||
"""Apply one declared mutation in `sandbox` and require its named test to go red.
|
||||
|
||||
The sandbox is left as it was found; callers still `reset_sandbox` between mutations because a
|
||||
driven guard can dirty the tree in ways this function does not know about.
|
||||
"""
|
||||
target = sandbox / mutation.target
|
||||
if not target.is_file():
|
||||
return Verdict(False, f"the mutation target {mutation.target} does not exist in the sandbox")
|
||||
|
||||
original = target.read_text(encoding="utf-8")
|
||||
occurrences = original.count(mutation.clause)
|
||||
if occurrences != 1:
|
||||
return Verdict(
|
||||
False,
|
||||
f"the declared clause occurs {occurrences} times in {mutation.target}, not once. "
|
||||
"RETARGET it rather than loosening the match — a clause that has moved, or that now "
|
||||
"matches a second site, means the recorded proof no longer points at what it claims to.",
|
||||
)
|
||||
|
||||
mutated = original.replace(mutation.clause, mutation.replacement, 1)
|
||||
if mutated == original:
|
||||
return Verdict(False, "the replacement is identical to the clause, so nothing was mutated")
|
||||
|
||||
target.write_text(mutated, encoding="utf-8")
|
||||
try:
|
||||
result = run_pytest(sandbox, [mutation.node_id])
|
||||
finally:
|
||||
target.write_text(original, encoding="utf-8")
|
||||
|
||||
output = result.stdout + result.stderr
|
||||
if result.returncode != 1:
|
||||
meaning = _PYTEST_RED_MEANINGS.get(result.returncode, f"unexpected pytest exit code {result.returncode}")
|
||||
return Verdict(False, f"{meaning}\n--- inner pytest output ---\n{output[-3000:]}")
|
||||
# MATCHED AGAINST THE EXCEPTION OUTPUT ALONE, not the whole run. `--tb=short` echoes the failing
|
||||
# SOURCE as well as the message, and every one of these assertions carries its message as a
|
||||
# string literal a line or two above — so matching the full output would let a red at assertion A
|
||||
# be certified by assertion B's text merely being on screen. Pytest prefixes exception lines with
|
||||
# `E `, and that is the only part that reports what actually failed.
|
||||
diagnostic = "\n".join(line[2:] for line in output.splitlines() if line.startswith("E "))
|
||||
# This couples the harness to pytest's traceback FORMAT, and pytest is deliberately unpinned in
|
||||
# `script-tests`. The coupling is fail-CLOSED: a release that stopped prefixing exception lines
|
||||
# with `E ` would empty `diagnostic` and every row would fail here naming its own expectation,
|
||||
# which is loud and instantly diagnosable. The alternative — matching the whole run — fails
|
||||
# silently in the direction that certifies rows on the wrong red. Note the join: a multi-line
|
||||
# assertion message arrives as several `E ` lines, so an expectation must not span a newline.
|
||||
if mutation.expect not in diagnostic:
|
||||
return Verdict(
|
||||
False,
|
||||
f"the named test went red, but NOT with the declared diagnostic {mutation.expect!r}. A red "
|
||||
"for a reason other than the one this row records is not evidence about the clause — a "
|
||||
"crash, a syntax error or an unrelated parametrisation all look like this. Re-declare "
|
||||
f"`expect` once you know what the mutation now produces.\n--- inner pytest output ---\n"
|
||||
f"{output[-3000:]}",
|
||||
)
|
||||
return Verdict(True, "the named test went red under the declared mutation, with the declared diagnostic")
|
||||
@@ -0,0 +1,316 @@
|
||||
"""The DECLARED clause mutations, one per `MUTATION`-graded row of `docs/guard-inventory.md`.
|
||||
|
||||
Data only. The machinery that applies these is `mutation_harness_lib.py`; the checks that keep this
|
||||
file honest are `test_mutation_harness.py`.
|
||||
|
||||
Every entry is declared by hand and none is inferred, which is the whole design constraint from
|
||||
ersatztv#790: "a harness that guesses which clause of a 90-line hook is *the* guard would manufacture
|
||||
exactly the confident-but-empty coverage this is meant to prevent". Where a proof test already names
|
||||
its own clause in source — the BOM guard's `= "efbbbf" ]; then`, `UNSET_CLAUSE`, `prove-fix.sh`'s
|
||||
`if [ "$RC" -eq 0 ]; then` — the entry reuses THAT string rather than inventing a second one, so a
|
||||
retarget in either place is caught by the other.
|
||||
|
||||
WHY AN ENTRY'S `target` MAY DIFFER FROM ITS `guard`. Some guards here ARE tests
|
||||
(`scripts/tests/test_*.py`). Disarming such a guard makes it ABSENT rather than red, so
|
||||
`testing.guard-ships-with-mutation-proof`'s checker-guard exception applies: the mutation goes into
|
||||
the guarded ARTIFACT — a deleted row, a planted phantom row — and the check must report it. Mutating
|
||||
a checker's own POPULATION instead is a trap that looks identical and is not: a shrunken population
|
||||
makes every real row report as PHANTOM, so the proof reddens on a false positive while saying
|
||||
nothing about the missing-row detection the row claims. `why` states per entry which shape applies
|
||||
and why; no count is kept here, because a count of the entries below is a second copy of them.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from scripts.tests.mutation_harness_lib import Mutation
|
||||
|
||||
CLAUSE = Mutation.CLAUSE
|
||||
DETECTOR = Mutation.DETECTOR
|
||||
|
||||
MUTATIONS: tuple[Mutation, ...] = (
|
||||
Mutation(
|
||||
guard=".claude/hooks/posttooluse-worktree-marker.sh",
|
||||
target=".claude/hooks/posttooluse-worktree-marker.sh",
|
||||
clause='printf \'%s\\n\' "$me" > "$abs/.claude-worktree-owner" 2>/dev/null || true',
|
||||
replacement="true",
|
||||
proof="test_worktree_ownership_guard.py::test_MUTATION_a_marker_hook_that_stops_WRITING_makes_the_guard_go_quiet",
|
||||
granularity=CLAUSE,
|
||||
expect="the UNMUTATED pair did not deny",
|
||||
why="The marker write is the hook's entire job; without it the guard has nothing to read and "
|
||||
"fails open. The clause string is the one the proof test itself passes to its `_mutate` helper.",
|
||||
),
|
||||
Mutation(
|
||||
guard=".claude/hooks/pretooluse-bom-guard.sh",
|
||||
target=".claude/hooks/pretooluse-bom-guard.sh",
|
||||
clause='= "efbbbf" ]; then',
|
||||
replacement='= "deadbeef" ]; then',
|
||||
proof="test_bom_guard_detection.py::test_DISARMING_the_BOM_comparison_stops_detection",
|
||||
granularity=CLAUSE,
|
||||
expect="the BOM comparison has moved",
|
||||
why="The BOM comparison is the guard's only detection logic. Same clause the proof test names.",
|
||||
),
|
||||
Mutation(
|
||||
guard=".claude/hooks/pretooluse-worktree-guard.sh",
|
||||
target=".claude/hooks/pretooluse-worktree-guard.sh",
|
||||
clause='marker="$root/.claude-worktree-owner"',
|
||||
replacement='marker="$root/.claude-worktree-owner-NOTHING-WRITES-THIS"',
|
||||
proof="test_worktree_ownership_guard.py::test_MUTATION_disarming_the_guards_MARKER_READ_stops_the_deny",
|
||||
granularity=CLAUSE,
|
||||
expect="the UNMUTATED guard did not deny",
|
||||
why="The marker read is what the ownership decision hangs on. Same clause the proof test names.",
|
||||
),
|
||||
Mutation(
|
||||
guard=".husky/pre-push",
|
||||
target=".husky/pre-push",
|
||||
clause="unset GIT_DIR GIT_WORK_TREE GIT_INDEX_FILE",
|
||||
replacement=": # clause removed by the mutation harness",
|
||||
proof="test_prepush_unsets_git_env.py::test_MUTATION_DELETING_the_unset_lets_drift_through_silently",
|
||||
granularity=CLAUSE,
|
||||
expect="the UNMUTATED pre-push did not catch the drift",
|
||||
why="Without the unset, every git call the pre-push chain makes is aimed at the repository git "
|
||||
"exported the environment for, not the one being pushed. `UNSET_CLAUSE` in the proof test.",
|
||||
),
|
||||
Mutation(
|
||||
guard="scripts/build_decisions_catalog.py",
|
||||
target="scripts/build_decisions_catalog.py",
|
||||
clause="want.strip() != have.strip()",
|
||||
replacement="False",
|
||||
proof="test_build_catalog_check_path.py::test_MUTATION_disarming_the_stale_comparison_stops_detection",
|
||||
granularity=CLAUSE,
|
||||
expect="the stale-detection clause has moved or been reworded",
|
||||
why="`main()`'s only stale-detection logic, per the proof test's own docstring, which uses this "
|
||||
"exact clause and this exact replacement.",
|
||||
),
|
||||
Mutation(
|
||||
guard="scripts/ci-step-ran.sh",
|
||||
target="scripts/ci-step-ran.sh",
|
||||
clause='if ! grep -qxF "$key" "$marker" 2>/dev/null; then',
|
||||
replacement="if false; then",
|
||||
proof="test_ci_dropped_step_guard.py::test_dropping_ANY_single_step_FAILS_the_guard",
|
||||
granularity=CLAUSE,
|
||||
expect="never having executed",
|
||||
why="The per-key membership test is what turns a dropped step into a red job; disarmed, every "
|
||||
"expected key reads as present and the guard passes a run in which nothing executed.",
|
||||
),
|
||||
Mutation(
|
||||
guard="scripts/decisions_validate.py",
|
||||
target="scripts/decisions_validate.py",
|
||||
clause="wing_faults=record_wing_faults() + yaml_faults,",
|
||||
replacement="wing_faults=yaml_faults,",
|
||||
proof="test_decisions_validate.py::test_main_actually_CALLS_the_wing_scan",
|
||||
granularity=CLAUSE,
|
||||
expect="a wing fault must fail the validator",
|
||||
why="The wiring the proof test exists for: deleting this call left the whole suite green while "
|
||||
"a real block-scalar record vanished under `decisions-validate: OK` (#609).",
|
||||
),
|
||||
Mutation(
|
||||
guard="scripts/prove-fix.sh",
|
||||
target="scripts/prove-fix.sh",
|
||||
clause='if [ "$RC" -eq 0 ]; then',
|
||||
replacement="if false; then",
|
||||
proof="test_prove_fix.py::test_MUTATION_disarming_the_UNPROVEN_clause_reddens_the_refusal_test",
|
||||
granularity=CLAUSE,
|
||||
expect="the clause under mutation is gone",
|
||||
why="The UNPROVEN branch: a named test that passes WITHOUT the fix must be refused. Same clause "
|
||||
"the proof test names.",
|
||||
),
|
||||
Mutation(
|
||||
guard="scripts/tests/test_ci_image_pin_population.py",
|
||||
target="scripts/tests/test_ci_image_pin_population.py",
|
||||
clause="for name in sorted(TOOLCHAIN_JOBS - set(pinned)):",
|
||||
replacement="for name in sorted(set()):",
|
||||
proof="test_ci_image_pin_population.py::test_a_single_job_losing_its_pin_is_DETECTED",
|
||||
granularity=CLAUSE,
|
||||
expect="the population check accepted a workflow in which a container job no longer runs",
|
||||
why="The against-the-registry direction, and the one the other two clauses cannot cover: a job "
|
||||
"that loses its `container:` block leaves `declared` and `pinned` equal, so only this "
|
||||
"comparison notices it has moved to the bare runner. This is the clause #790 asked for "
|
||||
"instead of neutering `pin_population_faults` wholesale.",
|
||||
),
|
||||
Mutation(
|
||||
guard="scripts/tests/test_guard_populations_derive_from_git.py",
|
||||
target="scripts/tests/test_guard_inventory.py",
|
||||
clause="if ref in tracked:",
|
||||
replacement="if (REPO_ROOT / ref).exists():",
|
||||
proof="test_guard_populations_derive_from_git.py::test_no_derivation_admits_an_untracked_file",
|
||||
granularity=CLAUSE,
|
||||
expect="after git stopped tracking them",
|
||||
why="THE GUARD IS A TEST, so the mutation goes into the guarded ARTIFACT — one of the "
|
||||
"derivations it watches — rather than into the checker, per the checker-guard exception. "
|
||||
"The clause is the exact defect this guard was written after: `derived_guard_files` read "
|
||||
"its CALLERS from the index and then admitted the paths they name on `Path.exists()`, so a "
|
||||
"tracked workflow naming a script that exists on one machine only entered the population "
|
||||
"there, red on that checkout and green in CI (#778's third shape, found by cold review "
|
||||
"inside #806 itself). Note what this mutation does NOT do: on a clean tree the mutated set "
|
||||
"is identical, so `test_guard_inventory.py`'s own assertions stay green — only narrowing "
|
||||
"the index, which is what the proof does, separates them. That is why the proof has to "
|
||||
"remove EVERY member rather than sample one.",
|
||||
),
|
||||
Mutation(
|
||||
guard="scripts/tests/test_guard_inventory.py",
|
||||
target="docs/guard-inventory.md",
|
||||
clause="| `.claude/hooks/decisions-guard.sh` | a commit | GUARD | NONE | — |\n",
|
||||
replacement="",
|
||||
proof="test_guard_inventory.py::test_the_inventory_covers_exactly_the_guards_that_exist",
|
||||
granularity=CLAUSE,
|
||||
expect="these guard files exist but have no row in guard-inventory.md",
|
||||
why="THE GUARD IS A TEST, so the mutation goes into the guarded ARTIFACT rather than into the "
|
||||
"checker — disarming a checker makes it absent, not red, and mutating its population instead "
|
||||
"would only demonstrate a false POSITIVE (a shrunken population reports every real row as "
|
||||
"phantom) while proving nothing about the missing-row detection the row claims. A deleted "
|
||||
"row is the defect this guard exists to catch, and it is one of the mutations #774 witnessed "
|
||||
"by hand.",
|
||||
),
|
||||
Mutation(
|
||||
guard="scripts/tests/test_hook_fire_log.py",
|
||||
target="scripts/tests/test_hook_fire_log.py",
|
||||
clause=" return faults\n\n\ndef strip_instrumentation",
|
||||
replacement=" return []\n\n\ndef strip_instrumentation",
|
||||
proof="test_hook_fire_log.py::test_a_hook_that_LOSES_its_instrumentation_is_DETECTED",
|
||||
granularity=DETECTOR,
|
||||
expect="left the check GREEN. The check is not load-bearing",
|
||||
why="NO CLAUSE-LEVEL MUTATION REDDENS THIS ONE, and that is a finding rather than a shortcut. "
|
||||
"`instrumentation_faults` accumulates from four independent arms and a stripped hook trips "
|
||||
"three of them at once (no sink source, no ETV_HOOK_FIRE_LIB assignment, no begin call), so "
|
||||
"disarming any single arm leaves the other two answering and the proof test stays green. The "
|
||||
"whole detector is therefore the smallest mutation this proof can witness — and the surviving "
|
||||
"single-arm mutation below is re-run every time so that claim is checked, not recited.",
|
||||
survived_clause=" if not _SOURCES_SINK.search(text):",
|
||||
survived_replacement=" if False:",
|
||||
),
|
||||
Mutation(
|
||||
guard="scripts/tests/test_remote_state_inventory.py",
|
||||
target="docs/remote-state-inventory.md",
|
||||
clause="| `scripts/post-review-verdict.sh` — commit-status write |",
|
||||
replacement="| `scripts/DELETED-BY-THE-MUTATION-HARNESS.sh` — not a real path |",
|
||||
proof="test_remote_state_inventory.py::test_every_in_scope_file_has_a_row_and_every_row_names_a_real_file",
|
||||
granularity=CLAUSE,
|
||||
expect="in scope but absent from docs/remote-state-inventory.md",
|
||||
why="THE GUARD IS A TEST, so the mutation goes into the guarded ARTIFACT: a real executable's "
|
||||
"row is renamed away, which is the MISSING-row defect the row's Blocks column claims — an "
|
||||
"in-scope file with no classification. Two shapes were tried and rejected. Emptying the "
|
||||
"guard's `git ls-files` derivation reddens the proof with an IndexError over an empty "
|
||||
"population: a crash, not a detection. Planting a PHANTOM row reddens "
|
||||
"`test_MUTATION_PROOF_a_dropped_row_and_a_phantom_row_are_both_detected` by contaminating "
|
||||
"the fixture that test builds for itself, and proves the opposite direction from the one the "
|
||||
"row claims. Renaming the row exercises both directions of the production set comparison at "
|
||||
"once and is matched on the missing half.",
|
||||
),
|
||||
Mutation(
|
||||
guard="scripts/tests/test_mutation_harness.py",
|
||||
target="scripts/tests/mutation_harness_lib.py",
|
||||
clause=" if mutation.expect not in diagnostic:",
|
||||
replacement=" if False:",
|
||||
proof="test_mutation_harness.py::test_MUTATION_disarming_the_DIAGNOSTIC_gate_accepts_a_red_for_the_wrong_reason",
|
||||
granularity=CLAUSE,
|
||||
expect="the UNMUTATED verdict already accepted it, so the mutant proves nothing",
|
||||
why="The target is not the guard for a structural reason: the harness keeps its machinery in "
|
||||
"`mutation_harness_lib.py` so a clause of it can be disarmed in an isolated copy at all. The "
|
||||
"clause is the DIAGNOSTIC gate — the check that a failing proof failed with the diagnostic "
|
||||
"its row declares. Disarmed, a red for any unrelated reason is certified as a guard doing "
|
||||
"its job, which is the shape that made two rows in this very file measure nothing. The other "
|
||||
"gate, the one requiring pytest exit code 1, carries its own proof in "
|
||||
"`test_MUTATION_disarming_the_EXIT_STATUS_gate_accepts_a_run_that_NEVER_RAN_A_TEST`; the "
|
||||
"inventory holds one ref per row, so this entry names the stronger of the two.",
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
# ------------------------------------------------------------------------------------------------
|
||||
# THE OTHER GUARDS — stated per guard, and compared for SET EQUALITY against the inventory
|
||||
# ------------------------------------------------------------------------------------------------
|
||||
#
|
||||
# #790's third Done-when box asks that guards whose mutation cannot be declared be STATED. A reason
|
||||
# keyed on the row's GRADE would be cheaper and is tautological: a new guard graded NONE inherits one
|
||||
# automatically and nobody ever looks at that particular guard. A count of them is no better — it
|
||||
# moves only on net change, so adding one undeclared guard while promoting another leaves it at 22.
|
||||
#
|
||||
# So this is keyed on the guard, and `test_every_GUARD_row_is_either_DECLARED_or_STATED_here` asserts
|
||||
# set equality against the inventory's GUARD rows in both directions. That makes it the same kind of
|
||||
# hand-maintained-but-machine-checked table as `docs/guard-inventory.md` itself: a new guard cannot
|
||||
# arrive without someone writing a line here about why it carries no mutation, and a line cannot
|
||||
# outlive the row it is about.
|
||||
#
|
||||
# WHAT `NONE` ACTUALLY MEANS, because the wording matters here: the row nominates no proof ref. It
|
||||
# does NOT mean the guard is untested. `scripts/ci-prove-ban-detects.sh` is graded NONE and is driven
|
||||
# end to end by `test_ci_release_path_scan_job.py`. Nominating a proof is a judgement about which
|
||||
# test is THE proof, which is #775's scope; this file can only verify one afterwards.
|
||||
|
||||
UNDECLARED: dict[str, str] = {
|
||||
# NO GROUPING. An earlier version sorted these into "driven through their deciding path" and
|
||||
# "not driven at all", and the sort was wrong twice in successive review rounds — in both
|
||||
# directions, over entries whose own text said the opposite. A category above a list is a second
|
||||
# classification of the same facts, and it drifts the moment one entry's situation changes. Each
|
||||
# entry states its own case instead.
|
||||
#
|
||||
# THE TWO THINGS THAT GO MISSING ARE DIFFERENT, and which one it is decides where the work goes.
|
||||
# A guard may be DRIVEN — `test_hook_fire_log.py` executes most hooks through their real deciding
|
||||
# branch, its matrix asserting instrumentation TRANSPARENCY (the wrapped and unwrapped runs
|
||||
# agree), never that the decision is right or that a particular clause produced it — and still
|
||||
# have no NOMINATED proof and no witnessed clause disarm. Nominating one is a judgement about
|
||||
# which test is THE proof, which is #775's scope; this file can only verify one afterwards. A
|
||||
# guard nothing executes at all needs the test first.
|
||||
#
|
||||
# `NONE` in the inventory means the row nominates no proof ref. It does NOT mean untested.
|
||||
".claude/hooks/decisions-guard.sh": "Driven to a block and to a pass by test_hook_fire_log.py's "
|
||||
"constructed cases, which assert transparency rather than the decision. No nominated proof, and "
|
||||
"no clause disarmed.",
|
||||
".claude/hooks/prepush-clean-worktree-check.sh": "Driven with a file both modified in the tree "
|
||||
"and present in the pushed set, by test_hook_fire_log.py, for transparency. No nominated proof, "
|
||||
"and no clause disarmed.",
|
||||
".claude/hooks/prepush-donewhen.sh": "Driven against a stub Gitea by test_hook_fire_log.py, so "
|
||||
"its real blocking path is reached — for transparency. No nominated proof, and no clause "
|
||||
"disarmed.",
|
||||
".claude/hooks/prepush-rebase-check.sh": "BEHAVIOUR-ONLY. A named test drives it and "
|
||||
"test_hook_fire_log.py reaches its behind-origin block, but which clause carries that decision "
|
||||
"has not been established by disarming one.",
|
||||
".claude/hooks/pretooluse-agent-ram.sh": "Driven at 5% and 15% free memory through a stubbed "
|
||||
"`memory_pressure`, by test_hook_fire_log.py, for transparency. No nominated proof, and no "
|
||||
"clause disarmed.",
|
||||
".claude/hooks/pretooluse-agent-model.sh": "Driven with and without a `model` in the payload by "
|
||||
"test_hook_fire_log.py's matrix, for transparency. No nominated proof, and no clause disarmed.",
|
||||
".claude/hooks/pretooluse-bash-guard.sh": "Driven with an ETV_UPDATE_GOLDENS command and a "
|
||||
"harmless one by test_hook_fire_log.py's matrix, for transparency. No nominated proof, and no "
|
||||
"clause disarmed.",
|
||||
".claude/hooks/pretooluse-nav-guard.sh": "Driven with an `/iptv/` URL by test_hook_fire_log.py's "
|
||||
"matrix, for transparency. No nominated proof, and no clause disarmed.",
|
||||
".claude/hooks/design-sync-reminder.sh": "Driven by test_hook_fire_log.py, which gives it its "
|
||||
"start/finish arguments and works around its self-throttle — for transparency, and never to the "
|
||||
"one-shot branch that fires on the first Stop after a UI change and then allows. A proof has to "
|
||||
"model that state transition rather than a single invocation.",
|
||||
".claude/hooks/pretooluse-merge-consent.sh": "BEHAVIOUR-ONLY. Several suites execute it, but consent "
|
||||
"is derived from several independent conditions, so which one a given red belongs to has to be "
|
||||
"established before a clause can be named.",
|
||||
".husky/commit-msg": "Outside the hook-fire population (that globs `.claude/hooks/*.sh`) and "
|
||||
"executed by no test: the repositories the suite builds are fresh `git init`s that never install "
|
||||
"husky, so the hook is absent rather than bypassed. A proof has to install or invoke it.",
|
||||
".husky/pre-commit": "Runs lint-staged, the decisions guard, the root-PNG check and the format "
|
||||
"gate. Only the decisions guard has an inventory row of its own — root-PNG and format are INLINE "
|
||||
"here, so this one row is the whole classification of both, and neither has a proof. Executed by "
|
||||
"no test, and nothing observes the dispatch itself.",
|
||||
"scripts/check-kickoff-guard.sh": "Nothing drives it. A proof needs a tree carrying a revived "
|
||||
"#237 reference, which is cheap and simply not written.",
|
||||
"scripts/check-review-verdict.sh": "BEHAVIOUR-ONLY. Its named test feeds the real script an "
|
||||
"input only one clause rejects, which proves it reacts, not that the clause is load-bearing.",
|
||||
"scripts/ci-detect-already-validated.sh": "Blocks nothing directly — it feeds the skip gate. The "
|
||||
"consequence a mutation would have to be observed through is a job that skips, which is visible "
|
||||
"only in a workflow run.",
|
||||
"scripts/ci-detect-docs-only.sh": "Same shape: it feeds the skip gate rather than blocking, so "
|
||||
"its effect is visible only in a workflow run, not in this suite.",
|
||||
"scripts/ci-prove-ban-detects.sh": "Driven end to end by test_ci_release_path_scan_job.py, and "
|
||||
"it runs a mutation of its own at CI time. Grading it here needs a decision about what a second "
|
||||
"mutation would add; the row nominates no ref today.",
|
||||
"scripts/e2e-functional.sh": "Needs a running instance. Its clauses are HTTP contract "
|
||||
"assertions, so a proof means booting the app — `scripts/e2e-local.sh`'s job, not this "
|
||||
"harness's.",
|
||||
"scripts/jq-preflight.sh": "BEHAVIOUR-ONLY. Its named test drives the real script below the "
|
||||
"version floor; no clause has been disarmed to show the floor comparison is what refuses.",
|
||||
"scripts/post-review-verdict.sh": "BEHAVIOUR-ONLY, and the most valuable upgrade on this list: "
|
||||
"it writes the required `review-verdict/h10` status. Its re-read-and-compare has several arms "
|
||||
"and naming one as THE clause needs the judgement #790 declines to make blind.",
|
||||
"scripts/pr-changed-files.sh": "BEHAVIOUR-ONLY. Its named test feeds a short page to the real "
|
||||
"enumeration; the pagination clause has not been disarmed.",
|
||||
"scripts/tests/test_ci_release_path_scan_job.py": "A GUARD that is a test, so the mutation would "
|
||||
"have to go into the guarded artifact — the release-path scan job in the workflow. Which "
|
||||
"weakening of that job is THE defect it exists to catch has not been settled.",
|
||||
}
|
||||
@@ -0,0 +1,197 @@
|
||||
"""`pretooluse-bom-guard.sh` actually detects a BOM — including where `xxd` does not exist.
|
||||
|
||||
The guard compared `head -c3 <file> | xxd -p` against `efbbbf`. **`xxd` ships with vim and is absent
|
||||
on plain Linux hosts, including this repo's CI runner** (verified there directly). On such a host the
|
||||
command substitution yields the empty string, never equals `efbbbf`, and the guard allows every BOM
|
||||
in silence. It had been fail-open on any machine without vim since it was written, and nothing
|
||||
noticed because `docs/guard-inventory.md` graded it `NONE` — no proof it could go red.
|
||||
|
||||
It surfaced only when an unrelated change (ersatztv#776) added an assertion that this hook must
|
||||
reach a real decision, and that assertion ran on Linux. Every prior review of this guard ran on
|
||||
macOS, where `xxd` exists — the *environment* was a sampled population, and the sample was
|
||||
unanimous.
|
||||
|
||||
So this file exists to make the guard's detection load-bearing rather than assumed:
|
||||
|
||||
* it drives the real hook end to end, through the real payload shape;
|
||||
* it drives it with `xxd` REMOVED FROM PATH, which is the regression;
|
||||
* and it performs a clause-level mutation — the comparison is disarmed in an isolated copy and the
|
||||
guard must stop detecting — which is what `testing.guard-ships-with-mutation-proof` asks for and
|
||||
what this guard has never had.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[2]
|
||||
HOOK = REPO_ROOT / ".claude" / "hooks" / "pretooluse-bom-guard.sh"
|
||||
|
||||
BOM = b"\xef\xbb\xbf"
|
||||
|
||||
|
||||
def _git(cwd: Path, *args: str) -> None:
|
||||
subprocess.run(
|
||||
["git", *args],
|
||||
cwd=str(cwd),
|
||||
check=True,
|
||||
capture_output=True,
|
||||
env={
|
||||
**os.environ,
|
||||
"GIT_AUTHOR_NAME": "t",
|
||||
"GIT_AUTHOR_EMAIL": "t@e",
|
||||
"GIT_COMMITTER_NAME": "t",
|
||||
"GIT_COMMITTER_EMAIL": "t@e",
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def _repo_with(tmp_path: Path, name: str, content: bytes) -> Path:
|
||||
# The guard scopes itself to repositories whose toplevel path matches `*ersatztv*`, so the
|
||||
# directory name is load-bearing, not decoration.
|
||||
repo = tmp_path / "ersatztv-scratch"
|
||||
repo.mkdir()
|
||||
_git(repo, "init", "-q", ".")
|
||||
(repo / name).write_bytes(content)
|
||||
_git(repo, "add", name)
|
||||
return repo
|
||||
|
||||
|
||||
def _run(hook: Path, repo: Path, env: dict) -> tuple[int, bytes]:
|
||||
payload = json.dumps(
|
||||
{
|
||||
"session_id": "s",
|
||||
"hook_event_name": "PreToolUse",
|
||||
"tool_name": "Bash",
|
||||
"cwd": str(repo),
|
||||
"tool_input": {"command": "git commit -m x"},
|
||||
}
|
||||
)
|
||||
p = subprocess.run(
|
||||
["bash", str(hook)], input=payload.encode(), capture_output=True, cwd=str(repo), env=env, timeout=60
|
||||
)
|
||||
return p.returncode, p.stdout
|
||||
|
||||
|
||||
def _path_without_xxd(tmp_path: Path) -> dict:
|
||||
"""A PATH that resolves everything the hook needs EXCEPT `xxd`.
|
||||
|
||||
Rebuilding PATH from symlinks rather than just dropping directories, because `xxd` usually lives
|
||||
in the same directory as `git` and `od`; removing that directory would starve the hook of tools
|
||||
it legitimately needs and the test would pass for the wrong reason.
|
||||
"""
|
||||
import re
|
||||
|
||||
# DERIVED from the hook, not hand-listed. The hand-listed version symlinked six tools the hook
|
||||
# never calls and omitted `tail`, which it does call for `cd`-detection — so a payload whose cwd
|
||||
# differs from the repo took a `tail: command not found` path and the guard ALLOWED a BOM for a
|
||||
# reason that had nothing to do with xxd. A stand-in PATH that starves the subject is a test
|
||||
# passing for the wrong reason, which is the whole subject of this file.
|
||||
words = set(re.findall(r"\b([a-z][a-z0-9_-]*)\b", HOOK.read_text()))
|
||||
bindir = tmp_path / "nobin"
|
||||
bindir.mkdir()
|
||||
linked = []
|
||||
for tool in sorted(words - {"xxd"}):
|
||||
found = shutil.which(tool)
|
||||
if found and not (bindir / tool).exists():
|
||||
(bindir / tool).symlink_to(found)
|
||||
linked.append(tool)
|
||||
# `head` is deliberately NOT here: the `od` change removed the only `head` call, and asserting a
|
||||
# tool the hook no longer uses is how a stand-in PATH drifts from its subject.
|
||||
for required in ("bash", "git", "od", "tr", "sort", "tail"):
|
||||
assert shutil.which(required, path=str(bindir)), (
|
||||
f"the stand-in PATH lost `{required}`, which the hook needs — the test would then pass "
|
||||
"because the guard was starved, not because it detected anything"
|
||||
)
|
||||
assert not shutil.which("xxd", path=str(bindir)), "the stand-in PATH still resolves xxd"
|
||||
return {**os.environ, "PATH": str(bindir)}
|
||||
|
||||
|
||||
# ------------------------------------------------------------------------------------------------
|
||||
# ANTI-VACUITY — if the fixture stops producing a BOM file, every assertion below is meaningless.
|
||||
# ------------------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_the_fixture_really_stages_a_BOM(tmp_path):
|
||||
repo = _repo_with(tmp_path, "Bad.cs", BOM + b"class A {}\n")
|
||||
assert (repo / "Bad.cs").read_bytes()[:3] == BOM
|
||||
staged = subprocess.run(
|
||||
["git", "diff", "--name-only", "--cached"], cwd=str(repo), capture_output=True, text=True
|
||||
).stdout.split()
|
||||
assert staged == ["Bad.cs"], f"nothing was staged, so the guard would have nothing to read: {staged}"
|
||||
|
||||
|
||||
# ------------------------------------------------------------------------------------------------
|
||||
# THE GUARD DECIDES
|
||||
# ------------------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_a_staged_BOM_is_DENIED(tmp_path):
|
||||
repo = _repo_with(tmp_path, "Bad.cs", BOM + b"class A {}\n")
|
||||
rc, out = _run(HOOK, repo, dict(os.environ))
|
||||
assert rc == 0, "the hook communicates by printing, and must always exit 0"
|
||||
assert b'"permissionDecision": "deny"' in out or b'"permissionDecision":"deny"' in out, (
|
||||
f"a staged BOM-carrying .cs was not denied: {out!r}"
|
||||
)
|
||||
|
||||
|
||||
def test_a_staged_BOM_is_DENIED_when_xxd_DOES_NOT_EXIST(tmp_path):
|
||||
"""THE REGRESSION. This is the case that was silently allowed on every host without vim."""
|
||||
repo = _repo_with(tmp_path, "Bad.cs", BOM + b"class A {}\n")
|
||||
rc, out = _run(HOOK, repo, _path_without_xxd(tmp_path))
|
||||
assert rc == 0
|
||||
assert b'"permissionDecision": "deny"' in out or b'"permissionDecision":"deny"' in out, (
|
||||
"with `xxd` absent the guard allowed a BOM. That is the fail-open this file exists to "
|
||||
f"prevent, and it is the state every non-vim host was in: {out!r}"
|
||||
)
|
||||
|
||||
|
||||
def test_a_clean_file_is_ALLOWED(tmp_path):
|
||||
"""The negative control. A guard that denies everything would pass the tests above."""
|
||||
repo = _repo_with(tmp_path, "Good.cs", b"class A {}\n")
|
||||
rc, out = _run(HOOK, repo, _path_without_xxd(tmp_path))
|
||||
assert rc == 0
|
||||
assert out == b"", f"a BOM-free file was not allowed silently: {out!r}"
|
||||
|
||||
|
||||
# ------------------------------------------------------------------------------------------------
|
||||
# THE MUTATION PROOF — disarm the comparison alone, detection must stop
|
||||
# ------------------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_DISARMING_the_BOM_comparison_stops_detection(tmp_path):
|
||||
"""`testing.guard-ships-with-mutation-proof`, which this guard has never carried.
|
||||
|
||||
The clause is disarmed in an isolated copy — the comparison is pointed at a byte sequence no
|
||||
file starts with — and the guard must go quiet. If it still denies, the deny is coming from
|
||||
somewhere other than the clause the guard is supposed to hang on, and the tests above prove
|
||||
nothing about it.
|
||||
"""
|
||||
text = HOOK.read_text()
|
||||
marker = '= "efbbbf" ]; then'
|
||||
assert marker in text, (
|
||||
"the BOM comparison has moved; retarget this mutation rather than loosening it — a mutation "
|
||||
"that silently stops mutating is the failure this file is about"
|
||||
)
|
||||
mutated = tmp_path / "mutated-bom-guard.sh"
|
||||
mutated.write_text(text.replace(marker, '= "deadbeef" ]; then', 1))
|
||||
|
||||
repo = _repo_with(tmp_path, "Bad.cs", BOM + b"class A {}\n")
|
||||
|
||||
# POSITIVE CONTROL FIRST. Without it this test passes when the guard detects NOTHING AT ALL —
|
||||
# verified: run against the pre-fix hook on Linux, where xxd is absent, and "the mutant is
|
||||
# silent" was trivially true. A green mutation proof over a dead check is the exact failure
|
||||
# `guard-ships-with-mutation-proof` exists to stop, and the inventory row cites THIS function.
|
||||
rc_live, out_live = _run(HOOK, repo, dict(os.environ))
|
||||
assert rc_live == 0 and b"deny" in out_live, (
|
||||
"the UNMUTATED guard did not detect the BOM, so 'the mutant is silent' proves nothing about "
|
||||
f"the clause: {out_live!r}"
|
||||
)
|
||||
|
||||
rc, out = _run(mutated, repo, dict(os.environ))
|
||||
assert rc == 0
|
||||
assert out == b"", f"disarming the BOM comparison did not stop detection, so it is not load-bearing: {out!r}"
|
||||
@@ -0,0 +1,546 @@
|
||||
"""`scripts/build_decisions_catalog.py`'s `--check` path — the one CI actually runs.
|
||||
|
||||
`decisions-guard` (`.gitea/workflows/pr-checks.yml`, the "Active catalog in sync" step) runs
|
||||
`python3 scripts/build_decisions_catalog.py --check`. `scripts/tests/test_build_catalog.py` covers
|
||||
`render_catalog()` directly and never calls `main()` at all, so nothing there proves:
|
||||
|
||||
* that `main()`'s stale-detection comparison (`want.strip() != have.strip()`) is load-bearing —
|
||||
a version that always agreed would pass every existing test;
|
||||
* that the `if __name__ == "__main__": raise SystemExit(main())` wiring actually turns a stale
|
||||
catalog into a non-zero process exit code, which is the only thing CI's `run:` step reads.
|
||||
|
||||
`testing.guard-ships-with-mutation-proof` (#775) is explicit that a guard is not tested because a
|
||||
test *involving* it passes: it ships with a mutation proof — disarm the guard's own clause, alone,
|
||||
and a named test must go red. This file is that proof for the catalog-guard, plus the subprocess
|
||||
proof that the `__main__` wiring is connected (the #751/#719 shape the decision record names: a
|
||||
green `main()` behind dead wiring).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib.util
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
import yaml
|
||||
|
||||
import scripts.build_decisions_catalog as bc
|
||||
import scripts.decisions_lib as dl
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[2]
|
||||
CATALOG_SCRIPT = REPO_ROOT / "scripts" / "build_decisions_catalog.py"
|
||||
DECISIONS_LIB = REPO_ROOT / "scripts" / "decisions_lib.py"
|
||||
SCRIPTS_INIT = REPO_ROOT / "scripts" / "__init__.py"
|
||||
|
||||
|
||||
def _current_catalog_text() -> str:
|
||||
return bc.render_catalog(dl.all_active_records())
|
||||
|
||||
|
||||
# ------------------------------------------------------------------------------------------------
|
||||
# ANTI-VACUITY — if the real corpus is empty, every assertion below passes for nothing.
|
||||
# ------------------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_the_real_corpus_is_non_empty_and_renders_a_real_catalog():
|
||||
records = dl.all_active_records()
|
||||
active = [r for r in records if r.status == "active" and r.key]
|
||||
assert active, "no active decision records were parsed — every test below would be vacuous"
|
||||
|
||||
text = _current_catalog_text()
|
||||
assert bc.BANNER in text, "render_catalog produced no banner — not a real catalog document"
|
||||
assert f"`{active[0].key}`" in text, (
|
||||
"render_catalog produced no row for a known active record — not a real catalog document"
|
||||
)
|
||||
|
||||
|
||||
# ------------------------------------------------------------------------------------------------
|
||||
# `main(["--check"])` — the comparison CI reads
|
||||
# ------------------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_check_returns_0_when_OUTPUT_matches_render_catalog(tmp_path, monkeypatch):
|
||||
fresh = _current_catalog_text()
|
||||
output = tmp_path / "README.md"
|
||||
output.write_text(fresh.rstrip("\n") + "\n", encoding="utf-8")
|
||||
monkeypatch.setattr(bc, "OUTPUT", output)
|
||||
|
||||
assert bc.main(["--check"]) == 0
|
||||
|
||||
|
||||
def test_check_returns_1_when_OUTPUT_is_stale(tmp_path, monkeypatch):
|
||||
fresh = _current_catalog_text()
|
||||
output = tmp_path / "README.md"
|
||||
# Append a line: the on-disk file no longer matches what render_catalog would produce.
|
||||
output.write_text(fresh.rstrip("\n") + "\nEXTRA STALE LINE\n", encoding="utf-8")
|
||||
monkeypatch.setattr(bc, "OUTPUT", output)
|
||||
|
||||
assert bc.main(["--check"]) == 1
|
||||
|
||||
|
||||
def test_generate_then_check_round_trips(tmp_path, monkeypatch):
|
||||
"""The no-argument path WRITES the catalog, and a subsequent --check must then pass.
|
||||
|
||||
This pins the property CI depends on: generate and check agree. If they ever diverged, `main([])`
|
||||
would produce a file that `main(["--check"])` immediately rejects — a self-contradiction that
|
||||
would make the generator useless for fixing the exact problem `--check` reports.
|
||||
"""
|
||||
output = tmp_path / "README.md"
|
||||
assert not output.exists()
|
||||
monkeypatch.setattr(bc, "OUTPUT", output)
|
||||
|
||||
assert bc.main([]) == 0
|
||||
assert output.exists(), "main([]) with no --check must write OUTPUT"
|
||||
written = output.read_text(encoding="utf-8")
|
||||
assert written.strip() == _current_catalog_text().strip()
|
||||
|
||||
assert bc.main(["--check"]) == 0, "the file main([]) just wrote must satisfy main(['--check'])"
|
||||
|
||||
|
||||
# ------------------------------------------------------------------------------------------------
|
||||
# THE `__main__` WIRING — proof CI's subprocess invocation actually surfaces staleness
|
||||
# ------------------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _seed_decisions_copy(root: Path) -> None:
|
||||
"""Copy only what build_decisions_catalog.py + decisions_lib.py need to resolve a real corpus."""
|
||||
(root / "scripts").mkdir(parents=True)
|
||||
(root / "scripts" / "__init__.py").write_bytes(SCRIPTS_INIT.read_bytes())
|
||||
(root / "scripts" / "build_decisions_catalog.py").write_bytes(CATALOG_SCRIPT.read_bytes())
|
||||
(root / "scripts" / "decisions_lib.py").write_bytes(DECISIONS_LIB.read_bytes())
|
||||
|
||||
docs = root / "docs"
|
||||
docs.mkdir()
|
||||
(docs / "decisions.md").write_bytes((REPO_ROOT / "docs" / "decisions.md").read_bytes())
|
||||
|
||||
dst_decisions = docs / "decisions"
|
||||
src_decisions = REPO_ROOT / "docs" / "decisions"
|
||||
dst_decisions.mkdir()
|
||||
for item in src_decisions.iterdir():
|
||||
if item.is_dir():
|
||||
_copy_tree(item, dst_decisions / item.name)
|
||||
else:
|
||||
(dst_decisions / item.name).write_bytes(item.read_bytes())
|
||||
|
||||
|
||||
def _copy_tree(src: Path, dst: Path) -> None:
|
||||
dst.mkdir(parents=True, exist_ok=True)
|
||||
for item in src.rglob("*"):
|
||||
rel = item.relative_to(src)
|
||||
target = dst / rel
|
||||
if item.is_dir():
|
||||
target.mkdir(parents=True, exist_ok=True)
|
||||
else:
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
target.write_bytes(item.read_bytes())
|
||||
|
||||
|
||||
WORKFLOW = REPO_ROOT / ".gitea" / "workflows" / "pr-checks.yml"
|
||||
|
||||
|
||||
def _active_runs(workflow: Path | None = None) -> list[str]:
|
||||
"""The `run` script of every step the `decisions-guard` job would ACTUALLY execute.
|
||||
|
||||
Parsed with `yaml.safe_load`, and returned WHOLE — not split into lines. Both choices are
|
||||
scar tissue.
|
||||
|
||||
Text-scanning for `run:` was round one, and cold review broke it three ways: a `run: |` block
|
||||
scalar was invisible; a job or step switched off still read as wired; and `run:` inside
|
||||
block-scalar *text* was extracted and executed. Round two parsed the YAML and matched a LINE
|
||||
beginning with `PYTHONPATH=.` — and review broke that too, with a heredoc:
|
||||
|
||||
run: |
|
||||
cat <<'EOF' > /dev/null
|
||||
PYTHONPATH=. python3 scripts/build_decisions_catalog.py --check
|
||||
EOF
|
||||
|
||||
The matched line is heredoc DATA. The extractor reported the guard as running, and the proof
|
||||
executed a command CI does not. Deciding which lines of a shell script are executed requires
|
||||
parsing shell, and `fixing-a-parser-bug-introduces-the-next-one` is explicit that this repo has
|
||||
lost that argument repeatedly — a regex over shell is not a parser, and round four would find
|
||||
round five.
|
||||
|
||||
So the line-level heuristic is WITHDRAWN. The whole `run` script is handed to `bash`, exactly as
|
||||
the runner does. The heredoc above then runs, writes to `/dev/null`, checks nothing, and exits
|
||||
0 — so the stale-catalog case fails to redden and the proof reports the defect instead of
|
||||
stepping around it. No shell parsing, and the ambiguous cases resolve by execution.
|
||||
"""
|
||||
doc = yaml.safe_load((workflow or WORKFLOW).read_text())
|
||||
|
||||
# The workflow's TRIGGERS, before its jobs. A `decisions-guard` job that is perfectly healthy
|
||||
# gates nothing if the workflow stopped running on pull requests, and starting at `jobs:` cannot
|
||||
# see that. Note `on` parses to the boolean True in YAML 1.1 (the Norway problem's cousin), so
|
||||
# the key is looked up both ways rather than assumed.
|
||||
triggers = (doc or {}).get("on", (doc or {}).get(True)) or {}
|
||||
names = set(triggers) if isinstance(triggers, dict) else {triggers} if isinstance(triggers, str) else set(triggers)
|
||||
assert "pull_request" in names, (
|
||||
"pr-checks.yml no longer runs on `pull_request`, so NO gate in it — including the catalog "
|
||||
f"guard — fires on a PR. Triggers found: {sorted(str(n) for n in names)}"
|
||||
)
|
||||
|
||||
jobs = (doc or {}).get("jobs") or {}
|
||||
job = jobs.get("decisions-guard")
|
||||
assert job is not None, (
|
||||
"no `decisions-guard` job in pr-checks.yml. Either it was renamed or it was removed — the "
|
||||
f"second is the far more serious finding. Jobs present: {sorted(jobs)}"
|
||||
)
|
||||
assert not _disabled(job), (
|
||||
"the `decisions-guard` job is disabled at the job level "
|
||||
f"(if: {job.get('if')!r}, continue-on-error: {job.get('continue-on-error')!r}), so nothing "
|
||||
"in it runs — including the catalog guard"
|
||||
)
|
||||
return [str(step["run"]) for step in (job.get("steps") or []) if step.get("run") and not _disabled(step)]
|
||||
|
||||
|
||||
def _unwrap(value: str) -> str:
|
||||
"""Strip an `${{ ... }}` expression wrapper, if present, and lowercase.
|
||||
|
||||
Written as a regex over the WHOLE value rather than `.strip("${{ }}")`, which strips a character
|
||||
SET — it would turn `"false}"` into `"false"` and reads as though it removed a wrapper it never
|
||||
checked for.
|
||||
"""
|
||||
inner = value.strip()
|
||||
m = re.fullmatch(r"\$\{\{(.*)\}\}", inner, flags=re.DOTALL)
|
||||
if m:
|
||||
inner = m.group(1)
|
||||
return inner.strip().lower()
|
||||
|
||||
|
||||
def _falsey(value) -> bool:
|
||||
"""A literal false, however this workflow dialect spells it.
|
||||
|
||||
`if: false`, `if: "false"` and `if: ${{ false }}` all mean never. The middle and last are the
|
||||
ones a text comparison misses; the last was a live false green — `${{ false }}` is the ordinary
|
||||
spelling in Actions-flavoured YAML, and it read as wired.
|
||||
|
||||
An expression that is merely falsy AT RUN TIME (`if: ${{ github.event_name == 'x' }}`) is not
|
||||
decidable here and is deliberately not guessed at.
|
||||
"""
|
||||
if value is False:
|
||||
return True
|
||||
if not isinstance(value, str):
|
||||
return False
|
||||
return _unwrap(value) == "false"
|
||||
|
||||
|
||||
def _truthy_literal(value) -> bool:
|
||||
if value is True:
|
||||
return True
|
||||
if not isinstance(value, str):
|
||||
return False
|
||||
return _unwrap(value) == "true"
|
||||
|
||||
|
||||
def _disabled(node: dict) -> bool:
|
||||
"""A job or step that cannot fail the run: switched off, or allowed to fail.
|
||||
|
||||
`continue-on-error: true` is the subtle one — the step still runs and still reports, but its
|
||||
failure does not fail the job, so it is not a gate.
|
||||
"""
|
||||
return _falsey(node.get("if", True)) or _truthy_literal(node.get("continue-on-error", False))
|
||||
|
||||
|
||||
def _ci_check_command(workflow: Path | None = None) -> list[str]:
|
||||
"""The catalog step's script, DERIVED from the workflow and executed whole.
|
||||
|
||||
`testing.guard-derives-population-from-source`: a hand-copied command is a second copy of the
|
||||
workflow that drifts silently, and this test's whole value is that it runs what CI runs.
|
||||
"""
|
||||
runs = _active_runs(workflow)
|
||||
matches = [r for r in runs if "build_decisions_catalog.py" in r and "--check" in r]
|
||||
assert matches, (
|
||||
"the `decisions-guard` job has no ACTIVE step mentioning "
|
||||
"`build_decisions_catalog.py --check`. The catalog guard has stopped running in CI — that "
|
||||
f"is the finding, not this test's failure. Active step scripts in that job: {runs}"
|
||||
)
|
||||
assert len(matches) == 1, f"expected exactly one such step, found {matches}"
|
||||
script = matches[0]
|
||||
assert "${{" not in script, (
|
||||
"the catalog step's script interpolates an Actions expression, which cannot be expanded "
|
||||
f"outside the runner — this proof would be executing something else: {script!r}"
|
||||
)
|
||||
# Substitute the interpreter only where `python3` is a bare command word. A plain
|
||||
# `str.replace` rewrites EVERY occurrence, including inside a path — `/usr/bin/python3` would
|
||||
# become `/usr/bin/<venv>/bin/python3` and fail with ENOENT, a red blaming the catalog guard for
|
||||
# something this line did.
|
||||
return ["bash", "-c", re.sub(r"(?<![\w/])python3\b", sys.executable, script)]
|
||||
|
||||
|
||||
def _proof_holds(workflow: Path, repo: Path) -> bool:
|
||||
"""Does the whole guarantee hold — fresh corpus passes AND stale corpus fails?
|
||||
|
||||
The disablement cases below assert on THIS rather than on whether extraction raises, because
|
||||
the ways a guard can stop gating do not all surface at the same place. Deletion and disablement
|
||||
surface as a failed extraction; a heredoc or an `echo` surfaces only when the script is run and
|
||||
reports success over a stale catalog. One predicate covers both.
|
||||
"""
|
||||
try:
|
||||
cmd = _ci_check_command(workflow)
|
||||
except AssertionError:
|
||||
return False
|
||||
|
||||
readme = repo / "docs" / "decisions" / "README.md"
|
||||
original = readme.read_text(encoding="utf-8")
|
||||
try:
|
||||
readme.write_text(original, encoding="utf-8")
|
||||
fresh = subprocess.run(cmd, cwd=str(repo), capture_output=True, text=True, timeout=60)
|
||||
if fresh.returncode != 0:
|
||||
return False
|
||||
readme.write_text(original + "\nSTALE INJECTED LINE\n", encoding="utf-8")
|
||||
stale = subprocess.run(cmd, cwd=str(repo), capture_output=True, text=True, timeout=60)
|
||||
return stale.returncode != 0
|
||||
finally:
|
||||
readme.write_text(original, encoding="utf-8")
|
||||
|
||||
|
||||
_STEP = (
|
||||
" - name: Active catalog in sync\n"
|
||||
" run: PYTHONPATH=. python3 scripts/build_decisions_catalog.py --check\n"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"label,mutate",
|
||||
[
|
||||
(
|
||||
"the step is COMMENTED OUT",
|
||||
lambda s: s.replace(
|
||||
_STEP,
|
||||
" # - name: Active catalog in sync\n"
|
||||
" # run: PYTHONPATH=. python3 scripts/build_decisions_catalog.py --check\n",
|
||||
1,
|
||||
),
|
||||
),
|
||||
("the step is DELETED", lambda s: s.replace(_STEP, "", 1)),
|
||||
(
|
||||
"the step is switched off with `if: false`",
|
||||
lambda s: s.replace(
|
||||
_STEP,
|
||||
" - name: Active catalog in sync\n if: false\n"
|
||||
" run: PYTHONPATH=. python3 scripts/build_decisions_catalog.py --check\n",
|
||||
1,
|
||||
),
|
||||
),
|
||||
(
|
||||
"the step is switched off with `if: ${{ false }}`",
|
||||
lambda s: s.replace(
|
||||
_STEP,
|
||||
" - name: Active catalog in sync\n if: ${{ false }}\n"
|
||||
" run: PYTHONPATH=. python3 scripts/build_decisions_catalog.py --check\n",
|
||||
1,
|
||||
),
|
||||
),
|
||||
(
|
||||
"the step is allowed to fail with `continue-on-error: true`",
|
||||
lambda s: s.replace(
|
||||
_STEP,
|
||||
" - name: Active catalog in sync\n continue-on-error: true\n"
|
||||
" run: PYTHONPATH=. python3 scripts/build_decisions_catalog.py --check\n",
|
||||
1,
|
||||
),
|
||||
),
|
||||
(
|
||||
"the whole JOB is switched off with `if: false`",
|
||||
lambda s: s.replace(
|
||||
" decisions-guard:\n name: decisions lifecycle\n runs-on: small\n"
|
||||
" if: github.event_name == 'pull_request'\n",
|
||||
" decisions-guard:\n name: decisions lifecycle\n runs-on: small\n if: false\n",
|
||||
1,
|
||||
),
|
||||
),
|
||||
(
|
||||
"the command survives only as TEXT in another step's `echo`",
|
||||
lambda s: s.replace(
|
||||
_STEP,
|
||||
" - name: Note\n run: |\n"
|
||||
" echo we no longer run: PYTHONPATH=. python3 "
|
||||
"scripts/build_decisions_catalog.py --check\n",
|
||||
1,
|
||||
),
|
||||
),
|
||||
(
|
||||
"the WORKFLOW no longer runs on pull requests",
|
||||
lambda s: s.replace("on:\n pull_request:", "on:\n workflow_dispatch:", 1),
|
||||
),
|
||||
(
|
||||
"the command survives only as HEREDOC DATA",
|
||||
lambda s: s.replace(
|
||||
_STEP,
|
||||
" - name: Note\n run: |\n"
|
||||
" cat <<'EOF' > /dev/null\n"
|
||||
" PYTHONPATH=. python3 scripts/build_decisions_catalog.py --check\n"
|
||||
" EOF\n",
|
||||
1,
|
||||
),
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_a_guard_that_stopped_RUNNING_is_DETECTED(tmp_path, label, mutate):
|
||||
"""Eight ways the catalog guard can stop gating, each of which must be caught.
|
||||
|
||||
Commenting out is one of them and was the only one the first version detected. The last two
|
||||
leave the command in the file, parseable and even matchable — as `echo` argument and as heredoc
|
||||
data — which is why the proof executes the step's whole script instead of a line lifted out of it.
|
||||
"""
|
||||
raw = WORKFLOW.read_text()
|
||||
mutated = mutate(raw)
|
||||
assert mutated != raw, f"the mutation for {label!r} matched nothing; RETARGET it"
|
||||
|
||||
alt = tmp_path / "pr-checks.yml"
|
||||
alt.write_text(mutated)
|
||||
repo = tmp_path / "repo-copy"
|
||||
repo.mkdir()
|
||||
_seed_decisions_copy(repo)
|
||||
|
||||
assert not _proof_holds(alt, repo), (
|
||||
f"the catalog guard still reported as gating when {label}. CI would run nothing and this "
|
||||
"file would report full coverage."
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"label,replacement",
|
||||
[
|
||||
(
|
||||
"block scalar",
|
||||
" - name: Active catalog in sync\n run: |\n"
|
||||
" PYTHONPATH=. python3 scripts/build_decisions_catalog.py --check\n",
|
||||
),
|
||||
(
|
||||
"backslash continuation",
|
||||
" - name: Active catalog in sync\n run: |\n"
|
||||
" PYTHONPATH=. python3 \\\n"
|
||||
" scripts/build_decisions_catalog.py --check\n",
|
||||
),
|
||||
(
|
||||
"a leading `set -euo pipefail`",
|
||||
" - name: Active catalog in sync\n run: |\n"
|
||||
" set -euo pipefail\n"
|
||||
" PYTHONPATH=. python3 scripts/build_decisions_catalog.py --check\n",
|
||||
),
|
||||
(
|
||||
"wrapped in a shell block",
|
||||
" - name: Active catalog in sync\n run: |\n"
|
||||
" if true; then\n"
|
||||
" PYTHONPATH=. python3 scripts/build_decisions_catalog.py --check\n"
|
||||
" fi\n",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_the_step_may_be_REFORMATTED_without_being_flagged(tmp_path, label, replacement):
|
||||
"""The negative controls for the eight above: a legitimate rewrite must NOT be flagged.
|
||||
|
||||
`run: |` is idiomatic in this very job (the sibling "Validate decision lifecycle" step uses it),
|
||||
and the text-scanning version reported that form as "the guard has stopped running in CI" — a
|
||||
false red whose message asserts a regression that has not happened. A detector that cannot tell
|
||||
a reformat from a removal trains its readers to ignore it, which is the whole subject of #806.
|
||||
|
||||
A line-matching version had to be taught each of these shapes one at a time, and the
|
||||
continuation case defeated two rounds of it. Executing the script gets all four for free: the
|
||||
question "is the guard still gating" is answered by running it, not by recognising how it was
|
||||
written.
|
||||
"""
|
||||
raw = WORKFLOW.read_text()
|
||||
mutated = raw.replace(_STEP, replacement, 1)
|
||||
assert mutated != raw, "retarget this reformatting; the step's text has changed"
|
||||
|
||||
alt = tmp_path / "pr-checks.yml"
|
||||
alt.write_text(mutated)
|
||||
repo = tmp_path / "repo-copy"
|
||||
repo.mkdir()
|
||||
_seed_decisions_copy(repo)
|
||||
|
||||
assert _proof_holds(alt, repo), f"a legitimate reformat ({label}) was reported as a removal"
|
||||
|
||||
|
||||
def test_CLI_subprocess_exits_nonzero_on_a_stale_catalog(tmp_path):
|
||||
"""The real CI invocation, as a subprocess, against a real corpus.
|
||||
|
||||
This is the `__main__` → `SystemExit(main())` wiring proof: `main()` returning 1 is worthless if
|
||||
the process still exits 0, and the workflow step reads nothing but the exit code. It is the
|
||||
#751/#719 shape the decision record names — a green result behind wiring that is not connected.
|
||||
|
||||
The FRESH half is not optional decoration. Without it this test passes whenever the subprocess
|
||||
dies for any reason at all — an import error, a missing file in the copy, a syntax error — none
|
||||
of which is the guard detecting anything. `arbitrary-sample-gives-false-negatives` in reverse:
|
||||
a non-zero exit is only evidence when the same harness is shown to exit zero on a clean corpus.
|
||||
"""
|
||||
repo = tmp_path / "repo-copy"
|
||||
repo.mkdir()
|
||||
_seed_decisions_copy(repo)
|
||||
cmd = _ci_check_command()
|
||||
|
||||
fresh = subprocess.run(cmd, cwd=str(repo), capture_output=True, text=True, timeout=60)
|
||||
assert fresh.returncode == 0, (
|
||||
"the copied corpus does not even pass --check when untouched, so a non-zero exit below "
|
||||
f"would be the harness failing rather than the guard firing: {fresh.stdout!r} {fresh.stderr!r}"
|
||||
)
|
||||
|
||||
readme = repo / "docs" / "decisions" / "README.md"
|
||||
readme.write_text(readme.read_text(encoding="utf-8") + "\nSTALE INJECTED LINE\n", encoding="utf-8")
|
||||
|
||||
stale = subprocess.run(cmd, cwd=str(repo), capture_output=True, text=True, timeout=60)
|
||||
assert stale.returncode != 0, (
|
||||
"the CLI wiring did not surface a stale catalog as a non-zero exit — CI would report "
|
||||
f"green over a stale README.md: stdout={stale.stdout!r} stderr={stale.stderr!r}"
|
||||
)
|
||||
|
||||
|
||||
# ------------------------------------------------------------------------------------------------
|
||||
# THE MUTATION PROOF — disarm `main()`'s stale-detection clause alone, --check must stop detecting
|
||||
# ------------------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _load_mutated_module(tmp_path: Path, mutated_source: str):
|
||||
mutated_path = tmp_path / "mutated_build_decisions_catalog.py"
|
||||
mutated_path.write_text(mutated_source, encoding="utf-8")
|
||||
spec = importlib.util.spec_from_file_location("mutated_build_decisions_catalog", mutated_path)
|
||||
assert spec and spec.loader
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
def test_MUTATION_disarming_the_stale_comparison_stops_detection(tmp_path, monkeypatch):
|
||||
"""`testing.guard-ships-with-mutation-proof` (#775).
|
||||
|
||||
`main()`'s ONLY stale-detection logic is the clause `want.strip() != have.strip()`. Disarm that
|
||||
clause alone — replace it with the constant `False` in an isolated copy of the module, changing
|
||||
nothing else — and `--check` must stop reporting staleness. If it still returns 1, the deny is
|
||||
coming from somewhere other than the clause the guard is supposed to hang on, and every test
|
||||
above proves nothing about it.
|
||||
"""
|
||||
text = CATALOG_SCRIPT.read_text(encoding="utf-8")
|
||||
clause = "want.strip() != have.strip()"
|
||||
assert clause in text, (
|
||||
"the stale-detection clause has moved or been reworded; RETARGET this mutation at its new "
|
||||
"location rather than loosening the string match — a mutation that silently stops mutating "
|
||||
"is the exact failure this file exists to catch"
|
||||
)
|
||||
mutated_source = text.replace(clause, "False", 1)
|
||||
assert mutated_source != text and clause not in mutated_source, (
|
||||
"the replacement did not change the source, so the mutant is the subject"
|
||||
)
|
||||
|
||||
fresh = _current_catalog_text()
|
||||
output = tmp_path / "README.md"
|
||||
output.write_text(fresh.rstrip("\n") + "\nEXTRA STALE LINE\n", encoding="utf-8")
|
||||
|
||||
# POSITIVE CONTROL FIRST. Without this, the mutation assertion below would pass just as well if
|
||||
# `main(["--check"])` never detected anything at all on this input — "the mutant is silent"
|
||||
# proves nothing unless the real guard is first shown to be loud on the exact same input.
|
||||
monkeypatch.setattr(bc, "OUTPUT", output)
|
||||
assert bc.main(["--check"]) == 1, (
|
||||
"the UNMUTATED guard did not detect the staleness on this input, so a silent mutant below "
|
||||
"would prove nothing about the clause"
|
||||
)
|
||||
|
||||
mutant = _load_mutated_module(tmp_path, mutated_source)
|
||||
mutant.OUTPUT = output
|
||||
assert mutant.main(["--check"]) == 0, (
|
||||
"disarming `want.strip() != have.strip()` alone did not stop --check from reporting "
|
||||
"staleness, so that clause is not what the guard's exit code hangs on"
|
||||
)
|
||||
@@ -0,0 +1,357 @@
|
||||
"""Proofs for `scripts/check-doc-narrative.py` (ersatztv#784).
|
||||
|
||||
Every case below is a defect a cold review DEMONSTRATED in the first, shell implementation. They are
|
||||
here because the never-fails invariant and the reported line numbers are both asserted in prose in
|
||||
four places (the script header, the workflow comment, `docs/guard-inventory.md` and
|
||||
`docs/remote-state-inventory.md`), and an invariant asserted only in prose is the shape this repo
|
||||
keeps getting wrong.
|
||||
|
||||
The line-number cases all compare against the TRUTH computed from the file on disk, never against a
|
||||
number written into the test — a hand-written expectation is a second copy of the parser.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
REPO = Path(__file__).resolve().parents[2]
|
||||
SCRIPT = REPO / "scripts" / "check-doc-narrative.py"
|
||||
|
||||
NARRATIVE = "I initially thought otherwise"
|
||||
HIT = re.compile(r"^::warning file=(?P<path>[^:]+)::(?P=path):(?P<line>\d+) ", re.MULTILINE)
|
||||
|
||||
|
||||
def run(cwd: Path, *args: str) -> subprocess.CompletedProcess:
|
||||
return subprocess.run([sys.executable, str(SCRIPT), *args], cwd=cwd, capture_output=True, text=True)
|
||||
|
||||
|
||||
def hits(out: str) -> set[tuple[str, int]]:
|
||||
return {(m.group("path"), int(m.group("line"))) for m in HIT.finditer(out)}
|
||||
|
||||
|
||||
def truth(root: Path, rel: str) -> set[tuple[str, int]]:
|
||||
"""Where the narrative marker ACTUALLY is, read back off disk."""
|
||||
text = (root / rel).read_text(encoding="utf-8")
|
||||
return {
|
||||
(rel, i)
|
||||
for i, line in enumerate(text.splitlines(), start=1)
|
||||
if "initially thought" in line or "an earlier draft" in line.lower()
|
||||
}
|
||||
|
||||
|
||||
def git(repo: Path, *args: str) -> None:
|
||||
subprocess.run(["git", *args], cwd=repo, check=True, capture_output=True)
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def repo(tmp_path: Path) -> Path:
|
||||
r = tmp_path / "r"
|
||||
(r / "docs").mkdir(parents=True)
|
||||
git(r.parent, "init", "-q", "r")
|
||||
git(r, "config", "user.email", "t@example.com")
|
||||
git(r, "config", "user.name", "t")
|
||||
(r / "docs" / "seed.md").write_text("seed\n", encoding="utf-8")
|
||||
git(r, "add", "-A")
|
||||
git(r, "commit", "-qm", "base")
|
||||
return r
|
||||
|
||||
|
||||
def commit(r: Path, msg: str = "c") -> None:
|
||||
git(r, "add", "-A")
|
||||
git(r, "commit", "-qm", msg)
|
||||
|
||||
|
||||
# --- the never-fails invariant -------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("args", "expected"),
|
||||
[
|
||||
((), "scanned "),
|
||||
(("--all",), "scanned "),
|
||||
(("--diff",), "--diff needs a base ref"), # no base ref at all
|
||||
(("--diff", ""), "--diff needs a base ref"), # `origin/` with base_ref unset
|
||||
(("--diff", "no-such-ref-xyz"), "SCANNED NOTHING"), # an unresolvable ref
|
||||
(("--diff", "origin/"), "SCANNED NOTHING"),
|
||||
(("--nonsense",), "unknown mode"),
|
||||
],
|
||||
)
|
||||
def test_every_argument_shape_exits_zero_HAVING_HANDLED_IT(repo: Path, args, expected: str) -> None:
|
||||
"""The header, the workflow comment and both inventories state exit 0 absolutely.
|
||||
|
||||
The expected message is asserted alongside the exit code on purpose: the script ends with a bare
|
||||
`except` that returns 0, so an exit-code-only assertion is satisfied by an unhandled crash and
|
||||
would pass against a script that handles none of these shapes.
|
||||
"""
|
||||
p = run(repo, *args)
|
||||
assert p.returncode == 0, f"args={args} exited {p.returncode}: {p.stderr}"
|
||||
assert expected in p.stdout, f"args={args} exited 0 but did not HANDLE it: {p.stdout!r}"
|
||||
assert "internal error" not in p.stdout, f"args={args} reached the last-resort handler: {p.stdout!r}"
|
||||
|
||||
|
||||
def test_an_unknown_mode_prints_usage_and_scans_nothing(repo: Path) -> None:
|
||||
"""Stated as the observable behaviour it actually pins. It is NOT a proof that the else-branch
|
||||
cannot reach `run_all`: the branch returns before the warnings are printed, so a mutant calling
|
||||
`run_all` there is silent and no black-box test can see it."""
|
||||
(repo / "docs" / "u.md").write_text(f"x\n{NARRATIVE}\n", encoding="utf-8")
|
||||
commit(repo)
|
||||
p = run(repo, "--nonsense")
|
||||
assert hits(p.stdout) == set()
|
||||
assert "scanned" not in p.stdout
|
||||
|
||||
|
||||
def test_a_DELETED_doc_is_not_reported_as_added_content(repo: Path) -> None:
|
||||
"""A deletion contributes no added lines.
|
||||
|
||||
Stated exactly: this pins the BEHAVIOUR, and it is NOT a proof of the `+++ /dev/null` arm, which
|
||||
is defensive — removing that arm reddens nothing, because a deletion yields no `+` lines either
|
||||
way. The script comment says the same. A docstring claiming a proof it does not have is worse
|
||||
than no docstring: it is the thing a later reader trusts instead of re-checking."""
|
||||
(repo / "docs" / "del.md").write_text(f"x\n{NARRATIVE}\n", encoding="utf-8")
|
||||
commit(repo)
|
||||
(repo / "docs" / "del.md").unlink()
|
||||
commit(repo)
|
||||
assert hits(run(repo, "--diff", "HEAD~1").stdout) == set()
|
||||
|
||||
|
||||
# Every knob DEMONSTRATED to break the parse — not every knob that reshapes diff output, which is a
|
||||
# universal nobody can check and which is the enumeration that failed three rounds running.
|
||||
# (`diff.mnemonicPrefix` is a live example of one that reshapes the header and has no row: it emits
|
||||
# `+++ w/f.md`. It is inert HERE for a reason worth stating exactly, because the obvious explanation
|
||||
# is the wrong one — not because `--dst-prefix=b/` beats it, though it does, but because git only
|
||||
# uses mnemonic prefixes when a diff side is the worktree or the index, and `run_diff` issues the
|
||||
# three-dot `base...HEAD` form, where git emits plain `a/`…`b/` either way. Measured, not reasoned.)
|
||||
# Three of these were each demonstrated turning
|
||||
# a real hit into `scanned 0 file(s)`, one at a time, in three separate rounds — which is why the fix
|
||||
# stopped pinning variants and removed the surface. The table is here so the next knob someone finds
|
||||
# gets a row instead of a round.
|
||||
FORMAT_KNOBS = [
|
||||
("core.quotePath", "true"), # quotes non-ASCII paths out of the population
|
||||
("diff.dstPrefix", "dst/"), # rewrites the header the path is read from
|
||||
("diff.srcPrefix", "src/"),
|
||||
("diff.noprefix", "true"),
|
||||
("color.diff", "always"), # injects ANSI escapes into every line
|
||||
("color.ui", "always"),
|
||||
("diff.renames", "false"), # turns a `git mv` into a whole-file add
|
||||
("diff.context", "9"), # a configured context must not beat the -U0 on the CLI
|
||||
("diff.external", "/bin/echo"), # replaces the output wholesale
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("key", "value"), FORMAT_KNOBS)
|
||||
def test_no_git_FORMAT_CONFIG_can_produce_a_false_clean(repo: Path, key: str, value: str) -> None:
|
||||
"""A false clean is the worst outcome available to an advisory check: it is indistinguishable
|
||||
from a real one and nobody looks twice."""
|
||||
(repo / "docs" / "pfx.md").write_text("x\n", encoding="utf-8")
|
||||
commit(repo)
|
||||
(repo / "docs" / "pfx.md").write_text(f"x\n{NARRATIVE}\n", encoding="utf-8")
|
||||
commit(repo)
|
||||
git(repo, "config", key, value)
|
||||
assert hits(run(repo, "--diff", "HEAD~1").stdout) == truth(repo, "docs/pfx.md"), f"{key}={value}"
|
||||
|
||||
|
||||
# Only PYTHONIOENCODING is listed. `LC_ALL=C` was here too and was VACUOUS — PEP 540 UTF-8 mode
|
||||
# means the pre-fix script passed it as well, so it read as a second witness where there was one.
|
||||
@pytest.mark.parametrize("env_name,env_value", [("PYTHONIOENCODING", "ascii")])
|
||||
def test_a_NON_UTF8_stdio_does_not_break_the_never_fails_invariant(repo: Path, env_name, env_value) -> None:
|
||||
"""Both the summary line and the last-resort handler carry U+2014, so under ascii stdio the code
|
||||
guaranteeing exit 0 was itself what raised."""
|
||||
import os
|
||||
|
||||
env = dict(os.environ, **{env_name: env_value})
|
||||
env.pop("PYTHONUTF8", None)
|
||||
p = subprocess.run([sys.executable, str(SCRIPT), "--all"], cwd=repo, capture_output=True, text=True, env=env)
|
||||
assert p.returncode == 0, f"{env_name}={env_value} exited {p.returncode}: {p.stderr}"
|
||||
|
||||
|
||||
def test_a_tracked_but_DELETED_doc_does_not_kill_the_run(repo: Path) -> None:
|
||||
"""`git ls-files` lists index entries; deleting a doc before committing is ordinary."""
|
||||
(repo / "docs" / "gone.md").write_text("x\n", encoding="utf-8")
|
||||
commit(repo)
|
||||
(repo / "docs" / "gone.md").unlink()
|
||||
p = run(repo, "--all")
|
||||
assert p.returncode == 0
|
||||
assert "skipped docs/gone.md" in p.stdout
|
||||
|
||||
|
||||
def test_an_unresolvable_base_REPORTS_that_it_scanned_nothing(repo: Path) -> None:
|
||||
"""A silent zero-file scan is indistinguishable from a clean one — the whole point of #751."""
|
||||
p = run(repo, "--diff", "no-such-ref-xyz")
|
||||
assert p.returncode == 0
|
||||
assert "SCANNED NOTHING" in p.stdout
|
||||
# The guard names the line this branch must NOT also print. It previously named "nothing to
|
||||
# flag", a string that occurs nowhere in this script (it belongs to the sibling parity step), so
|
||||
# it could never fail — a negative assertion over a literal that does not exist is not a guard.
|
||||
assert "scanned 0 file(s)" not in p.stdout, "printed a clean-looking summary after scanning nothing"
|
||||
|
||||
|
||||
def test_a_genuine_clean_scan_REPORTS_its_population(repo: Path) -> None:
|
||||
p = run(repo, "--all")
|
||||
assert "scanned 1 file(s); 0 advisory warning(s)" in p.stdout
|
||||
|
||||
|
||||
# --- line numbers, against truth read off disk ---------------------------------------------------
|
||||
|
||||
|
||||
def test_a_file_with_NO_trailing_newline_does_not_shift_later_lines(repo: Path) -> None:
|
||||
"""`\\ No newline at end of file` is a marker, not a line of the new file."""
|
||||
(repo / "docs" / "n.md").write_text("a\nb\nc", encoding="utf-8") # no trailing newline
|
||||
commit(repo)
|
||||
(repo / "docs" / "n.md").write_text(f"a\nb\nZ\n{NARRATIVE}\n", encoding="utf-8")
|
||||
commit(repo)
|
||||
assert hits(run(repo, "--diff", "HEAD~1").stdout) == truth(repo, "docs/n.md")
|
||||
|
||||
|
||||
def test_an_added_line_whose_TEXT_starts_with_plus_plus_is_content_not_a_header(repo: Path) -> None:
|
||||
"""Docs here embed diff output in fenced blocks, so `++ ` at column 0 is real."""
|
||||
(repo / "docs" / "p.md").write_text("p\n", encoding="utf-8")
|
||||
commit(repo)
|
||||
(repo / "docs" / "p.md").write_text(f"p\n++ a fenced diff line\n{NARRATIVE}\n", encoding="utf-8")
|
||||
commit(repo)
|
||||
assert hits(run(repo, "--diff", "HEAD~1").stdout) == truth(repo, "docs/p.md")
|
||||
|
||||
|
||||
def test_a_multi_hunk_file_numbers_every_hunk_from_its_own_header(repo: Path) -> None:
|
||||
body = [f"l{i}" for i in range(1, 31)]
|
||||
(repo / "docs" / "m.md").write_text("\n".join(body) + "\n", encoding="utf-8")
|
||||
commit(repo)
|
||||
body[4] = NARRATIVE # replace, early hunk
|
||||
body.insert(15, NARRATIVE) # pure insert, middle hunk
|
||||
body[-1] = NARRATIVE # replace, last hunk
|
||||
(repo / "docs" / "m.md").write_text("\n".join(body) + "\n", encoding="utf-8")
|
||||
commit(repo)
|
||||
assert hits(run(repo, "--diff", "HEAD~1").stdout) == truth(repo, "docs/m.md")
|
||||
|
||||
|
||||
def test_a_NON_ASCII_path_is_scanned_rather_than_silently_skipped(repo: Path) -> None:
|
||||
"""`core.quotePath` quotes the path, and a quoted path matches no scope rule — it vanishes."""
|
||||
(repo / "docs" / "café.md").write_text(f"x\n{NARRATIVE}\n", encoding="utf-8")
|
||||
commit(repo)
|
||||
assert hits(run(repo, "--all").stdout) == truth(repo, "docs/café.md")
|
||||
assert hits(run(repo, "--diff", "HEAD~1").stdout) == truth(repo, "docs/café.md")
|
||||
|
||||
|
||||
def test_a_final_UNTERMINATED_line_is_still_scanned_in_all_mode(repo: Path) -> None:
|
||||
(repo / "docs" / "t.md").write_text(f"ok\n{NARRATIVE}", encoding="utf-8") # no trailing newline
|
||||
commit(repo)
|
||||
assert hits(run(repo, "--all").stdout) == truth(repo, "docs/t.md")
|
||||
|
||||
|
||||
def test_a_RENAME_does_not_re_flag_the_whole_pre_existing_file(repo: Path) -> None:
|
||||
"""Without rename detection a `git mv` reports every line of the file as newly added."""
|
||||
(repo / "docs" / "r1.md").write_text("a\n" + f"{NARRATIVE}\n" + "b\n", encoding="utf-8")
|
||||
commit(repo)
|
||||
git(repo, "mv", "docs/r1.md", "docs/r2.md")
|
||||
commit(repo)
|
||||
assert hits(run(repo, "--diff", "HEAD~1").stdout) == set()
|
||||
|
||||
# `diff.renames` defaults to true, so the assertion above passes with or without the explicit
|
||||
# `--find-renames`. Turning the config off is what makes that flag load-bearing and this test a
|
||||
# real proof of it rather than a restatement of a git default.
|
||||
git(repo, "config", "diff.renames", "false")
|
||||
assert hits(run(repo, "--diff", "HEAD~1").stdout) == set(), "the --find-renames pin is not doing its job"
|
||||
|
||||
|
||||
# --- the population ------------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.parametrize("mode", ["--all", "--diff"])
|
||||
def test_decision_records_are_exempt_in_BOTH_modes(repo: Path, mode: str) -> None:
|
||||
for rel in ("docs/decisions/records/x/y.md", "docs/decisions/archive/x/y.md"):
|
||||
p = repo / rel
|
||||
p.parent.mkdir(parents=True, exist_ok=True)
|
||||
p.write_text(f"x\n{NARRATIVE}\n", encoding="utf-8")
|
||||
commit(repo)
|
||||
out = run(repo, mode, "HEAD~1").stdout if mode == "--diff" else run(repo, mode).stdout
|
||||
assert hits(out) == set()
|
||||
|
||||
|
||||
def test_markdown_outside_the_stated_population_is_not_scanned(repo: Path) -> None:
|
||||
"""The population is `docs/**/*.md` minus `docs/decisions/**`, plus root-level `*.md`. A skill
|
||||
under `.claude/` is out of scope, and the record's `mechanics:` says so."""
|
||||
(repo / ".claude" / "skills" / "s").mkdir(parents=True)
|
||||
(repo / ".claude" / "skills" / "s" / "SKILL.md").write_text(f"{NARRATIVE}\n", encoding="utf-8")
|
||||
(repo / "README.md").write_text(f"{NARRATIVE}\n", encoding="utf-8")
|
||||
commit(repo)
|
||||
assert hits(run(repo, "--all").stdout) == {("README.md", 1)}
|
||||
|
||||
|
||||
# --- the detector actually detects ---------------------------------------------------------------
|
||||
|
||||
|
||||
def test_a_MULTI_FILE_diff_scans_every_file_not_just_the_first(repo: Path) -> None:
|
||||
"""Every real CI run is multi-file. Without the `in_hunk` reset on `diff --git`, the parser stays
|
||||
inside the previous file's hunk and silently drops every file after the first."""
|
||||
for name in ("a", "b", "c"):
|
||||
(repo / "docs" / f"{name}.md").write_text("x\n", encoding="utf-8")
|
||||
commit(repo)
|
||||
for name in ("a", "b", "c"):
|
||||
(repo / "docs" / f"{name}.md").write_text(f"x\n{NARRATIVE}\n", encoding="utf-8")
|
||||
commit(repo)
|
||||
p = run(repo, "--diff", "HEAD~1")
|
||||
assert hits(p.stdout) == {(f"docs/{n}.md", 2) for n in ("a", "b", "c")}
|
||||
assert "scanned 3 file(s); 3 advisory warning(s)" in p.stdout
|
||||
|
||||
|
||||
def test_the_reported_POPULATION_COUNT_matches_the_files_actually_scanned(repo: Path) -> None:
|
||||
"""The count is the observable that made every false clean in this file's history visible. A
|
||||
mutant that never populated the scanned set reported `scanned 0 file(s); 3 warning(s)` — green,
|
||||
and self-contradictory."""
|
||||
(repo / "docs" / "one.md").write_text("x\n", encoding="utf-8")
|
||||
(repo / "docs" / "two.md").write_text("x\n", encoding="utf-8")
|
||||
commit(repo)
|
||||
(repo / "docs" / "one.md").write_text(f"x\n{NARRATIVE}\n", encoding="utf-8")
|
||||
(repo / "docs" / "two.md").write_text("x\nharmless\n", encoding="utf-8")
|
||||
commit(repo)
|
||||
assert "scanned 2 file(s); 1 advisory warning(s)" in run(repo, "--diff", "HEAD~1").stdout
|
||||
|
||||
|
||||
def test_a_NON_UTF8_LOCALE_does_not_silently_empty_the_scan(repo: Path) -> None:
|
||||
"""`subprocess.run(errors="replace")` is the other half of the locale channel: without it a
|
||||
UTF-8 doc under an ascii locale raises inside `git()`, the bare `except` catches it, and the
|
||||
whole scan degrades to nothing while still exiting 0."""
|
||||
import os
|
||||
|
||||
(repo / "docs" / "u8.md").write_text(f"héllo — em dash\n{NARRATIVE}\n", encoding="utf-8")
|
||||
commit(repo)
|
||||
env = dict(os.environ, LC_ALL="C", PYTHONUTF8="0", PYTHONIOENCODING="utf-8")
|
||||
p = subprocess.run(
|
||||
[sys.executable, str(SCRIPT), "--diff", "HEAD~1"],
|
||||
cwd=repo,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
env=env,
|
||||
)
|
||||
assert p.returncode == 0, p.stderr
|
||||
assert "internal error" not in p.stdout, p.stdout
|
||||
assert hits(p.stdout) == {("docs/u8.md", 2)}
|
||||
|
||||
|
||||
def test_the_pattern_is_case_insensitive(repo: Path) -> None:
|
||||
(repo / "docs" / "case.md").write_text("AN EARLIER DRAFT of this said otherwise\n", encoding="utf-8")
|
||||
commit(repo)
|
||||
assert hits(run(repo, "--all").stdout) == {("docs/case.md", 1)}
|
||||
|
||||
|
||||
def test_MUTATION_neutering_the_pattern_makes_a_known_hit_go_quiet(repo: Path, tmp_path: Path) -> None:
|
||||
"""The clause-level disarm: with PATTERNS unable to match, a file that IS flagged stops being
|
||||
flagged. Without this, every assertion above is satisfied by a detector that finds nothing."""
|
||||
(repo / "docs" / "d.md").write_text(f"x\n{NARRATIVE}\n", encoding="utf-8")
|
||||
commit(repo)
|
||||
assert hits(run(repo, "--all").stdout) == truth(repo, "docs/d.md")
|
||||
|
||||
disarmed = tmp_path / "disarmed.py"
|
||||
src = SCRIPT.read_text(encoding="utf-8")
|
||||
marker = "PATTERNS = re.compile("
|
||||
assert src.count(marker) == 1
|
||||
disarmed.write_text(
|
||||
src.replace(marker, 'PATTERNS = re.compile(r"(?!x)x" # disarmed\n or ', 1), encoding="utf-8"
|
||||
)
|
||||
p = subprocess.run([sys.executable, str(disarmed), "--all"], cwd=repo, capture_output=True, text=True)
|
||||
assert p.returncode == 0, p.stderr
|
||||
assert hits(p.stdout) == set(), "the disarmed detector still flagged something — the mutation did not take"
|
||||
@@ -419,3 +419,63 @@ def test_non_hex_head_is_an_input_error():
|
||||
text=True,
|
||||
)
|
||||
assert p.returncode == 2
|
||||
|
||||
|
||||
# --- read-side POLARITY regression, rescued from the withdrawn parity test (ersatztv#774) ------
|
||||
|
||||
|
||||
# The vocabulary as BOTH scripts spell it. A literal list, and per
|
||||
# `testing.guard-derives-population-from-source` that is legitimate HERE and would not be for a
|
||||
# completeness claim: the property below is PER-MEMBER — "each of these words classifies as exactly
|
||||
# one thing" — so a word missing from this list is simply an untested word, not a defect the list
|
||||
# conceals. It is emphatically NOT a claim that these are the only words the scripts accept. Proving
|
||||
# THAT needs one shared vocabulary both scripts read, which is ersatztv#788.
|
||||
POSITIVE_WORDS = ["MERGEABLE", "APPROVED", "LGTM"]
|
||||
NEGATIVE_WORDS = ["BLOCKED", "NOT-MERGEABLE"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("word", "expected"),
|
||||
[(w, "positive") for w in POSITIVE_WORDS] + [(w, "negative") for w in NEGATIVE_WORDS],
|
||||
)
|
||||
def test_each_verdict_word_retains_its_established_polarity(word, expected):
|
||||
"""The five established tokens still classify the way reviewers rely on. READ side only.
|
||||
|
||||
NAMED FOR WHAT IT IS. An earlier version called itself a disjointness test and its docstring
|
||||
said "no word may be in both vocabularies". Cold review was right that this is an overclaim:
|
||||
pinning the observable classification of five hardcoded tokens cannot establish a universal
|
||||
property over every token the scripts accept, and for THAT property an omitted token is not a
|
||||
vacuous pass — it is precisely the untested member. This is a polarity regression, and the
|
||||
honest scope is the five words listed. Universal disjointness needs one shared vocabulary both
|
||||
scripts read (ersatztv#788).
|
||||
|
||||
WHY IT SURVIVED ITS PARENT. That test asserted set equality between the write side's `case` arms
|
||||
and the read side's `POS_RE`/`NEG_RE` by parsing shell with regexes, and six review rounds each
|
||||
found another construction it mis-read; it was deleted rather than patched a seventh time. But
|
||||
it carried a SECOND, separable invariant that had nothing to do with parsing, and deleting the
|
||||
file silently took that with it — caught by cold review of the deletion, which is exactly the
|
||||
"enumerate what a workaround provided before removing it" rule
|
||||
(`process.enumerate-workaround-behaviors-before-deleting`).
|
||||
|
||||
WHY THIS VERSION IS SOUND WHERE ITS PARENT WAS NOT. It EXECUTES the real classifier instead of
|
||||
reading its source, so no shell construction can fool it. `check-review-verdict.sh` sets
|
||||
`is_pos` and `is_neg` from two INDEPENDENT `grep -iqE` calls, so a word in both patterns sets
|
||||
both flags and precedence decides.
|
||||
|
||||
WHICH DIRECTION THIS CATCHES, MEASURED RATHER THAN REASONED — the first draft of this docstring
|
||||
asserted it and was checked afterwards, which is the wrong order. `check-review-verdict.sh:212`
|
||||
reads `if [ "$is_pos" = 1 ]; then head_pos=1; else head_neg=1; fi`, so `is_pos` wins per line
|
||||
and an overlapping word resolves POSITIVE. Both mutations were run against this test:
|
||||
|
||||
* `blocked` added to `POS_RE` -> `BLOCKED` classifies `positive` -> RED. That is the dangerous
|
||||
direction — a verdict meant to block reporting as approval — and it is caught.
|
||||
* `mergeable` added to `NEG_RE` -> still `positive`, stays green. NOT a gap: because `is_pos`
|
||||
wins, that edit has no observable effect at all. `NEG_RE` is shadowed by `POS_RE` for any
|
||||
overlapping word, so there is no behaviour there to catch.
|
||||
"""
|
||||
got, rc = classify([verdict(word, HEAD)])
|
||||
assert (got, rc) == (expected, 0), (
|
||||
f"'{word}' classified as {got!r} (rc={rc}), expected {expected!r}. If it now matches BOTH "
|
||||
"POS_RE and NEG_RE, the classifier's two independent greps both fire and precedence picks "
|
||||
"the verdict instead of the reviewer — a word meant to block could report positive."
|
||||
)
|
||||
|
||||
@@ -0,0 +1,733 @@
|
||||
"""The dropped-step guard on docker-build.yml's two REQUIRED jobs (ersatztv#756).
|
||||
|
||||
WHAT THIS IS PROTECTING. A `run:` body the runner declines to interpolate is DROPPED, and the job
|
||||
still concludes `success` (ersatztv#751, `ci.workflow-run-body-no-expressions`). #751 fixed that in
|
||||
`review-verdict.yml`, where the consequence is fail-CLOSED — `review-verdict/h10` is absent and the
|
||||
merge is blocked. It left the two places where the same drop is fail-OPEN: `Build & test (.NET)` and
|
||||
`EF migration integrity (SQLite + MySql)` are the other two required contexts on `main`, so a dropped
|
||||
step there sends a required check green having done no work.
|
||||
|
||||
THE TESTS COME IN THREE KINDS AND NONE SUBSTITUTES FOR ANOTHER, which is the lesson #751 paid for:
|
||||
|
||||
* STATIC — the marker set and the guard's expectations agree, and the guard is positioned so it
|
||||
can actually run. Cheap, and the only kind that catches a NEW step added without a marker.
|
||||
* BEHAVIOURAL — the guard's real command line is EXECUTED against markers written by the steps'
|
||||
real marker lines, both extracted from the parsed workflow. A structural test cannot prove an
|
||||
exit code, and `exit 1` in a body is satisfiable by dead code.
|
||||
* A LIVE PROBE — that the runner still executes a LATER step after dropping an earlier one, on the
|
||||
BUILD lane rather than the `small` lane #751 measured. That is the premise the whole guard rests
|
||||
on and no test here can establish it; it is recorded in docs/ci-cd.md and on the issue.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
import yaml
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[2]
|
||||
# ASSESSED FOR ersatztv#806: this file has NO filesystem-derived population. Its members come from
|
||||
# the PARSED workflow (`_marked(job)` reads the marked steps out of `_DOC`), which is already an
|
||||
# authoritative machine-readable source, so the index changes nothing here. Its known residual is at
|
||||
# the other altitude — `MARKED_JOBS` is a hand-written mirror of the required contexts on `main`,
|
||||
# a SCOPE rather than a population, and `testing.guard-derives-population-from-source` already
|
||||
# carries it as this repo's canonical residual gap. #806 does not close it.
|
||||
WORKFLOW = REPO_ROOT / ".gitea" / "workflows" / "docker-build.yml"
|
||||
SCRIPT = REPO_ROOT / "scripts" / "ci-step-ran.sh"
|
||||
|
||||
# The jobs whose contexts branch protection REQUIRES on `main`. Read live on 2026-08-10:
|
||||
# Build ErsatzTV Image / Build & test (.NET) (pull_request)
|
||||
# Build ErsatzTV Image / EF migration integrity (SQLite + MySql) (pull_request)
|
||||
# review-verdict/h10
|
||||
# The third is guarded by test_pr_changed_files.py; these two are this file's subject. `build`,
|
||||
# `api-docs` and `format` are deliberately NOT here — they are not required, and all three
|
||||
# legitimately interpolate into a `run:` body, so extending the absolute rule to them would be false.
|
||||
# Per-step markers apply to the two REQUIRED contexts, where a dropped step is fail-OPEN.
|
||||
MARKED_JOBS = ("test", "migrations")
|
||||
|
||||
# The delimiter ban is WIDER than the marker set, and the extra job is not an afterthought.
|
||||
# `build`'s "Smoke + IPTV E2E" step runs AFTER `Build and push`, so on a `v*` tag the image is
|
||||
# already in the registry as the release candidate and this step is what decides whether it was ever
|
||||
# booted. A drop there publishes an unsmoked candidate and goes green, and `DeployStack jazz-media`
|
||||
# promotes exactly that image — not a "smaller cost than a required context", which is what an
|
||||
# earlier draft of the decision record claimed. Its two payloads moved into the step's `env:`, which
|
||||
# is the free half of the escape hatch, so the ban costs nothing there.
|
||||
#
|
||||
# `functional-e2e` is deliberately NOT here even though it is delimiter-free today: it is advisory by
|
||||
# declaration (not a required check, not a `needs:` of `build`), so the rule stays "ban where a drop
|
||||
# is consequential" rather than "ban wherever it happens to be free right now".
|
||||
# `api-docs` and `format` keep one delimiter each, both `github.base_ref` in a detect step, and gate
|
||||
# nothing that ships.
|
||||
DELIMITER_BAN_JOBS = ("test", "migrations", "build")
|
||||
|
||||
# THE RAW OPENER, not a closed `${{ … }}` pair — found by cold review. The runner's rewrite is
|
||||
# triggered by the OPENER; a closed-pair regex therefore misses `# ${{` with no closer, which would
|
||||
# sail through an "absolute" ban and still drop the step. Nothing in these jobs may contain the
|
||||
# opener at all, so matching it directly is both simpler and strictly stronger. `_EXPR` is kept for
|
||||
# reporting the payload of a well-formed one in the failure message.
|
||||
_OPENER = re.compile(r"\$\{\{")
|
||||
_EXPR = re.compile(r"\$\{\{(.*?)\}\}", re.S)
|
||||
_MARK = re.compile(r'ci-step-ran\.sh"?\s+mark\s+(\S+)')
|
||||
|
||||
|
||||
# ONE parse, shared. `yaml.safe_load` per call returns a fresh object graph, so an identity test
|
||||
# across two helpers (`steps[-1] is guard`) would compare structurally-equal but distinct dicts and
|
||||
# fail — or, worse in the other direction, an `is not` filter would exclude nothing and a step would
|
||||
# match as its own guard. That is not hypothetical: test_pr_changed_files.py records exactly this
|
||||
# going wrong in the #751 guard test, where the assertions then ran against the wrong step.
|
||||
_DOC = yaml.safe_load(WORKFLOW.read_text())
|
||||
|
||||
|
||||
def _doc():
|
||||
return _DOC
|
||||
|
||||
|
||||
def _steps(job: str):
|
||||
return _doc()["jobs"][job]["steps"]
|
||||
|
||||
|
||||
def _run_steps(job: str):
|
||||
return [s for s in _steps(job) if s.get("run")]
|
||||
|
||||
|
||||
def _guard(job: str):
|
||||
"""The trailing assert step. Located by CONTENT, never by index.
|
||||
|
||||
Locating it as `steps[-1]` here and then asserting it is last elsewhere would be circular — the
|
||||
position test would hold by construction. This finds the step that invokes the assert
|
||||
sub-command, and `test_the_guard_is_the_LAST_step` independently checks where it sits.
|
||||
"""
|
||||
hits = [s for s in _run_steps(job) if "ci-step-ran.sh assert" in s["run"]]
|
||||
assert len(hits) == 1, f"job '{job}' has {len(hits)} assert steps, expected exactly 1"
|
||||
return hits[0]
|
||||
|
||||
|
||||
def _marked(job: str):
|
||||
"""[(step, key)] for every step that records its own execution, in declaration order."""
|
||||
out = []
|
||||
for s in _run_steps(job):
|
||||
m = _MARK.search(s["run"])
|
||||
if m:
|
||||
out.append((s, m.group(1)))
|
||||
return out
|
||||
|
||||
|
||||
def _guard_buckets(job: str):
|
||||
"""(always_keys, gated_keys) as the guard's own argv spells them."""
|
||||
argv = _guard(job)["run"].split()
|
||||
assert "--always" in argv and "--gated" in argv, argv
|
||||
a, g = argv.index("--always"), argv.index("--gated")
|
||||
return argv[a + 1 : g], argv[g + 1 :]
|
||||
|
||||
|
||||
# Mirrors the `if:` every gated step in these jobs carries. Compared as a normalised string rather
|
||||
# than by parsing the expression: what matters is that a step's gating and the guard's bucketing are
|
||||
# the SAME condition, and any rewrite of one that is not mirrored in the other should be loud.
|
||||
SKIP_GATE = "steps.detect.outputs.docs_only!='true'&&steps.revalidate.outputs.skip!='true'"
|
||||
|
||||
|
||||
def _is_gated(step) -> bool:
|
||||
return re.sub(r"\s+", "", str(step.get("if", ""))) == SKIP_GATE
|
||||
|
||||
|
||||
# ------------------------------------------------------------------------------------------------
|
||||
# STATIC
|
||||
# ------------------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.parametrize("job", DELIMITER_BAN_JOBS)
|
||||
def test_the_delimiter_banned_jobs_have_NO_expression_delimiter_in_any_run_body(job):
|
||||
"""The absolute rule from `review-verdict.yml`, extended to the two required build jobs.
|
||||
|
||||
This is the cheaper and more general half of #756: the drop mechanism REQUIRES an opener in the
|
||||
scalar, so a job with none is immune by construction and the runtime markers are a backstop
|
||||
rather than the only line of defence.
|
||||
|
||||
The scope is the three jobs in `DELIMITER_BAN_JOBS` — see the comment there for why `build` is in
|
||||
and `functional-e2e` is not. Do NOT restate this docstring as "scoped to the required pair":
|
||||
round 2 moved `build`'s two payloads into `env:` and brought it into the ban, and this docstring
|
||||
sits directly above the decorator that parametrises over the wider set.
|
||||
|
||||
The escape hatch when a value really is needed is the step's `env:` block, which is interpolated
|
||||
PER VALUE, so a payload that does not evaluate cannot take the body with it.
|
||||
|
||||
The `run:` SCALAR AS PARSED, comments and all. A shell comment inside a `run:` body is NOT inert
|
||||
— that is the whole #751 defect — so this must never filter comments out. Ordinary YAML comments
|
||||
outside a `run:` body ARE inert and are not read here.
|
||||
"""
|
||||
offenders = []
|
||||
for s in _run_steps(job):
|
||||
for m in _OPENER.finditer(s["run"]):
|
||||
closed = _EXPR.match(s["run"], m.start())
|
||||
payload = closed.group(1).strip() if closed else "<unclosed opener>"
|
||||
offenders.append(f"{s.get('name', '?')}: {payload!r}")
|
||||
assert not offenders, (
|
||||
f"job '{job}' of docker-build.yml has an expression delimiter inside a run: body — "
|
||||
f"{offenders}. A dropped step in this job is CONSEQUENTIAL — `test`/`migrations` write "
|
||||
"REQUIRED status contexts, and `build` publishes the release candidate before its smoke step "
|
||||
"runs. Even in a comment a delimiter is unsafe: the runner rewrites the WHOLE body into a "
|
||||
"format(...) call, and if the payload does not parse it DROPS THE STEP and reports the job "
|
||||
"green — so the check passes having done no work (ersatztv#751/#756). Pass the value in "
|
||||
"through the step's `env:` "
|
||||
"block instead; to describe an expression in prose, name it rather than quoting the "
|
||||
"delimiters."
|
||||
)
|
||||
# ANTI-VACUITY. A walk that reached no bodies, or only the trivial ones, would make the
|
||||
# assertion above green while proving nothing. Counted against the job's own step list read
|
||||
# here, so a helper that silently stopped yielding steps is caught rather than rewarded.
|
||||
declared = sum(1 for s in _steps(job) if isinstance(s, dict) and s.get("run"))
|
||||
assert len(_run_steps(job)) == declared >= 3, (
|
||||
f"the walk reached {len(_run_steps(job))} run: bodies but job '{job}' declares {declared}"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("job", MARKED_JOBS)
|
||||
def test_every_consequential_run_step_marks_itself_as_its_FIRST_act(job):
|
||||
"""The completeness half — and the only test that catches a NEWLY ADDED step with no marker.
|
||||
|
||||
A guard that checks a fixed list can go quietly incomplete: someone adds a `Test SPA (part 2)`
|
||||
step, it is never marked, the guard never expects it, and a drop of exactly that step is
|
||||
invisible again. So the expectation is DERIVED from the workflow rather than written down twice.
|
||||
|
||||
EXEMPT: steps carrying `continue-on-error: true`. Those are advisory by construction (the
|
||||
peak-anon sampler, the coverage summary) — the workflow already declares that their failure must
|
||||
not redden the job, so their non-execution cannot be a fail-open either. Making them mandatory
|
||||
would be asserting the opposite of what `continue-on-error` means.
|
||||
|
||||
FIRST ACT, not merely present. A marker written at the END of a body records completion, not
|
||||
execution — and this repo has legitimate early-exit paths. More importantly a marker further down
|
||||
can be skipped by an early `exit 0` while the step did nothing, which is the fail-open again one
|
||||
line lower. `set -euo pipefail` is allowed to precede it: it cannot fail, and it is what makes
|
||||
the rest of the body honest.
|
||||
"""
|
||||
missing, late = [], []
|
||||
for s in _run_steps(job):
|
||||
if s.get("continue-on-error") is True or "ci-step-ran.sh assert" in s["run"]:
|
||||
continue
|
||||
m = _MARK.search(s["run"])
|
||||
if not m:
|
||||
missing.append(s.get("name", "?"))
|
||||
continue
|
||||
# By LINE, not by byte offset. The marker sits mid-line (the command is quoted and
|
||||
# prefixed with $GITHUB_WORKSPACE), so slicing at `m.start()` counts the marker's OWN line
|
||||
# prefix as a preceding command and reddens every correctly-written step.
|
||||
lines = s["run"].splitlines()
|
||||
at = next(i for i, ln in enumerate(lines) if _MARK.search(ln))
|
||||
preceding = [ln.strip() for ln in lines[:at] if ln.strip() and not ln.strip().startswith("#")]
|
||||
if [ln for ln in preceding if not ln.startswith("set -")]:
|
||||
late.append((s.get("name", "?"), preceding))
|
||||
assert not missing, (
|
||||
f"these run: steps of the REQUIRED job '{job}' do not record that they executed: {missing}. "
|
||||
"A step the runner drops concludes success, so without a marker its non-execution takes the "
|
||||
"whole required context green having done no work (ersatztv#756). Add "
|
||||
'`"${GITHUB_WORKSPACE:-.}/scripts/ci-step-ran.sh" mark <key>` as the step\'s first line and '
|
||||
"the key to the guard step's --always/--gated list."
|
||||
)
|
||||
assert not late, (
|
||||
f"these steps of '{job}' mark themselves only after other commands have run: {late}. The "
|
||||
"marker must be the first act, or a body that exits early records nothing while the guard "
|
||||
"still expects it — or worse, records success for work that did not happen."
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("job", MARKED_JOBS)
|
||||
def test_the_guard_expects_EXACTLY_the_set_of_marked_keys_in_the_right_bucket(job):
|
||||
"""Set equality in BOTH directions, plus the bucket, because each failure is silent differently.
|
||||
|
||||
A key marked but not expected → the guard never notices that step being dropped: a fail-open
|
||||
that looks fully guarded. A key expected but not marked → the guard reddens on every single run,
|
||||
which is fail-closed but reads as "this guard is broken" and is how a correct guard gets deleted.
|
||||
|
||||
The BUCKET has to match the step's own `if:`. A gated step listed under `--always` reddens every
|
||||
docs-only and already-validated run — the two paths whose entire purpose is to report green in
|
||||
seconds. An always-run step listed under `--gated` stops being checked the moment either skip
|
||||
gate fires, which is a fail-open on precisely the runs where least else is happening.
|
||||
"""
|
||||
marked = _marked(job)
|
||||
keys = [k for _, k in marked]
|
||||
assert len(keys) == len(set(keys)), (
|
||||
f"job '{job}' reuses a marker key: {[k for k in keys if keys.count(k) > 1]}. Two steps "
|
||||
"sharing a key means either one satisfies the guard for both, so dropping one is invisible."
|
||||
)
|
||||
always, gated = _guard_buckets(job)
|
||||
assert sorted(always + gated) == sorted(keys), (
|
||||
f"job '{job}': the guard expects {sorted(always + gated)} but the steps mark "
|
||||
f"{sorted(keys)}. Keys marked-but-unexpected are unguarded drops; keys "
|
||||
"expected-but-unmarked redden every run."
|
||||
)
|
||||
# AN UNRECOGNISED `if:` IS REJECTED, never silently bucketed — found by both reviewers. The
|
||||
# protocol only knows two conditions: absent (always runs) and exactly the skip gate. A marked
|
||||
# step carrying a third condition (`if: github.event_name == 'push'`, or the `always() && <gate>`
|
||||
# spelling the peak-anon steps already use) would fall through to "always", the suite would go
|
||||
# green, and the guard would then demand a step the runner legitimately skipped — reddening a
|
||||
# REQUIRED context and deadlocking `main`. There is already a near-miss in this file: `Report
|
||||
# peak container memory` carries that third spelling and escapes only because it is
|
||||
# `continue-on-error: true` and therefore exempt from marking.
|
||||
for step, key in marked:
|
||||
cond = re.sub(r"\s+", "", str(step.get("if", "")))
|
||||
assert cond in ("", SKIP_GATE), (
|
||||
f"job '{job}': marked step {step.get('name')!r} has an `if:` the guard protocol does not "
|
||||
f"model ({step.get('if')!r}). Only 'absent' and the exact skip gate are understood; "
|
||||
"anything else would be bucketed as --always and would fail the job on a run where the "
|
||||
"step is legitimately skipped. Extend the protocol deliberately, or leave the step "
|
||||
"unmarked."
|
||||
)
|
||||
want = "gated" if _is_gated(step) else "always"
|
||||
got = "gated" if key in gated else "always"
|
||||
assert want == got, (
|
||||
f"job '{job}': step {step.get('name')!r} is {want} (if: {step.get('if')!r}) but the "
|
||||
f"guard lists its key {key!r} under --{got}."
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("job", MARKED_JOBS)
|
||||
def test_the_guard_is_the_LAST_step_carries_no_if_and_is_not_advisory(job):
|
||||
"""Position and condition, which together are what make the guard reachable and quiet.
|
||||
|
||||
LAST, because a guard placed before a marked step reads a marker not yet written and fails on
|
||||
every run.
|
||||
|
||||
NO `if:` — a deliberate departure from the #751 guard's `if: always()`, and the thing most likely
|
||||
to be "corrected" back. That job has one real step, so `always()` costs nothing. These jobs have
|
||||
a dozen, and a genuine failure in an early one SKIPS every later step: an `always()` guard would
|
||||
then report "these steps never executed: typecheck web-test build dotnet-test" on top of every
|
||||
ordinary red build. That is the runner obeying its own gating, not a dropped step, and a guard
|
||||
that cries wolf on every red build gets deleted.
|
||||
|
||||
The default `if:` is `success()`, and the invariant that makes relying on it safe rather than
|
||||
lucky: this step is skipped only when an earlier step FAILED, and that failure already fails the
|
||||
job. So `guard skipped => job red`, and every path to a green job runs the guard. A dropped step
|
||||
is invisible precisely because it concludes `success` — which keeps the job green and therefore
|
||||
reaches here.
|
||||
|
||||
NOT `continue-on-error`, which would let it observe the failure and go green anyway — the whole
|
||||
defect, one attribute over.
|
||||
"""
|
||||
steps = _steps(job)
|
||||
guard = _guard(job)
|
||||
assert steps[-1] is guard, (
|
||||
f"the dropped-step guard is not the last step of '{job}' — it is at index "
|
||||
f"{steps.index(guard)} of {len(steps)}, so any marked step after it would be unguarded and "
|
||||
"the guard would read a marker that has not been written yet."
|
||||
)
|
||||
assert "if" not in guard, (
|
||||
f"the '{job}' guard carries `if: {guard.get('if')!r}`. It must have none: the default "
|
||||
"`success()` is what keeps it silent on ordinary red builds, and `always()` would make it "
|
||||
"announce a false 'these steps never executed' on every failing run. See the comment above "
|
||||
"the step for why this is a deliberate departure from the #751 guard."
|
||||
)
|
||||
assert guard.get("continue-on-error") is not True, (
|
||||
f"the '{job}' guard is continue-on-error, so it detects the dropped step and lets the job go "
|
||||
"green regardless — which is the defect it exists to remove."
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("job", MARKED_JOBS)
|
||||
def test_the_guards_OWN_body_cannot_be_dropped_by_the_mechanism_it_guards_against(job):
|
||||
"""A guard the guarded mechanism can silently delete is worse than no guard.
|
||||
|
||||
Its absence is silent too: the job simply goes green with nothing checked, which is
|
||||
indistinguishable from a clean run. #751 states the rule; here it is stronger than there,
|
||||
because the body is a single command with no delimiter possible rather than 20 lines of prose
|
||||
that must be kept clean by hand.
|
||||
|
||||
The gate VALUES arrive through `env:`, which the runner interpolates per value — a bad payload
|
||||
there fails that value, not the body. Both are additionally held to naming a real context by
|
||||
test_every_workflow_expression_names_a_REAL_context_or_function in test_pr_changed_files.py.
|
||||
"""
|
||||
guard = _guard(job)
|
||||
assert not _OPENER.search(guard["run"]), (
|
||||
f"the '{job}' guard's own run body contains an expression delimiter, so the runner can drop "
|
||||
"the guard the same way it drops the steps the guard is watching — and that absence is "
|
||||
"silent as well."
|
||||
)
|
||||
assert guard["run"].strip().startswith("scripts/ci-step-ran.sh assert"), (
|
||||
f"the '{job}' guard is no longer a bare invocation: {guard['run']!r}. Keeping it to one "
|
||||
"command is what makes a delimiter impossible rather than merely absent."
|
||||
)
|
||||
# THE VALUES, not just the names — found by cold review. Asserting the keys alone accepts
|
||||
# `ETV_DOCS_ONLY: ${{ steps.detect.outputs.doc_only }}` (note the typo), which names a real
|
||||
# context so the repo-wide expression check passes it too. The guard would then read an EMPTY
|
||||
# value on a docs-only run, demand the gated steps that were correctly skipped, and redden a
|
||||
# REQUIRED context on every docs-only PR.
|
||||
# THE TWO MAPPINGS MUST BE PRESENT AND CORRECT — but this deliberately does NOT demand that the
|
||||
# `env:` block contain ONLY them. An earlier version compared the whole dict, which false-redded
|
||||
# on adding an unrelated variable (an `LC_ALL`, say) and on the equally-valid `${{x}}` spacing;
|
||||
# a red here blocks every merge through the combined status, so brittleness is a real cost and
|
||||
# not a free strictness win. Whitespace inside the delimiters is normalised for the same reason.
|
||||
env = {k: re.sub(r"\s+", "", str(v)) for k, v in (guard.get("env") or {}).items()}
|
||||
for name, want in (
|
||||
("ETV_DOCS_ONLY", "${{steps.detect.outputs.docs_only}}"),
|
||||
("ETV_REVALIDATE_SKIP", "${{steps.revalidate.outputs.skip}}"),
|
||||
):
|
||||
assert env.get(name) == want, (
|
||||
f"the '{job}' guard's env: has {name}={guard.get('env', {}).get(name)!r}, expected the "
|
||||
f"output the gated steps' own `if:` reads ({want}). A typo here is SILENT rather than "
|
||||
"loud: it still names a real context, so the repo-wide expression check passes it, the "
|
||||
"value arrives empty, and the guard then demands steps that were legitimately skipped — "
|
||||
"reddening a REQUIRED context on every docs-only run."
|
||||
)
|
||||
|
||||
|
||||
# ------------------------------------------------------------------------------------------------
|
||||
# BEHAVIOURAL — the guard's real command line, against markers written by the steps' real lines
|
||||
# ------------------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _mark_line(step) -> str:
|
||||
"""The step's OWN marker line, verbatim from the workflow.
|
||||
|
||||
Extracted rather than rebuilt in Python ON PURPOSE. A test that composed the command itself
|
||||
would keep passing after the workflow and the script drifted apart on the path, the quoting or
|
||||
the sub-command — and that divergence is exactly the failure that makes the guard fail on every
|
||||
run and then get deleted as broken. Running the real line proves the two agree by construction.
|
||||
"""
|
||||
line = next(ln for ln in step["run"].splitlines() if _MARK.search(ln))
|
||||
return line.strip()
|
||||
|
||||
|
||||
# THE GATE VALUES DEFAULT TO `"false"`, WHICH IS WHAT THE RUNNER ACTUALLY SENDS — and getting this
|
||||
# wrong made the whole suite blind. Found by cold review, which demonstrated it: every behavioural
|
||||
# test used to leave these UNSET, so the guard was never once driven at its production values. Change
|
||||
# the gate in `ci-step-ran.sh` from `= "true"` to `-n` — a one-token regression — and all 30 tests
|
||||
# stayed GREEN while the guard, run with the real environment, reported
|
||||
# `Skip gate fired (docs_only='false') … All 2 expected step(s) executed` and exited 0. `Build`,
|
||||
# `Test` and both migration replays would have been unguarded on every ordinary run, with the guard
|
||||
# announcing that it had proved everything.
|
||||
#
|
||||
# THE COMPLETE VALUE SET, and where each comes from — worth spelling out, because the obvious reading
|
||||
# of the evidence is wrong. Both producers document `true|false` and write exactly that
|
||||
# (`scripts/ci-detect-docs-only.sh` -> `docs_only=`, `scripts/ci-detect-already-validated.sh` ->
|
||||
# `skip=`), so an ordinary run sends `false` and a skipping run sends `true`.
|
||||
#
|
||||
# The live log of the probe this change cites (run 1910, job 8064) shows `ETV_DOCS_ONLY: false` and
|
||||
# `ETV_REVALIDATE_SKIP:` EMPTY — but do NOT read that as revalidate's normal output. `revalidate` was
|
||||
# the step the probe deliberately dropped, so it wrote no output at all. The empty string is
|
||||
# therefore not an odd third state: it is the SIGNATURE OF THE VERY FAILURE THIS GUARD EXISTS TO
|
||||
# CATCH, which is exactly why the gate must treat anything that is not `true` as "widen what is
|
||||
# required". `None` (unset) is the same case reached a different way.
|
||||
#
|
||||
# A test double is an assertion about what the real system sends, and the earlier version of this one
|
||||
# was wrong about the only field the guard branches on.
|
||||
GATE_VALUES_IN_THE_WILD = ("false", "", None)
|
||||
|
||||
|
||||
def _env(tmp_path, **extra):
|
||||
env = {
|
||||
"PATH": os.environ["PATH"],
|
||||
"GITHUB_WORKSPACE": str(REPO_ROOT),
|
||||
"RUNNER_TEMP": str(tmp_path),
|
||||
"GITHUB_JOB": "test",
|
||||
"GITHUB_RUN_ID": "424242",
|
||||
"GITHUB_RUN_ATTEMPT": "7",
|
||||
"ETV_DOCS_ONLY": "false",
|
||||
"ETV_REVALIDATE_SKIP": "false",
|
||||
}
|
||||
env.update(extra)
|
||||
return {k: v for k, v in env.items() if v is not None}
|
||||
|
||||
|
||||
def _run(script: str, env):
|
||||
return subprocess.run(["bash", "-c", script], cwd=REPO_ROOT, env=env, capture_output=True, text=True)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("gate", GATE_VALUES_IN_THE_WILD, ids=["gate-false", "gate-empty", "gate-unset"])
|
||||
@pytest.mark.parametrize("job", MARKED_JOBS)
|
||||
def test_the_guard_PASSES_when_every_step_marked_itself(job, gate, tmp_path):
|
||||
"""The positive control. Without it, a guard that always failed would satisfy every case below.
|
||||
|
||||
`GITHUB_JOB` is set to the job under test, so this also covers the marker file being keyed per
|
||||
job: if it were not, the two jobs would share a file and one job's markers would answer for the
|
||||
other's dropped steps.
|
||||
"""
|
||||
marks = [_mark_line(s) for s, _ in _marked(job)]
|
||||
guard = _guard(job)["run"]
|
||||
# Parametrised over every NOT-SKIPPING spelling the runner emits — `false` on an ordinary run,
|
||||
# empty when the producing step was dropped, absent if the output is never set. All three must
|
||||
# require the gated steps; a gate that treats any of them as a skip is fail-open on that path.
|
||||
env = _env(tmp_path, GITHUB_JOB=job, ETV_DOCS_ONLY=gate, ETV_REVALIDATE_SKIP=gate)
|
||||
r = _run("\n".join(["set -e", *marks, guard]), env)
|
||||
assert r.returncode == 0, (
|
||||
f"the '{job}' guard rejected a run in which every step marked itself — the steps and the "
|
||||
f"guard disagree, so this would fail on every run.\n{r.stdout}\n{r.stderr}"
|
||||
)
|
||||
assert "All" in r.stdout and "executed" in r.stdout, r.stdout
|
||||
# The other half of the identity contract: with GITHUB_RUN_ATTEMPT set (`_env` sends 7) the line
|
||||
# must report the REAL value and say so. A mis-derivation (`${marker#*-}` rather than `##`) or an
|
||||
# inverted provenance test would otherwise ship silently, and the operator reading this line to
|
||||
# settle the promotion question would read it wrong.
|
||||
assert f"Marker identity: job={job} run=424242 attempt=7 (from the runner)" in r.stdout, (
|
||||
f"the guard misreported its marker identity: {r.stdout!r}"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("job", MARKED_JOBS)
|
||||
def test_dropping_ANY_single_step_FAILS_the_guard(job, tmp_path):
|
||||
"""Every marked step, one at a time — not a sample.
|
||||
|
||||
An arbitrary sample gives false negatives here: the interesting drop is `Test` or the migration
|
||||
replay, and a test that only omitted the first step would prove the guard catches the one case
|
||||
that was never fail-open anyway. Dropping each key in turn is the only version that establishes
|
||||
the property the issue asks for.
|
||||
"""
|
||||
marked = _marked(job)
|
||||
guard = _guard(job)["run"]
|
||||
for dropped_step, dropped_key in marked:
|
||||
d = tmp_path / dropped_key
|
||||
d.mkdir()
|
||||
marks = [_mark_line(s) for s, k in marked if k != dropped_key]
|
||||
r = _run("\n".join(["set -e", *marks, guard]), _env(d, GITHUB_JOB=job))
|
||||
assert r.returncode != 0, (
|
||||
f"job '{job}': the guard went GREEN with {dropped_step.get('name')!r} "
|
||||
f"(key {dropped_key!r}) never having executed. That is a REQUIRED context reporting "
|
||||
f"success having skipped that work — the exact fail-open of ersatztv#756.\n{r.stdout}"
|
||||
)
|
||||
assert dropped_key in (r.stdout + r.stderr), (
|
||||
f"the guard failed but did not name the missing step {dropped_key!r}: {r.stdout}"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("job", MARKED_JOBS)
|
||||
@pytest.mark.parametrize("gate", ["ETV_DOCS_ONLY", "ETV_REVALIDATE_SKIP"])
|
||||
def test_a_fired_skip_gate_does_not_require_the_gated_steps(gate, job, tmp_path):
|
||||
"""The docs-only and already-validated paths must still report green in seconds.
|
||||
|
||||
They are the reason these jobs are never `if:`-skipped at the JOB level (a skipped required
|
||||
context is a state this repo deliberately does not rely on — ersatztv#416/#418), so a guard that
|
||||
reddened them would make every docs-only PR unmergeable. Which is #751's user-visible symptom
|
||||
arriving from the opposite direction, and worth a test rather than a comment.
|
||||
"""
|
||||
marks = [_mark_line(s) for s, k in _marked(job) if k in _guard_buckets(job)[0]]
|
||||
guard = _guard(job)["run"]
|
||||
r = _run("\n".join(["set -e", *marks, guard]), _env(tmp_path, GITHUB_JOB=job, **{gate: "true"}))
|
||||
assert r.returncode == 0, (
|
||||
f"with {gate}=true the guard still demanded the gated steps, so every docs-only / "
|
||||
f"already-validated run of a REQUIRED job would be red.\n{r.stdout}\n{r.stderr}"
|
||||
)
|
||||
assert "Skip gate fired" in r.stdout, r.stdout
|
||||
|
||||
|
||||
@pytest.mark.parametrize("job", MARKED_JOBS)
|
||||
def test_a_fired_skip_gate_STILL_requires_the_ALWAYS_steps(job, tmp_path):
|
||||
"""The negative control for the test above — otherwise `ETV_DOCS_ONLY=true` would be a blanket
|
||||
off-switch and the previous test would be passing for the wrong reason.
|
||||
|
||||
This is the case that matters most on a docs-only run: the detect steps are the only things that
|
||||
execute, so if their drop were unguarded the skip path would be entirely unchecked.
|
||||
"""
|
||||
guard = _guard(job)["run"]
|
||||
r = _run("\n".join(["set -e", guard]), _env(tmp_path, GITHUB_JOB=job, ETV_DOCS_ONLY="true"))
|
||||
assert r.returncode != 0, (
|
||||
"with ETV_DOCS_ONLY=true and NO steps marked at all, the guard passed — the skip gate is "
|
||||
"acting as a blanket off-switch rather than as a narrowing of what is expected."
|
||||
)
|
||||
assert "detect" in (r.stdout + r.stderr), r.stdout
|
||||
|
||||
|
||||
@pytest.mark.parametrize("job", MARKED_JOBS)
|
||||
def test_an_EMPTY_gate_value_requires_the_gated_steps(job, tmp_path):
|
||||
"""A dropped `detect` step leaves its outputs EMPTY, not 'false'.
|
||||
|
||||
Reading empty as "skipped" would mean the one drop that disables the detect step also disables
|
||||
the guard for everything downstream — the guard switching itself off in response to the very
|
||||
failure it exists to catch. The direction has to be: anything that is not exactly `true` widens
|
||||
what is required.
|
||||
"""
|
||||
guard = _guard(job)["run"]
|
||||
r = _run("\n".join(["set -e", guard]), _env(tmp_path, GITHUB_JOB=job, ETV_DOCS_ONLY="", ETV_REVALIDATE_SKIP=""))
|
||||
assert r.returncode != 0
|
||||
assert _guard_buckets(job)[1][-1] in (r.stdout + r.stderr), (
|
||||
f"empty gate values were read as a skip, so the gated steps went unchecked: {r.stdout}"
|
||||
)
|
||||
|
||||
|
||||
def test_a_STALE_marker_from_another_run_cannot_satisfy_the_guard(tmp_path):
|
||||
"""A marker from another run, attempt or job must never answer for this one.
|
||||
|
||||
Do NOT restate this as "RUNNER_TEMP is /tmp, not a private per-job directory". That is a #751
|
||||
measurement taken on a job with no `container:`, and it does not transfer: these two jobs run
|
||||
inside the CI toolchain image, so their `/tmp` is the container's own. The fresh container is
|
||||
what actually rules out staleness here; the keying is defence in depth against a lane change
|
||||
nobody would think to re-check this against, and that is why it is still worth testing.
|
||||
"""
|
||||
marks = [_mark_line(s) for s, _ in _marked("test")]
|
||||
guard = _guard("test")["run"]
|
||||
# Run 1 marks everything.
|
||||
first = _env(tmp_path, GITHUB_JOB="test", GITHUB_RUN_ID="111", GITHUB_RUN_ATTEMPT="1")
|
||||
assert _run("\n".join(["set -e", *marks]), first).returncode == 0
|
||||
# Run 2 shares RUNNER_TEMP but marks nothing. It must NOT inherit run 1's markers.
|
||||
second = _env(tmp_path, GITHUB_JOB="test", GITHUB_RUN_ID="222", GITHUB_RUN_ATTEMPT="1")
|
||||
r = _run(guard, second)
|
||||
assert r.returncode != 0, (
|
||||
"a marker file left by a DIFFERENT run satisfied the guard, so a run whose steps were all "
|
||||
f"dropped would pass silently.\n{r.stdout}"
|
||||
)
|
||||
# ...and a RETRY of run 1 must not inherit run 1's either.
|
||||
retry = _env(tmp_path, GITHUB_JOB="test", GITHUB_RUN_ID="111", GITHUB_RUN_ATTEMPT="2")
|
||||
assert _run(guard, retry).returncode != 0, (
|
||||
"a re-run inherited the first attempt's markers, so a step dropped only on the retry passes"
|
||||
)
|
||||
# ...nor may the OTHER job in the same run inherit them.
|
||||
sibling = _env(tmp_path, GITHUB_JOB="migrations", GITHUB_RUN_ID="111", GITHUB_RUN_ATTEMPT="1")
|
||||
assert _run(_guard("migrations")["run"], sibling).returncode != 0, (
|
||||
"the two required jobs share one marker file, so one job's markers answer for the other's dropped steps"
|
||||
)
|
||||
|
||||
|
||||
def test_assert_with_no_expected_keys_REFUSES_instead_of_passing(tmp_path):
|
||||
"""The script's own anti-vacuity check, exercised rather than trusted.
|
||||
|
||||
`assert` with an empty expectation list would print "All 0 expected step(s) executed" and exit 0
|
||||
— a guard that proves nothing while reporting that it proved everything. That is how a guard
|
||||
ends up shipped and dead, which this repo has now done twice (#751's fence, #751's own guard).
|
||||
"""
|
||||
r = _run(f"{SCRIPT} assert", _env(tmp_path))
|
||||
assert r.returncode == 2, f"expected a usage refusal, got {r.returncode}: {r.stdout} {r.stderr}"
|
||||
assert "no expected keys" in (r.stdout + r.stderr)
|
||||
|
||||
|
||||
def test_mark_APPENDS_so_one_step_does_not_erase_its_predecessors(tmp_path):
|
||||
"""`>` instead of `>>` in the script would leave only the last step's key.
|
||||
|
||||
The guard would then redden on every run — fail-closed, but it would look like the guard is
|
||||
broken rather than like a real drop, and that is the state in which a correct guard gets removed.
|
||||
"""
|
||||
env = _env(tmp_path)
|
||||
assert _run(f"{SCRIPT} mark alpha && {SCRIPT} mark beta", env).returncode == 0
|
||||
r = _run(f"{SCRIPT} assert --always alpha beta", env)
|
||||
assert r.returncode == 0, f"the second mark erased the first: {r.stdout} {r.stderr}"
|
||||
|
||||
|
||||
def test_a_key_is_matched_WHOLE_not_as_a_substring(tmp_path):
|
||||
"""`build` must not be satisfied by `web-build`, and `test` not by `web-test`.
|
||||
|
||||
Both pairs are live key names in the `test` job, so a substring match would mean dropping the
|
||||
real `Build` or `Test` step — the two most consequential steps in the whole workflow — is
|
||||
invisible because an SPA step of a similar name ran.
|
||||
"""
|
||||
env = _env(tmp_path)
|
||||
assert _run(f"{SCRIPT} mark web-build && {SCRIPT} mark web-test", env).returncode == 0
|
||||
r = _run(f"{SCRIPT} assert --always build", env)
|
||||
assert r.returncode != 0, (
|
||||
"the key 'build' was satisfied by a marker for 'web-build' — a dropped `dotnet build` would pass unnoticed"
|
||||
)
|
||||
|
||||
|
||||
def test_a_degraded_run_IDENTITY_refuses_rather_than_sharing_a_marker_path(tmp_path):
|
||||
"""`GITHUB_RUN_ID` absent must REFUSE, not fall back to a name every run shares.
|
||||
|
||||
The first version of `marker_path` defaulted to `nojob`/`norunid`/`1`. Those are reusable, so a
|
||||
leftover marker from any earlier run on the host would satisfy the guard on a run whose step was
|
||||
dropped — a silent PASS, which is the precise failure the run-keying exists to remove,
|
||||
reintroduced by the code implementing it. Found by cold review.
|
||||
|
||||
Asserted on BOTH sub-commands: a refusal that only `assert` honoured would let `mark` write to a
|
||||
shared path and leave the two disagreeing about where the file is.
|
||||
"""
|
||||
env = _env(tmp_path)
|
||||
for var in ("GITHUB_RUN_ID", "GITHUB_JOB", "GITHUB_RUN_ATTEMPT"):
|
||||
degraded = {k: v for k, v in env.items() if k != var}
|
||||
for argv in (f"{SCRIPT} mark alpha", f"{SCRIPT} assert --always alpha"):
|
||||
r = _run(argv, degraded)
|
||||
assert r.returncode != 0, (
|
||||
f"with {var} unset, `{argv.split()[-2]}` continued and used a fallback path that "
|
||||
f"other runs also use — a stale marker there passes the guard on a dropped run.\n"
|
||||
f"{r.stdout}{r.stderr}"
|
||||
)
|
||||
assert "cannot identify this run" in (r.stdout + r.stderr), (
|
||||
f"refused, but without naming the cause: {r.stdout!r} {r.stderr!r}"
|
||||
)
|
||||
assert not list(tmp_path.iterdir()), (
|
||||
"a degraded-identity `mark` still created a marker file somewhere under RUNNER_TEMP"
|
||||
)
|
||||
|
||||
|
||||
def test_the_marker_identity_is_REPORTED_on_stdout_every_run(tmp_path):
|
||||
"""The line that settled `GITHUB_RUN_ATTEMPT`, kept as standing evidence.
|
||||
|
||||
Worth recording HOW that was settled, because the first two attempts were both bad. Grepping a
|
||||
job log for the variable NAME proves nothing (logs do not dump the environment). Inferring it
|
||||
from the ABSENCE of a "not set" warning proves nothing either, because that warning goes to
|
||||
stderr and whether step stderr reaches a job log here was itself never established — the control
|
||||
offered for that was an `::error::` this script writes to STDOUT. So the script was made to
|
||||
REPORT its resolved identity on stdout, where capture is not in question, and the answer was read
|
||||
off ersatztv#756's own PR run: `Marker identity: job=test run=1916 attempt=1 (from the runner)`,
|
||||
and the same for `migrations`. That is what promoted the variable from warn-and-default to
|
||||
required.
|
||||
|
||||
Asserted because cold review demonstrated three mutations of this reporting — deleting the echo,
|
||||
mis-deriving the attempt, inverting the provenance — all surviving a 50-green suite. It is a
|
||||
documented contract (the record's `mechanics:`), and a future reader is told to trust it.
|
||||
"""
|
||||
marks = [_mark_line(s) for s, _ in _marked("test")]
|
||||
r = _run(
|
||||
"\n".join(["set -e", *marks, _guard("test")["run"]]),
|
||||
_env(tmp_path, GITHUB_RUN_ID="1916", GITHUB_RUN_ATTEMPT="4"),
|
||||
)
|
||||
assert r.returncode == 0, r.stdout + r.stderr
|
||||
assert "Marker identity: job=test run=1916 attempt=4 (from the runner)" in r.stdout, (
|
||||
"the guard did not report the identity its marker path was actually keyed on, so a reader "
|
||||
f"cannot audit the keying from a run log: {r.stdout!r}"
|
||||
)
|
||||
|
||||
|
||||
def test_a_skip_gate_that_empties_the_expected_set_REFUSES(tmp_path):
|
||||
"""The anti-vacuity check has to run AFTER the gate, not only on argv. Cold review reproduced
|
||||
this exactly:
|
||||
|
||||
ETV_DOCS_ONLY=true … assert --always --gated foo
|
||||
-> "All 0 expected step(s) executed", exit 0
|
||||
|
||||
The argv check cannot see it, because the set is emptied by the gate rather than by the caller.
|
||||
Unreachable with today's argv, but it contradicted the comment directly above it — and "reports
|
||||
that it proved everything while proving nothing" is the failure this whole file exists to remove.
|
||||
"""
|
||||
r = _run(f"{SCRIPT} assert --always --gated foo", _env(tmp_path, ETV_DOCS_ONLY="true"))
|
||||
assert r.returncode != 0, f"the guard passed with an empty post-gate expectation set: {r.stdout!r}"
|
||||
assert "no expected keys" in (r.stdout + r.stderr).lower() or "NO expected keys" in r.stderr
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"revalidate", ["true", "false", "", None], ids=lambda v: f"reval-{v if v is not None else 'unset'}"
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"docs_only", ["true", "false", "", None], ids=lambda v: f"docs-{v if v is not None else 'unset'}"
|
||||
)
|
||||
def test_the_skip_gate_over_the_WHOLE_value_matrix(docs_only, revalidate, tmp_path):
|
||||
"""Every combination of the two gate values, not just the diagonal — cold review's last finding.
|
||||
|
||||
Round 3 fixed the suite's blindness to the production value `false`, but still only exercised
|
||||
matched pairs and single-`true` cases. `(true, true)` is REACHABLE — a docs-only PR merged to
|
||||
`main` whose tree was already validated sets both — and an exclusive-or regression would pass
|
||||
every other test here while demanding all the gated markers on a run that legitimately skipped
|
||||
those steps. That reddens BOTH required contexts, which is the false-red direction: it deadlocks
|
||||
every merge rather than letting one through.
|
||||
|
||||
The property asserted is the whole contract in one line: with only the `--always` keys marked,
|
||||
the guard passes exactly when the gate says the gated steps were skipped — `true` in EITHER
|
||||
variable, and nothing else. Sixteen cases, so no combination is a special case anyone has to
|
||||
remember.
|
||||
|
||||
On `unset`: the workflow's `env:` block always defines both, emitting EMPTY for an output the
|
||||
producing step never wrote, so unset is not reachable through the workflow. It is covered because
|
||||
the script is also runnable by hand, and because "not exactly true" is the property that must
|
||||
hold for every spelling rather than for an enumerated list.
|
||||
"""
|
||||
job = "test"
|
||||
always, gated = _guard_buckets(job)
|
||||
marks = [_mark_line(s) for s, k in _marked(job) if k in always]
|
||||
r = _run(
|
||||
"\n".join(["set -e", *marks, _guard(job)["run"]]),
|
||||
_env(tmp_path, ETV_DOCS_ONLY=docs_only, ETV_REVALIDATE_SKIP=revalidate),
|
||||
)
|
||||
should_skip = docs_only == "true" or revalidate == "true"
|
||||
assert (r.returncode == 0) is should_skip, (
|
||||
f"with docs_only={docs_only!r} and revalidate={revalidate!r} the guard "
|
||||
f"{'passed' if r.returncode == 0 else 'failed'}, expected it to "
|
||||
f"{'skip the gated keys' if should_skip else 'require them'}. The gate must treat a value as "
|
||||
"a skip if and only if it is exactly `true` in EITHER variable.\n" + r.stdout + r.stderr
|
||||
)
|
||||
@@ -0,0 +1,351 @@
|
||||
"""The CI-image pin guard must see a container job that carries NO pin (ersatztv#774).
|
||||
|
||||
WHAT THIS IS PROTECTING. `pr-checks.yml`'s `ci-image-pin` job states the invariant in its own error
|
||||
text — "Every container: job must pin ersatztv-ci:<7-char-sha>" — and then does not check it. What
|
||||
it checks is:
|
||||
|
||||
mapfile -t pins < <(grep -oE 'ersatztv-ci:[0-9a-f]+' .gitea/workflows/docker-build.yml \
|
||||
| cut -d: -f2 | sort -u)
|
||||
[ "${#pins[@]}" -eq 1 ]
|
||||
|
||||
`sort -u` collapses to DISTINCT VALUES, so the count is a property of the pins that are PRESENT. A
|
||||
job that carries a `container:` block with no `ersatztv-ci:` pin — or no `container:` block at all —
|
||||
contributes nothing to grep's output, so it cannot move that count. Delete the `container:` block
|
||||
from `test` and four pins remain: still one distinct value, still green, and a REQUIRED context now
|
||||
runs on the bare runner instead of the toolchain image. That is ersatztv#774's Family A exactly: a
|
||||
guard that cannot see the member that is MISSING, because its population is the set of matches
|
||||
rather than the set of jobs.
|
||||
|
||||
THE SPLIT WITH THE SHELL GUARD IS DELIBERATE, and is not two copies of one rule (which would be
|
||||
#773's Family C). Two different assertions over the same subject:
|
||||
|
||||
* `ci-image-pin` (shell, pr-checks.yml) owns the questions that need GIT HISTORY — does the pin
|
||||
resolve to a commit, is it exactly 7 chars, is it the last commit to touch `docker/ci`. A
|
||||
pytest cannot answer those without a full clone.
|
||||
* this file owns the question that needs the PARSED YAML — is the set of jobs declaring a
|
||||
`container:` exactly the set of jobs pinning the image. A shell grep structurally cannot answer
|
||||
that, which is why it was never asked.
|
||||
|
||||
Neither restates the other, and each says so above the code.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import copy
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
import yaml
|
||||
|
||||
from scripts.tests.tracked_files import tracked_paths
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[2]
|
||||
WORKFLOWS_DIR = REPO_ROOT / ".gitea" / "workflows"
|
||||
WORKFLOW = WORKFLOWS_DIR / "docker-build.yml"
|
||||
# Resolved against the GIT INDEX rather than `Path.glob` (ersatztv#806), and `*.yaml` alongside
|
||||
# `*.yml`: Gitea accepts both spellings, so a `.yaml` workflow was structurally invisible to the
|
||||
# scope check below while reading as covered.
|
||||
WORKFLOWS = (".gitea/workflows", ("*.yml", "*.yaml"))
|
||||
|
||||
|
||||
def workflow_files() -> list[Path]:
|
||||
"""THE WORKFLOW POPULATION, from the git index. Named rather than inline so the shared proof in
|
||||
`test_guard_populations_derive_from_git.py` can assert it never admits an untracked file."""
|
||||
return tracked_paths(*WORKFLOWS)
|
||||
|
||||
|
||||
# The image repository, without the tag. Matched as a whole path rather than by the bare
|
||||
# `ersatztv-ci` token so a job pointing at a LOOK-ALIKE registry (a personal fork, a typo'd host)
|
||||
# is a fault rather than a silent pass — the shell guard's `grep -oE 'ersatztv-ci:[0-9a-f]+'` reads
|
||||
# the tail of any string and would accept `evil.example/ersatztv-ci:32747a0`.
|
||||
IMAGE_REPO = "192.168.1.95:3000/timothy/ersatztv-ci"
|
||||
_PIN = re.compile(rf"^{re.escape(IMAGE_REPO)}:([0-9a-f]+)$")
|
||||
|
||||
_DOC = yaml.safe_load(WORKFLOW.read_text())
|
||||
|
||||
# THE HAND-REVIEWED REGISTRY of jobs that must run inside the CI toolchain image, cross-checked in
|
||||
# BOTH directions against what the workflow actually declares. This is deliberately a literal, and
|
||||
# the reason is the one case set equality between two DERIVED sets cannot cover.
|
||||
#
|
||||
# The first version of this file compared `container_jobs(doc)` against `pinned_jobs(doc)` and
|
||||
# nothing else. That is blind to the mutation that matters most: delete a `container:` block and the
|
||||
# job leaves BOTH sets together, so the comparison stays balanced and reports green — which is the
|
||||
# very fail-open this file was written to close, reproduced one level up. A guard whose expected set
|
||||
# shrinks in step with the thing it is guarding is not a guard.
|
||||
#
|
||||
# So the population needs one anchor that does NOT move when the workflow does, and a reviewed list
|
||||
# is the only available one: nothing else in the repo records which jobs are supposed to need the
|
||||
# toolchain. This is the same shape as `web/src/api/pageSizeCallSites.guard.test.ts` — discovery on
|
||||
# one side, a reviewed registry on the other, compared both ways — and it is the SCOPE half of
|
||||
# `testing.guard-derives-population-from-source`, not an exception to it. Editing this set is a
|
||||
# reviewable act; a job silently losing its container block is not.
|
||||
TOOLCHAIN_JOBS = frozenset({"test", "migrations", "functional-e2e", "api-docs", "format"})
|
||||
|
||||
# `scan` and `build` deliberately run on the bare runner: `scan` is `runs-on: small` and needs only
|
||||
# python, and `build` drives docker/buildx on the host. Listed here so their ABSENCE above reads as
|
||||
# a decision rather than an oversight.
|
||||
BARE_RUNNER_JOBS = frozenset({"scan", "build"})
|
||||
|
||||
|
||||
def _jobs(doc) -> dict:
|
||||
return doc["jobs"]
|
||||
|
||||
|
||||
def container_jobs(doc) -> set[str]:
|
||||
"""Every job declaring a `container:`. THE AUTHORITATIVE POPULATION.
|
||||
|
||||
Derived from the parsed workflow, which is the only thing that knows the whole of it. A literal
|
||||
list here would reintroduce the defect one file over — correct on the day it was written and
|
||||
unable to report the day a sixth job appeared.
|
||||
"""
|
||||
return {name for name, job in _jobs(doc).items() if isinstance(job, dict) and "container" in job}
|
||||
|
||||
|
||||
def pinned_jobs(doc) -> dict[str, str]:
|
||||
"""job -> pinned tag, for every job whose container image is the CI toolchain image."""
|
||||
out = {}
|
||||
for name, job in _jobs(doc).items():
|
||||
if not isinstance(job, dict):
|
||||
continue
|
||||
image = str((job.get("container") or {}).get("image", ""))
|
||||
m = _PIN.match(image)
|
||||
if m:
|
||||
out[name] = m.group(1)
|
||||
return out
|
||||
|
||||
|
||||
def pin_population_faults(doc) -> list[str]:
|
||||
"""Set equality in BOTH directions, plus tag agreement. Accumulated, never fail-fast.
|
||||
|
||||
Both directions are reported separately because they are different defects. UNPINNED (a
|
||||
container job the guard cannot see) is the fail-open this file exists for. PINNED-BUT-NOT-A-
|
||||
CONTAINER-JOB cannot arise from `pinned_jobs` as written, but is computed anyway so that a
|
||||
future change to either helper cannot quietly make the comparison one-sided.
|
||||
"""
|
||||
declared = container_jobs(doc)
|
||||
pinned = pinned_jobs(doc)
|
||||
|
||||
faults = []
|
||||
# AGAINST THE REGISTRY FIRST. This is the direction the two derived sets cannot cover: a job
|
||||
# that loses its `container:` block leaves `declared` and `pinned` together, so their equality
|
||||
# survives untouched while the job quietly moves to the bare runner.
|
||||
for name in sorted(TOOLCHAIN_JOBS - set(pinned)):
|
||||
faults.append(
|
||||
f"job '{name}' is registered as needing the CI toolchain image but does not pin it "
|
||||
"(its container: block is missing or points elsewhere) — it is running on the bare "
|
||||
"runner"
|
||||
)
|
||||
for name in sorted(set(pinned) - TOOLCHAIN_JOBS):
|
||||
faults.append(f"job '{name}' pins the toolchain image but is not in TOOLCHAIN_JOBS")
|
||||
|
||||
for name in sorted(declared - set(pinned)):
|
||||
image = str((_jobs(doc)[name].get("container") or {}).get("image", ""))
|
||||
faults.append(
|
||||
f"job '{name}' declares a container: but its image is {image!r}, not "
|
||||
f"{IMAGE_REPO}:<sha>. ci-image-pin's grep cannot see this job at all, so the pin it "
|
||||
"reports as current says nothing about what this job actually runs in."
|
||||
)
|
||||
for name in sorted(set(pinned) - declared):
|
||||
faults.append(f"job '{name}' pins the image without declaring a container: block")
|
||||
|
||||
tags = set(pinned.values())
|
||||
if len(tags) > 1:
|
||||
faults.append(
|
||||
f"jobs pin DIFFERENT tags: {sorted((n, t) for n, t in pinned.items())}. All container "
|
||||
"jobs must run the same toolchain image."
|
||||
)
|
||||
return faults
|
||||
|
||||
|
||||
# ------------------------------------------------------------------------------------------------
|
||||
# THE LIVE ASSERTION
|
||||
# ------------------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_every_container_job_pins_the_CI_toolchain_image():
|
||||
faults = pin_population_faults(_DOC)
|
||||
assert not faults, (
|
||||
"docker-build.yml has a container: job the CI-image pin guard cannot see:\n "
|
||||
+ "\n ".join(faults)
|
||||
+ "\n\n`ci-image-pin` counts DISTINCT pin strings, so a job with no pin contributes nothing "
|
||||
"to that count and passes silently while running on the bare runner. See ersatztv#774."
|
||||
)
|
||||
|
||||
|
||||
def test_the_registry_and_the_workflow_agree_on_which_jobs_use_the_toolchain():
|
||||
"""BOTH directions against the reviewed registry — the anchor that does not move.
|
||||
|
||||
Left-to-right catches a job silently LOSING its container block (the mutation set equality
|
||||
between two derived sets cannot see, because both sides shrink together). Right-to-left catches
|
||||
a NEW container job nobody registered. Neither direction is optional and the messages differ,
|
||||
because the two are opposite mistakes.
|
||||
"""
|
||||
declared = container_jobs(_DOC)
|
||||
assert TOOLCHAIN_JOBS - declared == frozenset(), (
|
||||
f"these jobs are registered as needing the CI toolchain image but no longer declare a "
|
||||
f"container: block — {sorted(TOOLCHAIN_JOBS - declared)}. They are now running on the bare "
|
||||
"runner. If that is deliberate, move them to BARE_RUNNER_JOBS in this file and say why in "
|
||||
"the PR."
|
||||
)
|
||||
assert declared - TOOLCHAIN_JOBS == frozenset(), (
|
||||
f"these jobs declare a container: but are not in TOOLCHAIN_JOBS — "
|
||||
f"{sorted(declared - TOOLCHAIN_JOBS)}. Add them to the registry so the pin guard covers "
|
||||
"them, or they will run on an image nothing checks."
|
||||
)
|
||||
|
||||
|
||||
def test_the_registry_partitions_every_job_in_the_workflow():
|
||||
"""No job may be in neither list. ANTI-VACUITY with teeth, and the reason it is here:
|
||||
|
||||
a floor like `len(container_jobs) >= 3` would be satisfied by a broken parse that happened to
|
||||
find four jobs, and would say nothing about a NEW job appearing in a third state nobody
|
||||
considered. Partitioning the whole `jobs:` map means every job is a decision someone recorded.
|
||||
"""
|
||||
all_jobs = set(_jobs(_DOC))
|
||||
unclassified = all_jobs - TOOLCHAIN_JOBS - BARE_RUNNER_JOBS
|
||||
assert not unclassified, (
|
||||
f"docker-build.yml has job(s) {sorted(unclassified)} that are in neither TOOLCHAIN_JOBS nor "
|
||||
"BARE_RUNNER_JOBS. Every job must be one or the other, so that 'runs on the bare runner' is "
|
||||
"always a recorded decision rather than an omission."
|
||||
)
|
||||
assert not (TOOLCHAIN_JOBS & BARE_RUNNER_JOBS), "a job cannot be in both lists"
|
||||
assert all_jobs == TOOLCHAIN_JOBS | BARE_RUNNER_JOBS, (
|
||||
f"the registry names jobs that do not exist: {sorted((TOOLCHAIN_JOBS | BARE_RUNNER_JOBS) - all_jobs)}"
|
||||
)
|
||||
|
||||
|
||||
# ------------------------------------------------------------------------------------------------
|
||||
# MUTATION PROOFS — disarm the invariant one way at a time, each must be DETECTED (ersatztv#775)
|
||||
# ------------------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _mutants():
|
||||
"""(id, mutated doc) for each single-job way the invariant can be broken.
|
||||
|
||||
Every container job in turn, not a sample: the interesting drop is whichever job someone
|
||||
actually edits, and proving detection on only the first would prove the case least likely to
|
||||
happen (ersatztv#773 §3 Family A, applied to this file's own tests).
|
||||
"""
|
||||
for job in sorted(container_jobs(_DOC)):
|
||||
dropped = copy.deepcopy(_DOC)
|
||||
del dropped["jobs"][job]["container"]
|
||||
yield f"{job}-container-removed", dropped
|
||||
|
||||
unpinned = copy.deepcopy(_DOC)
|
||||
unpinned["jobs"][job]["container"]["image"] = "mcr.microsoft.com/dotnet/sdk:10.0"
|
||||
yield f"{job}-image-swapped", unpinned
|
||||
|
||||
lookalike = copy.deepcopy(_DOC)
|
||||
lookalike["jobs"][job]["container"]["image"] = "evil.example/timothy/ersatztv-ci:32747a0"
|
||||
yield f"{job}-lookalike-registry", lookalike
|
||||
|
||||
skewed = copy.deepcopy(_DOC)
|
||||
skewed["jobs"][job]["container"]["image"] = f"{IMAGE_REPO}:deadbee"
|
||||
yield f"{job}-tag-skewed", skewed
|
||||
|
||||
|
||||
_MUTANTS = list(_mutants())
|
||||
|
||||
|
||||
@pytest.mark.parametrize("doc", [m for _, m in _MUTANTS], ids=[i for i, _ in _MUTANTS])
|
||||
def test_a_single_job_losing_its_pin_is_DETECTED(doc):
|
||||
"""The proof this guard can go red. Without it, `pin_population_faults` returning a constant
|
||||
empty list would satisfy the live assertion above and prove nothing — which is how #621 and
|
||||
#685 both shipped."""
|
||||
assert pin_population_faults(doc), (
|
||||
"the population check accepted a workflow in which a container job no longer runs the pinned toolchain image"
|
||||
)
|
||||
|
||||
|
||||
def test_the_mutation_set_is_not_empty():
|
||||
"""The positive control for the parametrisation itself.
|
||||
|
||||
If `container_jobs` ever returned an empty set, `_mutants()` would yield nothing, pytest would
|
||||
collect zero cases from the decorator above, and the file would report all-green having proved
|
||||
nothing. That is the vacuous-by-sampling shape this whole issue is about, and it is reachable
|
||||
here through a single broken helper.
|
||||
"""
|
||||
expected = 4 * len(TOOLCHAIN_JOBS)
|
||||
assert len(_MUTANTS) == expected, (
|
||||
f"expected 4 mutations per registered job ({expected}), got {len(_MUTANTS)}. A floor rather "
|
||||
"than an equality here would let a `container_jobs()` that degraded to 3 of 5 jobs pass "
|
||||
"while silently testing less — the message would still claim 4 per job."
|
||||
)
|
||||
|
||||
|
||||
def test_docker_build_is_the_ONLY_workflow_pinning_the_toolchain_image():
|
||||
"""This file reads ONE workflow, which is itself a scope mirror needing its own check.
|
||||
|
||||
`WORKFLOW` hardcodes `docker-build.yml`, and the implicit claim — that no other workflow uses
|
||||
the toolchain image — mirrors a machine-readable source (the tracked `.gitea/workflows/*.y*ml`)
|
||||
that nothing consulted. `renovate.yml` already declares a `container:` with a different image,
|
||||
so the shape is live. A future workflow adopting `ersatztv-ci:` would acquire no pin-population
|
||||
guard, no single-tag check and no partition, silently, while `pin_population_faults`'s own error
|
||||
text claims "All container jobs must run the same toolchain image".
|
||||
|
||||
Found by cold review, which correctly noted this file criticises `MARKED_JOBS` for exactly this
|
||||
and then shipped the same shape without even the dated comment `MARKED_JOBS` carries.
|
||||
|
||||
The population comes from the GIT INDEX (ersatztv#806). A `Path.glob` here answered a question
|
||||
about the machine rather than about the repo: an untracked scratch workflow left in
|
||||
`.gitea/workflows/` would be parsed and could redden this test on one checkout while CI, which
|
||||
never sees it, stayed green. The pattern set gained `*.yaml` in the same change — Gitea accepts
|
||||
both spellings, so a `.yaml` workflow adopting the toolchain image was invisible here while this
|
||||
test read as covering every workflow.
|
||||
|
||||
Checked by PARSING each workflow's `container.image`, not by grepping the file. A text search
|
||||
reports `ci-image.yml`, which names the image because it BUILDS and PUSHES it — a producer, not
|
||||
a consumer. Grepping would have made this test permanently red on a correct tree, which is the
|
||||
fastest route to a correct guard being deleted.
|
||||
"""
|
||||
others = []
|
||||
for p in workflow_files():
|
||||
if p.name == WORKFLOW.name:
|
||||
continue
|
||||
doc = yaml.safe_load(p.read_text()) or {}
|
||||
for name, job in (doc.get("jobs") or {}).items():
|
||||
if not isinstance(job, dict):
|
||||
continue
|
||||
image = str((job.get("container") or {}).get("image", ""))
|
||||
# Keyed on the IMAGE REPOSITORY, not on `_PIN`'s literal-tag match. A job written as
|
||||
# `image: <repo>:${{ matrix.tag }}` runs on the toolchain image but fails `_PIN`, so
|
||||
# keying on the pin would have let a templated tag slip the whole check — found by cold
|
||||
# review, which constructed exactly that. The tag being an expression is itself a fault
|
||||
# (nothing could then verify WHICH image ran), so this reports the job either way.
|
||||
if image.startswith(f"{IMAGE_REPO}:"):
|
||||
others.append(f"{p.name}:{name}")
|
||||
assert not others, (
|
||||
f"{sorted(others)} run container jobs on the CI toolchain image, but this file only checks "
|
||||
f"{WORKFLOW.name}, so they have no pin-population guard at all. Extend the check to cover "
|
||||
"them rather than leaving the coverage implied."
|
||||
)
|
||||
|
||||
|
||||
def test_the_shell_guards_grep_sees_the_same_tags_the_jobs_run():
|
||||
"""Ties the two halves together, so they cannot drift into disagreeing about the subject.
|
||||
|
||||
`ci-image-pin` reads the file with a grep for `ersatztv-ci:<hex>`. This compares what that grep
|
||||
sees against what the parsed jobs actually run.
|
||||
|
||||
DISTINCT VALUES rather than a count, deliberately. The counts legitimately differ: the file's
|
||||
header comment at docker-build.yml:32 documents the pin in prose, so the shell guard's grep
|
||||
reads SIX strings where the YAML has five pinned jobs. Asserting on the count would either fail
|
||||
today or have to hardcode "+1 for the comment", which breaks the moment a second comment
|
||||
mentions the pin.
|
||||
|
||||
What actually has to hold for the shell guard's verdict to be sound is that its `sort -u` set
|
||||
equals the set of tags the jobs really run. Comparing the distinct sets says exactly that — and
|
||||
as a free side effect it makes the header comment SELF-CHECKING: bump the five image lines and
|
||||
forget the comment, and the sets diverge here with a message naming both, instead of the shell
|
||||
guard reporting "pins MORE THAN ONE ersatztv-ci tag" and pointing at prose.
|
||||
"""
|
||||
text = WORKFLOW.read_text()
|
||||
grepped = {m for m in re.findall(r"ersatztv-ci:([0-9a-f]+)", text)}
|
||||
parsed = set(pinned_jobs(_DOC).values())
|
||||
assert grepped == parsed, (
|
||||
f"ci-image-pin's grep sees the distinct tags {sorted(grepped)} but the parsed container "
|
||||
f"jobs run {sorted(parsed)}. A tag mentioned in the file but not run by any job (a stale "
|
||||
"header comment) makes the shell guard's 'MORE THAN ONE pin' check fire on prose; a tag "
|
||||
"run but not greppable means the shell guard is not checking that job at all."
|
||||
)
|
||||
@@ -0,0 +1,682 @@
|
||||
r"""The `scan` job — the delimiter ban made fail-CLOSED on the release path (ersatztv#767).
|
||||
|
||||
WHAT THIS IS PROTECTING. #756 brought `build` into the delimiter ban, because a dropped
|
||||
`Smoke + IPTV E2E` publishes a release candidate that was never booted and reports the job green.
|
||||
But the ban was enforced ONLY by `test_the_delimiter_banned_jobs_have_NO_expression_delimiter_in_any_run_body`
|
||||
in `script-tests` — `on: pull_request`, not a required context. Nothing re-checked it on a `v*` tag
|
||||
push, which is exactly when the candidate is published.
|
||||
|
||||
WHY A JOB AND NOT A STEP IN `build`, and why this file is structural. The first cut of #767 put a
|
||||
bespoke stdlib scanner in `build` itself. Two independent reviews killed it on two counts, and both
|
||||
are worth keeping written down because both are easy to re-invent:
|
||||
|
||||
* A guard step inside `build` cannot protect `build`. If the runner drops it, the job carries on
|
||||
and publishes — fail-OPEN. The defence offered was "the guard's own body has no opener, so it
|
||||
cannot be dropped", but the only thing enforcing THAT was the same PR-only test being
|
||||
backstopped. Circular. As a `needs:` of `build`, a red here means `build` never runs at all.
|
||||
* The bespoke scanner hand-parsed YAML (to avoid provisioning PyYAML on `build`'s bare runner) and
|
||||
had ~10 false NEGATIVES within one review round — flow mappings, a quoted `"run":` key, aliases,
|
||||
multiline quoted scalars. It was strictly WEAKER than the check it backstopped, in the only
|
||||
direction that matters. The fix was to delete it and run the real PyYAML-based test, which needs
|
||||
no second definition of "what is a `run:` body" and so has no drift surface.
|
||||
|
||||
The detection ALGORITHM is not reimplemented here — it lives in `test_ci_dropped_step_guard.py`, and
|
||||
this job runs that file. What this file holds is the WIRING that makes the ban fail-closed (the job
|
||||
exists, `build` depends on it, nothing can skip it, its own steps cannot be silently dropped) plus
|
||||
ONE end-to-end probe that executes the scan step's real command against a poisoned copy of the repo
|
||||
and requires it to fail.
|
||||
|
||||
THAT PROBE IS THE LOAD-BEARING TEST, and it exists because shape assertions lost twice. Round 2
|
||||
replaced a substring check with checks ABOUT the command — bare-token argument, no `-k`, no `||` —
|
||||
and round 3 then disarmed the gate seven more ways that all left the suite green: `echo`ing the
|
||||
command instead of running it, flags moved past a `\` continuation, a trailing `exit 0`,
|
||||
`if false; then … fi`, `set +e`, and `PYTEST_ADDOPTS` in the step's `env:`. Two further disarms lived
|
||||
in the sibling module where no shape check could ever reach — the ban test's parametrize list swapped
|
||||
to `MARKED_JOBS`, and its opener regex neutered. Running the command settles every one of them,
|
||||
because it asks the only question that matters: with a delimiter in `build`'s `Smoke` body, does this
|
||||
command fail? The shape checks are kept as a faster, more specific signal, not as the guarantee.
|
||||
|
||||
ROUND 4 THEN FOUND THE TWIN OF THE ROUND-3 FIX, which is the reusable lesson here: fixing the STEP
|
||||
`env:` tier did not generalise, and the same `PYTEST_ADDOPTS` disarm placed one tier up — on the JOB
|
||||
— defeated the probe, because the probe reconstructed only the step's env. It now layers all three
|
||||
tiers (workflow, job, step). The same round found that `needs:` is not by itself a gate: an
|
||||
`always()` in `build`'s `if:` downgrades the edge to mere ordering, and the delimiter ban does not
|
||||
cover `if:` expressions, so nothing else would have objected.
|
||||
|
||||
ENUMERATING THAT LIST THEN FOUND A FOURTH TIER the probe cannot ever reach: a step writing to
|
||||
`$GITHUB_ENV` injects into LATER steps at runtime, so it is invisible to any static reconstruction of
|
||||
the workflow text. Measured — `PYTEST_ADDOPTS` supplied that way makes the ban command exit 0 on a
|
||||
poisoned tree while the probe reports healthy. That one is BANNED rather than modelled
|
||||
(`test_no_step_in_the_scan_job_writes_to_GITHUB_ENV`), because emulating the runner's semantics would
|
||||
be a second implementation of precisely the kind #767 already deleted once. The rule after any fix
|
||||
here: enumerate the tiers and the twin, and where a tier cannot be observed, forbid it.
|
||||
|
||||
(Two disarms review reported were checked and are NOT real: `--ignore=` and a `conftest.py`
|
||||
`collect_ignore` do not suppress a file pytest was given explicitly as an argument — measured, the
|
||||
ban test still ran and still failed. Recorded so they are not re-litigated. But note what refuting
|
||||
them did NOT establish: the WORKING attacks through that same configuration channel — a repo-root
|
||||
`pytest.ini` `addopts`, or `pytest_collection_modifyitems` — were simply never tried, and both
|
||||
disarm everything here. Refuting two variants of a channel is not clearing the channel.)
|
||||
|
||||
WHAT THIS DOES NOT CLAIM. That no step can ever fail to run for a reason other than the interpolation
|
||||
drop. This job's own steps carry #756 markers and a trailing assert, so the regress terminates where
|
||||
the sibling guards' does — to fail open you must drop the pytest step AND the assert step. And the
|
||||
probe runs the command, not the RUNNER: that a red `scan` actually skips `build` is a live
|
||||
measurement recorded on the issue, which no test here can establish.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
import yaml
|
||||
|
||||
# Imports the shared index derivation to build a HERMETIC fixture copy, not to derive a guard
|
||||
# population — see `_repo_copy`. Recorded as such in POPULATION_EXEMPT in
|
||||
# `test_guard_populations_derive_from_git.py`; the exemption lives there, not here, because a marker
|
||||
# a file grants itself is a kill switch any prose mention can trip.
|
||||
from scripts.tests import tracked_files
|
||||
|
||||
# Both spellings, matching what the converted guards consider the workflow set.
|
||||
WORKFLOWS = (".gitea/workflows", ("*.yml", "*.yaml"))
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[2]
|
||||
WORKFLOW = REPO_ROOT / ".gitea" / "workflows" / "docker-build.yml"
|
||||
SCRIPT = REPO_ROOT / "scripts" / "ci-step-ran.sh"
|
||||
|
||||
_DOC = yaml.safe_load(WORKFLOW.read_text())
|
||||
_OPENER = re.compile(r"\$\{\{")
|
||||
_MARK = re.compile(r'ci-step-ran\.sh"?\s+mark\s+(\S+)')
|
||||
|
||||
JOB = "scan"
|
||||
BAN_TEST_FILE = "scripts/tests/test_ci_dropped_step_guard.py"
|
||||
|
||||
|
||||
def _job():
|
||||
assert JOB in _DOC["jobs"], f"the `{JOB}` job is gone — the release path is unguarded again"
|
||||
return _DOC["jobs"][JOB]
|
||||
|
||||
|
||||
def _steps():
|
||||
return _job()["steps"]
|
||||
|
||||
|
||||
def _run_steps():
|
||||
return [s for s in _steps() if s.get("run")]
|
||||
|
||||
|
||||
def _guard():
|
||||
"""The trailing assert step, located by CONTENT — never by index, so that
|
||||
`test_the_guard_is_the_LAST_step` is not true by construction."""
|
||||
hits = [s for s in _run_steps() if "ci-step-ran.sh assert" in s["run"]]
|
||||
assert len(hits) == 1, f"expected exactly 1 assert step in `{JOB}`, found {len(hits)}"
|
||||
return hits[0]
|
||||
|
||||
|
||||
def _marked():
|
||||
out = []
|
||||
for s in _run_steps():
|
||||
m = _MARK.search(s["run"])
|
||||
if m:
|
||||
out.append((s, m.group(1)))
|
||||
return out
|
||||
|
||||
|
||||
# ------------------------------------------------------------------------------------------------
|
||||
# WIRING — the properties that make the ban fail-closed
|
||||
# ------------------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_build_DEPENDS_on_the_scan_job():
|
||||
"""This single edge is the whole fail-closed property.
|
||||
|
||||
Without it the scan is advisory: it could go red while `build` publishes anyway.
|
||||
"""
|
||||
needs = _DOC["jobs"]["build"]["needs"]
|
||||
needs = [needs] if isinstance(needs, str) else needs
|
||||
assert JOB in needs, f"`build` no longer needs `{JOB}` — a red scan would not stop a release"
|
||||
|
||||
|
||||
def test_build_does_not_NEUTRALISE_the_edge_with_an_always_style_condition():
|
||||
"""`needs:` alone is not the gate — `always()` downgrades it to mere ordering.
|
||||
|
||||
A dependent whose `if:` is `always()` (or `!cancelled()`, or `failure()`) runs even when the job
|
||||
it needs went red, so `build` would publish with a red `scan`. Reproduced by review:
|
||||
`if: ${{ always() && github.event_name != 'pull_request' }}` left the whole suite green while the
|
||||
edge no longer gated anything. The delimiter ban does not cover `if:` expressions, so nothing
|
||||
else in this repo would have objected either.
|
||||
"""
|
||||
cond = str(_DOC["jobs"]["build"].get("if", ""))
|
||||
for escape in ("always(", "!cancelled(", "cancelled(", "failure("):
|
||||
assert escape not in cond.replace(" ", ""), (
|
||||
f"`build`'s `if:` contains {escape!r}, which makes it run despite a red `{JOB}` — the "
|
||||
f"`needs:` edge stops being a gate: {cond!r}"
|
||||
)
|
||||
|
||||
|
||||
def test_the_scan_job_runs_the_out_of_pytest_positive_control():
|
||||
"""The proof that the ban would DETECT must be invoked, and must not be a test.
|
||||
|
||||
Everything else in this file lives inside pytest, and pytest's own configuration outranks it: a
|
||||
repo-root `pytest.ini` with `addopts = -k "not delimiter_banned"`, or a `conftest.py` with
|
||||
`pytest_collection_modifyitems`, deselects the ban test AND every test guarding it, leaving the
|
||||
suite green with a delimiter in `Smoke`. Measured, both forms. `scripts/ci-prove-ban-detects.sh`
|
||||
is the answer precisely because it is a shell step: it cannot be deselected.
|
||||
|
||||
This test can itself be deselected, of course — which is why it is not what makes the gate safe.
|
||||
It is here so that REMOVING the step from the workflow is loud in review; the step's own presence
|
||||
in the job, plus its `selfcheck` marker in the trailing assert, is what makes it loud at runtime.
|
||||
"""
|
||||
script = "scripts/ci-prove-ban-detects.sh"
|
||||
assert (REPO_ROOT / script).is_file(), f"{script} is missing"
|
||||
assert os.access(REPO_ROOT / script, os.X_OK), f"{script} is not executable"
|
||||
assert any(script in s["run"] for s in _run_steps()), (
|
||||
f"no step in `{JOB}` runs {script} — the ban is asserted to HOLD but never proved to DETECT"
|
||||
)
|
||||
assert "selfcheck" in {k for _, k in _marked()}, (
|
||||
"the positive-control step records no marker, so a dropped one would go unnoticed"
|
||||
)
|
||||
|
||||
|
||||
def test_no_step_in_the_scan_job_writes_to_GITHUB_ENV():
|
||||
"""The one env tier the probe structurally CANNOT reconstruct, so it is banned instead.
|
||||
|
||||
`_scan_body_and_env` layers the workflow, job and step `env:` blocks — all of which are visible
|
||||
in the workflow text. A step writing `FOO=bar >> $GITHUB_ENV` injects into LATER steps at
|
||||
runtime, which no static reconstruction can see. Measured: with `PYTEST_ADDOPTS` supplied that
|
||||
way, the ban command exits 0 on a tree whose `Smoke` body is poisoned — a full disarm that the
|
||||
execution probe reports as healthy, because the probe's subprocess never receives it.
|
||||
|
||||
Banning the mechanism is the fix rather than modelling it: nothing in this job needs to pass a
|
||||
value to a later step, and a guard that has to emulate the runner's `GITHUB_ENV` semantics would
|
||||
be a second implementation of exactly the kind that #767 already deleted once.
|
||||
"""
|
||||
offenders = [s.get("name") for s in _run_steps() if "GITHUB_ENV" in s["run"]]
|
||||
assert not offenders, (
|
||||
f"step(s) {offenders} in `{JOB}` write to GITHUB_ENV, which injects environment into later "
|
||||
"steps at runtime where no static check — including this file's execution probe — can see "
|
||||
"it. `PYTEST_ADDOPTS` supplied that way disarms the ban entirely."
|
||||
)
|
||||
|
||||
|
||||
def test_the_scan_job_is_not_advisory_at_JOB_level():
|
||||
"""`continue-on-error` on the JOB reports success to dependents even when it failed.
|
||||
|
||||
The step-level check elsewhere in this file does not see this tier — the same one-tier-up miss
|
||||
that let a job-level `env:` disarm the probe.
|
||||
"""
|
||||
assert not _job().get("continue-on-error"), (
|
||||
f"`{JOB}` is job-level advisory — a failure would report success to `build`"
|
||||
)
|
||||
|
||||
|
||||
def test_the_scan_job_has_NO_job_level_if():
|
||||
"""Two failure modes at once, in opposite directions.
|
||||
|
||||
An `if:` that excludes the tag push would leave the release path unguarded — the exact hole
|
||||
#767 closed. An `if:` that skipped it for any other reason would SKIP `build` too (a skipped
|
||||
dependency skips its dependents), breaking every release. Neither is wanted: it always runs.
|
||||
"""
|
||||
job = _job()
|
||||
assert "if" not in job, f"`{JOB}` must carry no job-level `if:`, found {job.get('if')!r}"
|
||||
|
||||
|
||||
def _pytest_line() -> str:
|
||||
"""The ONE non-comment line that actually invokes pytest, from the job's real body.
|
||||
|
||||
A substring test over the whole body is not enough, and that is not hypothetical: review
|
||||
disarmed the gate three ways that all left the suite green — commenting the invocation out and
|
||||
echoing instead (the filename still appears, in the comment), appending `-k 'not
|
||||
delimiter_banned'`, and appending `|| true`. Each left `scan` green with the ban unchecked and
|
||||
`build` publishing. So this locates the executable line and the caller asserts its shape.
|
||||
"""
|
||||
lines = [
|
||||
ln.strip()
|
||||
for s in _run_steps()
|
||||
for ln in s["run"].splitlines()
|
||||
if "python3 -m pytest" in ln and not ln.strip().startswith("#")
|
||||
]
|
||||
assert len(lines) == 1, f"expected exactly 1 pytest invocation in `{JOB}`, found {len(lines)}"
|
||||
return lines[0]
|
||||
|
||||
|
||||
def test_the_scan_job_actually_invokes_the_ban_test():
|
||||
"""Otherwise the job is an expensive no-op that reports green.
|
||||
|
||||
The path must appear as a BARE TOKEN on the pytest line — not merely somewhere in the body —
|
||||
so that commenting the invocation out is a red. Renaming the ban test without updating the
|
||||
workflow is a red here too, rather than a silently unguarded release path.
|
||||
"""
|
||||
assert (REPO_ROOT / BAN_TEST_FILE).is_file()
|
||||
assert BAN_TEST_FILE in _pytest_line().split(), (
|
||||
f"`{JOB}` does not pass {BAN_TEST_FILE} to pytest as an argument: {_pytest_line()!r}"
|
||||
)
|
||||
|
||||
|
||||
def _scan_body_and_env():
|
||||
"""The ban step's REAL `run:` body, and the FULL env the runner would give it.
|
||||
|
||||
ALL THREE TIERS, lowest precedence first: workflow `env:`, job `env:`, step `env:`. Reconstructing
|
||||
only the step tier is not a smaller version of this — it is a hole, and review reproduced it: a
|
||||
`PYTEST_ADDOPTS: -k "not delimiter_banned"` placed on the JOB disarmed the real gate (exit 0 with
|
||||
a poisoned `Smoke` body) while every test here stayed green, because the probe never saw that
|
||||
tier. It is the exact twin of the step-level `PYTEST_ADDOPTS` disarm caught one round earlier —
|
||||
which is the lesson: after fixing one tier, enumerate the others rather than assuming the fix
|
||||
generalised.
|
||||
"""
|
||||
step = next(s for s in _run_steps() if "python3 -m pytest" in s["run"])
|
||||
env = {}
|
||||
for tier in (_DOC.get("env"), _job().get("env"), step.get("env")):
|
||||
env.update({str(k): str(v) for k, v in (tier or {}).items()})
|
||||
return step["run"], env
|
||||
|
||||
|
||||
def _repo_copy(tmp_path: Path) -> Path:
|
||||
"""A minimal executable copy of the repo: the tracked workflows plus the tracked `scripts/`.
|
||||
|
||||
ASSESSED FOR ersatztv#806. This is NOT a completeness guard — it is a fixture assembling a
|
||||
harness, and no assertion in this file is about which files it found; the probes assert what the
|
||||
scan command DOES to the copy. It takes its file LIST from the index anyway, for hermeticity
|
||||
rather than completeness: `shutil.copytree` copied whatever was on disk, so untracked files and
|
||||
`scripts/__pycache__` entered a tree whose behaviour the probes then measure.
|
||||
|
||||
WHAT THAT DOES AND DOES NOT BUY, stated exactly, because a fixture described as hermetic stops
|
||||
being questioned. The list comes from the index; the CONTENT comes from the working tree, so an
|
||||
unstaged edit to a tracked `scripts/**` file is still copied in. Making the content hermetic too
|
||||
would need `git show`/`git archive` and would mean the probes stop testing the tree under edit,
|
||||
which is the wrong trade for a test whose job is to catch a disarm in that tree.
|
||||
|
||||
An extra WORKFLOW in the copy is inert, but check what the step runs before relying on that:
|
||||
`docker-build.yml:723` runs TWO files, `test_ci_dropped_step_guard.py` AND this one, and both
|
||||
parse only `docker-build.yml`.
|
||||
|
||||
THE COPY IS NOT A GIT REPOSITORY, and that is the constraint to know before touching either file
|
||||
that step runs. Neither may derive a population through `scripts/tests/tracked_files.py`:
|
||||
`git ls-files` inside the copy fails, and the release-path scan step fails with it. The copying
|
||||
happens HERE, in the real repo, which is why this fixture may use the index while its subjects
|
||||
may not. `test_ci_dropped_step_guard.py`'s `MARKED_JOBS` is explicitly left open as a residual
|
||||
gap, so a future session is invited to edit exactly that file — this paragraph is what stands
|
||||
between that edit and a broken release gate.
|
||||
"""
|
||||
dst = tmp_path / "repo"
|
||||
(dst / ".gitea" / "workflows").mkdir(parents=True)
|
||||
for wf in tracked_files.tracked_paths(*WORKFLOWS):
|
||||
shutil.copy2(wf, dst / ".gitea" / "workflows" / wf.name)
|
||||
for rel in tracked_files._git_ls_files():
|
||||
if not rel.startswith("scripts/"):
|
||||
continue
|
||||
source = REPO_ROOT / rel
|
||||
# Diagnosed, not raised as a bare FileNotFoundError one line after using the module whose
|
||||
# whole point is reporting this case comprehensibly.
|
||||
assert source.is_file(), (
|
||||
f"git tracks {rel} but there is no file there, so the harness copy would be incomplete "
|
||||
"and the probes below would measure a tree that is missing part of the thing under test."
|
||||
)
|
||||
target = dst / rel
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
shutil.copy2(source, target)
|
||||
return dst
|
||||
|
||||
|
||||
# THE RECURSION FENCE. The scan step's real command runs this very file, whose two probe tests would
|
||||
# each spawn the command again — unbounded recursion, discovered by running it. The fence lets the
|
||||
# nested run skip just those two probes while still executing the ban test, which is the part that
|
||||
# matters. Deliberately NOT a `-k`/`--ignore` on the nested command line: narrowing the command would
|
||||
# mean the thing under test is no longer the command the workflow runs.
|
||||
#
|
||||
# IT IS A NONCE, NOT A CONSTANT. A constant fence is a one-line silent kill switch: exporting
|
||||
# `ETV_SCAN_PROBE_NESTED=1` anywhere — a job `env:`, a runner default — would delete both
|
||||
# load-bearing probes everywhere, with nothing going red, after which the disarms they exist to catch
|
||||
# become unobservable again. So the parent writes a fresh value into the copied tree and passes the
|
||||
# same value in the environment; the skip fires only when the two agree, which cannot happen outside
|
||||
# a run this file itself spawned.
|
||||
_FENCE = "ETV_SCAN_PROBE_NESTED"
|
||||
_NONCE_FILE = ".scan-probe-nonce"
|
||||
|
||||
|
||||
def _fence_active() -> bool:
|
||||
value = os.environ.get(_FENCE)
|
||||
if not value:
|
||||
return False
|
||||
try:
|
||||
return (Path.cwd() / _NONCE_FILE).read_text().strip() == value
|
||||
except OSError:
|
||||
return False
|
||||
|
||||
|
||||
_nested = pytest.mark.skipif(
|
||||
_fence_active(),
|
||||
reason="nested invocation of the scan step's own command — the outer run owns this probe",
|
||||
)
|
||||
|
||||
|
||||
def _run_the_real_scan_body(repo: Path, tmp_path: Path):
|
||||
body, step_env = _scan_body_and_env()
|
||||
env = {
|
||||
"PATH": os.environ["PATH"],
|
||||
"HOME": os.environ.get("HOME", str(tmp_path)),
|
||||
"GITHUB_WORKSPACE": str(repo),
|
||||
"RUNNER_TEMP": str(tmp_path / "runner"),
|
||||
"GITHUB_JOB": JOB,
|
||||
"GITHUB_RUN_ID": "990099",
|
||||
"GITHUB_RUN_ATTEMPT": "1",
|
||||
}
|
||||
env.update(step_env)
|
||||
# The fence is applied AFTER the workflow's own env tiers, so the file under test cannot control
|
||||
# it in either direction — it could otherwise turn the nested probes back on (recursion) or off.
|
||||
nonce = uuid.uuid4().hex
|
||||
(repo / _NONCE_FILE).write_text(nonce)
|
||||
env[_FENCE] = nonce
|
||||
(tmp_path / "runner").mkdir(exist_ok=True)
|
||||
return subprocess.run(["bash", "-c", body], cwd=repo, env=env, capture_output=True, text=True)
|
||||
|
||||
|
||||
@_nested
|
||||
def test_the_scan_step_REALLY_FAILS_on_a_poisoned_workflow(tmp_path):
|
||||
"""THE POSITIVE CONTROL, and the one test here that is not a shape assertion.
|
||||
|
||||
It executes the ban step's actual `run:` body — with its actual `env:` — against a copy of the
|
||||
repo whose `build`/`Smoke` body carries an injected delimiter, and requires a non-zero exit.
|
||||
|
||||
This exists because shape checks lost. An earlier round asserted things ABOUT the command
|
||||
(the ban test's path is a bare token, no `-k`/`--deselect`, no `||`) and review then disarmed the
|
||||
gate anyway, every time leaving the suite green: `echo`ing the command instead of running it,
|
||||
flags moved past a `\\` continuation, a trailing `exit 0`, `if false; then … fi`, `set +e`, and
|
||||
`PYTEST_ADDOPTS` in the step's `env:`. Enumerating shapes of a command cannot win; running the
|
||||
command settles them together, including the two in the sibling module that no shape check could
|
||||
reach (a parametrize list swapped to `MARKED_JOBS`, and a neutered opener regex). No count is
|
||||
given on purpose — an earlier draft's tally disagreed with its own list, and the number was never
|
||||
the point.
|
||||
|
||||
The negative control below is what stops this passing for the wrong reason. And note the limit of
|
||||
this probe: it runs the command in a COPY, so it does not inherit a repo-root pytest config —
|
||||
that tier is covered by `scripts/ci-prove-ban-detects.sh`, which runs in the real checkout.
|
||||
"""
|
||||
repo = _repo_copy(tmp_path)
|
||||
wf = repo / ".gitea" / "workflows" / "docker-build.yml"
|
||||
text = wf.read_text()
|
||||
anchor = ' IMG="${IMAGE}:${SMOKE_SHORT_SHA}"'
|
||||
assert anchor in text, "anchor for the poison is gone — rewrite this control"
|
||||
wf.write_text(text.replace(anchor, " # ${{ steps.meta.outputs.short }}\n" + anchor, 1))
|
||||
|
||||
res = _run_the_real_scan_body(repo, tmp_path)
|
||||
assert res.returncode != 0, (
|
||||
"the scan step PASSED on a workflow whose `build`/`Smoke` body contains an expression "
|
||||
f"delimiter — the release-path gate is disarmed.\nstdout:\n{res.stdout}\nstderr:\n{res.stderr}"
|
||||
)
|
||||
|
||||
|
||||
@_nested
|
||||
def test_the_scan_step_PASSES_on_the_clean_workflow(tmp_path):
|
||||
"""The negative control for the test above.
|
||||
|
||||
Without it, a scan step that failed for ANY reason — a missing interpreter, a bad path, a
|
||||
syntax error in the body — would satisfy the positive control while proving nothing about
|
||||
detection. This also demonstrates the copied tree is a working harness rather than a broken one.
|
||||
"""
|
||||
res = _run_the_real_scan_body(_repo_copy(tmp_path), tmp_path)
|
||||
assert res.returncode == 0, (
|
||||
f"the scan step failed on a CLEAN workflow.\nstdout:\n{res.stdout}\nstderr:\n{res.stderr}"
|
||||
)
|
||||
|
||||
|
||||
@_nested
|
||||
def test_the_PROOF_SCRIPT_itself_refuses_when_the_ban_is_deselected(tmp_path):
|
||||
"""A positive control for the positive control — the one guard that had none.
|
||||
|
||||
Everything else here is guarded by something; `ci-prove-ban-detects.sh` was guarded only by a
|
||||
presence-and-executable-bit check, so a plausible "simplify" edit (relaxing the exit-code test
|
||||
back to "any non-zero means it noticed") would silently reinstate a full disarm with every test
|
||||
green. That is the "a guard only exercised on the happy path proves nothing" failure this file
|
||||
argues about every other guard.
|
||||
|
||||
The disarm reproduced here is the real one: a repo-root `conftest.py` that deselects the ban test
|
||||
entirely, which makes pytest exit 5 (nothing collected) rather than fail — the exact reading that
|
||||
made an earlier draft of the script report the gate healthy while an unsmoked candidate would
|
||||
publish.
|
||||
"""
|
||||
repo = _repo_copy(tmp_path)
|
||||
(repo / "conftest.py").write_text(
|
||||
"def pytest_collection_modifyitems(config, items):\n"
|
||||
" items[:] = [i for i in items if 'test_ci_dropped_step_guard' not in str(i.fspath)]\n"
|
||||
)
|
||||
res = subprocess.run(
|
||||
["bash", str(repo / "scripts" / "ci-prove-ban-detects.sh")],
|
||||
cwd=repo,
|
||||
env={**os.environ, "GITHUB_WORKSPACE": str(repo)},
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
assert res.returncode != 0, (
|
||||
"ci-prove-ban-detects.sh vouched for the gate while the ban test was deselected at the "
|
||||
f"repo-root config tier.\nstdout:\n{res.stdout}\nstderr:\n{res.stderr}"
|
||||
)
|
||||
# NON-ZERO IS NOT ENOUGH — the script could exit non-zero because its own harness broke (a moved
|
||||
# anchor, an unwritable tree, no python3), which would make this test pass while proving nothing.
|
||||
# Require one of its real verdicts. Measured, this path yields "pytest exited 5" (nothing
|
||||
# collected), which is precisely the deselection disarm and precisely the reading an earlier draft
|
||||
# mistook for success.
|
||||
combined = res.stdout + res.stderr
|
||||
# THE SPECIFIC branch, not a disjunction over the script's verdicts. Measured: a total deselect
|
||||
# makes pytest exit 5, so this lands on "cannot prove anything". Accepting any verdict would let
|
||||
# the scenario drift onto a branch it was not written for while still looking green — this file's
|
||||
# own subject, one level down.
|
||||
assert "could not prove anything" in combined, (
|
||||
"expected the cannot-prove branch (a total deselect makes pytest exit 5); got a different "
|
||||
f"verdict, so this test no longer covers what it was written for.\nstdout:\n{res.stdout}\n"
|
||||
f"stderr:\n{res.stderr}"
|
||||
)
|
||||
|
||||
|
||||
@_nested
|
||||
def test_the_PROOF_SCRIPT_refuses_when_the_WRONG_test_fails(tmp_path):
|
||||
"""The third branch, which the aggregate 'deselect ⇒ non-zero' control does not reach.
|
||||
|
||||
Deselecting only the `[build]` parametrisation while some unrelated test fails gives pytest exit
|
||||
1 — a real test failure, just not the one that proves anything. An earlier draft read that as
|
||||
success. Reproduced here because that branch was added to fix a live bug and was otherwise
|
||||
exercised by nothing: making it unreachable left both guard files green.
|
||||
"""
|
||||
repo = _repo_copy(tmp_path)
|
||||
(repo / "conftest.py").write_text(
|
||||
"def pytest_collection_modifyitems(config, items):\n"
|
||||
" items[:] = [i for i in items if 'in_any_run_body[build]' not in i.name]\n"
|
||||
)
|
||||
# The unrelated failure has to live INSIDE the ban file: the script runs that file and nothing
|
||||
# else, so a failing test in a sibling module is never collected and the run would exit 0 —
|
||||
# landing on the "not enforcing" branch instead of the one under test. (First draft of this test
|
||||
# did exactly that and was red for the wrong reason.)
|
||||
ban = repo / BAN_TEST_FILE
|
||||
ban.write_text(ban.read_text() + "\n\ndef test_an_unrelated_failure_for_this_probe():\n assert False\n")
|
||||
res = subprocess.run(
|
||||
["bash", str(repo / "scripts" / "ci-prove-ban-detects.sh")],
|
||||
cwd=repo,
|
||||
env={**os.environ, "GITHUB_WORKSPACE": str(repo)},
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
combined = res.stdout + res.stderr
|
||||
assert res.returncode != 0, f"the script read an unrelated test's failure as proof.\n{combined}"
|
||||
# THE SPECIFIC branch. This scenario is built to land on "wrong test failed" (exit 1, no `[build]`
|
||||
# failure); accepting "could not prove anything" too would let it drift onto the exit-5 branch and
|
||||
# silently cover a branch it was not written for, while still looking green.
|
||||
assert "NOT the expected one" in combined, (
|
||||
"expected the wrong-test-failed branch; got a different verdict, so this scenario no longer "
|
||||
f"covers the branch it was written for.\n{combined}"
|
||||
)
|
||||
|
||||
|
||||
@_nested
|
||||
def test_the_PROOF_SCRIPT_passes_on_a_clean_tree(tmp_path):
|
||||
"""Negative control for the test above: it must not simply always fail."""
|
||||
repo = _repo_copy(tmp_path)
|
||||
res = subprocess.run(
|
||||
["bash", str(repo / "scripts" / "ci-prove-ban-detects.sh")],
|
||||
cwd=repo,
|
||||
env={**os.environ, "GITHUB_WORKSPACE": str(repo)},
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
assert res.returncode == 0, (
|
||||
f"ci-prove-ban-detects.sh failed on a clean tree.\nstdout:\n{res.stdout}\nstderr:\n{res.stderr}"
|
||||
)
|
||||
assert (repo / ".gitea" / "workflows" / "docker-build.yml").read_text() == (WORKFLOW.read_text()), (
|
||||
"the script did not restore the workflow file it poisoned"
|
||||
)
|
||||
|
||||
|
||||
def test_the_pytest_invocation_cannot_DESELECT_or_swallow_its_result():
|
||||
"""Selection flags and shell escapes are the cheap ways to keep the job green while it checks
|
||||
nothing. `-k`/`-m`/`--deselect`/`--ignore` can drop the ban test from a run that still reports
|
||||
passes; `|| true` and friends discard the exit status the `needs:` edge depends on."""
|
||||
line = _pytest_line()
|
||||
tokens = line.split()
|
||||
# Only the tokens AFTER `pytest` are pytest's own arguments. Checking the whole line would flag
|
||||
# the `-m` in `python3 -m pytest`, which is how the interpreter is invoked — a false positive
|
||||
# that would make this test red on the correct command.
|
||||
args = tokens[tokens.index("pytest") + 1 :]
|
||||
banned = {"-k", "-m", "--deselect", "--ignore", "--collect-only", "--co"}
|
||||
assert not (banned & set(args)), f"pytest invocation may deselect tests: {line!r}"
|
||||
for op in ("||", "&&", ";", "|"):
|
||||
assert op not in tokens, f"pytest exit status is not decisive — {op!r} in {line!r}"
|
||||
|
||||
|
||||
def test_the_ban_SCOPE_still_covers_build():
|
||||
"""The whole release-path property rests on one literal in the sibling module.
|
||||
|
||||
`DELIMITER_BAN_JOBS` is what the ban test parametrises over. Drop `"build"` from it and the
|
||||
suite still reports passes while nothing checks the job that publishes the image — green, and
|
||||
the release path unguarded. Nothing else in scripts/tests referenced that constant, so this is
|
||||
the pin. (An earlier design imported the tuple, which pinned it as a side effect; the import
|
||||
went away with that design and took the protection with it.)
|
||||
"""
|
||||
import scripts.tests.test_ci_dropped_step_guard as ban
|
||||
|
||||
assert "build" in ban.DELIMITER_BAN_JOBS, (
|
||||
"`build` was dropped from DELIMITER_BAN_JOBS — the `scan` job would go green while the job "
|
||||
"that publishes the release candidate is unchecked (ersatztv#767)"
|
||||
)
|
||||
|
||||
|
||||
def test_no_step_in_the_scan_job_is_advisory():
|
||||
"""`continue-on-error: true` would make the whole gate a no-op while every other test here
|
||||
stayed green — it is the cheapest way to accidentally disarm this."""
|
||||
offenders = [s.get("name") for s in _steps() if s.get("continue-on-error")]
|
||||
assert not offenders, f"advisory step(s) in `{JOB}`: {offenders}"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"step_name",
|
||||
[s.get("name", "?") for s in yaml.safe_load(WORKFLOW.read_text())["jobs"][JOB]["steps"] if s.get("run")],
|
||||
)
|
||||
def test_every_run_body_in_the_scan_job_is_delimiter_free(step_name):
|
||||
"""The guard must not be vulnerable to the defect it guards against.
|
||||
|
||||
Not a proof that it always runs — a construction argument about ONE mechanism, the same axiom
|
||||
the sibling guards rest on. It is asserted per step so a failure names which step regressed.
|
||||
"""
|
||||
step = next(s for s in _run_steps() if s.get("name", "?") == step_name)
|
||||
assert not _OPENER.search(step["run"]), (
|
||||
f"step {step_name!r} of `{JOB}` contains an expression delimiter; the runner would rewrite "
|
||||
"the whole body and DROP the step while reporting success (ersatztv#751). Pass values "
|
||||
"through `env:`, which is interpolated per value."
|
||||
)
|
||||
|
||||
|
||||
# ------------------------------------------------------------------------------------------------
|
||||
# THE JOB'S OWN DROPPED-STEP GUARD
|
||||
# ------------------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_every_consequential_step_marks_itself():
|
||||
"""Every `run:` step except the guard records that it executed."""
|
||||
marked = {s.get("name") for s, _ in _marked()}
|
||||
expected = {s.get("name") for s in _run_steps() if s is not _guard()}
|
||||
assert marked == expected, f"unmarked step(s) in `{JOB}`: {expected - marked}"
|
||||
|
||||
|
||||
def test_the_guard_expectations_match_the_markers_exactly():
|
||||
"""The set the guard waits for IS the set the steps write — derived from the workflow, not
|
||||
restated here, so adding a step without a marker is a red."""
|
||||
argv = _guard()["run"].split()
|
||||
assert "--always" in argv, argv
|
||||
always = argv[argv.index("--always") + 1 :]
|
||||
assert "--gated" not in argv, "every step in this job is unconditional; there is nothing to gate"
|
||||
assert sorted(always) == sorted(k for _, k in _marked())
|
||||
|
||||
|
||||
def test_the_guard_is_the_LAST_step():
|
||||
assert _steps()[-1] is _guard(), "the assert must run after the steps it checks"
|
||||
|
||||
|
||||
def test_the_guard_has_no_if():
|
||||
"""Same reasoning as the sibling guards: the default `success()` is wanted, because a genuine
|
||||
early failure legitimately skips later steps and already fails the job."""
|
||||
assert "if" not in _guard()
|
||||
|
||||
|
||||
# ------------------------------------------------------------------------------------------------
|
||||
# BEHAVIOURAL — the guard's REAL command line, against the steps' REAL marker lines
|
||||
# ------------------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _mark_line(step) -> str:
|
||||
"""The step's own marker line, verbatim from the workflow — never rebuilt in Python, so a
|
||||
drift between the workflow and the script cannot hide behind a test that composed its own."""
|
||||
return next(ln for ln in step["run"].splitlines() if _MARK.search(ln)).strip()
|
||||
|
||||
|
||||
def _env(tmp_path, **extra):
|
||||
env = {
|
||||
"PATH": os.environ["PATH"],
|
||||
"GITHUB_WORKSPACE": str(REPO_ROOT),
|
||||
"RUNNER_TEMP": str(tmp_path),
|
||||
"GITHUB_JOB": JOB,
|
||||
"GITHUB_RUN_ID": "424242",
|
||||
"GITHUB_RUN_ATTEMPT": "7",
|
||||
}
|
||||
env.update(extra)
|
||||
return {k: v for k, v in env.items() if v is not None}
|
||||
|
||||
|
||||
def _run(script: str, env):
|
||||
return subprocess.run(["bash", "-c", script], cwd=REPO_ROOT, env=env, capture_output=True, text=True)
|
||||
|
||||
|
||||
def test_the_guard_PASSES_when_every_step_ran(tmp_path):
|
||||
env = _env(tmp_path)
|
||||
for step, _ in _marked():
|
||||
assert _run(_mark_line(step), env).returncode == 0
|
||||
res = _run(_guard()["run"], env)
|
||||
assert res.returncode == 0, res.stderr
|
||||
|
||||
|
||||
@pytest.mark.parametrize("dropped", [k for _, k in _marked()])
|
||||
def test_the_guard_FAILS_when_a_step_was_dropped(tmp_path, dropped):
|
||||
"""The positive control. Drop each key in turn — the guard must go red and NAME it.
|
||||
|
||||
A guard only ever exercised on the happy path is indistinguishable from one that passes
|
||||
unconditionally, which is the failure this whole mechanism exists to remove.
|
||||
"""
|
||||
env = _env(tmp_path)
|
||||
for step, key in _marked():
|
||||
if key != dropped:
|
||||
assert _run(_mark_line(step), env).returncode == 0
|
||||
res = _run(_guard()["run"], env)
|
||||
assert res.returncode != 0, f"guard passed despite '{dropped}' never running: {res.stdout}"
|
||||
# BOTH streams: the script's `::error::` lands on stdout here while other diagnostics go to
|
||||
# stderr, and a test that picked the wrong one would assert on an empty string and pass for the
|
||||
# wrong reason on any message change.
|
||||
assert dropped in (res.stdout + res.stderr), (res.stdout, res.stderr)
|
||||
|
||||
|
||||
def test_the_guard_REFUSES_to_pass_with_no_expectations(tmp_path):
|
||||
"""`assert` with an empty expectation set would report success having checked nothing."""
|
||||
res = _run(f"{SCRIPT} assert --always", _env(tmp_path))
|
||||
assert res.returncode != 0
|
||||
@@ -744,7 +744,9 @@ def test_wing_faults_block_scalar_record_fails_loudly(tmp_path):
|
||||
this corpus's very long `rule:` values — makes the whole record silently invisible."""
|
||||
records, archive = _wing(tmp_path)
|
||||
bad = records / "ci" / "blockscalar.md"
|
||||
bad.write_text(_GOOD.replace("rule: 'a rule on one quoted line'\n", "rule: >-\n a long rule wrapped\n over two lines\n"))
|
||||
bad.write_text(
|
||||
_GOOD.replace("rule: 'a rule on one quoted line'\n", "rule: >-\n a long rule wrapped\n over two lines\n")
|
||||
)
|
||||
|
||||
# Precondition: this really is the silent-vanish case, not some other parse error.
|
||||
assert dl.parse_file(bad) == [], "expected the reader to drop the record entirely"
|
||||
@@ -805,8 +807,10 @@ def test_wing_faults_exempts_stripped_legacy_archive_files(tmp_path):
|
||||
|
||||
def test_validate_surfaces_wing_faults_as_errors(tmp_path):
|
||||
"""Faults must arrive as validator ERRORS (exit 1), not warnings."""
|
||||
errs = _v([_rec(key="ci.a", source=Path("docs/decisions/records/ci/a.md"), heading="A")],
|
||||
wing_faults=["docs/decisions/records/ci/x.md: parsed to 0 records, expected exactly 1"])
|
||||
errs = _v(
|
||||
[_rec(key="ci.a", source=Path("docs/decisions/records/ci/a.md"), heading="A")],
|
||||
wing_faults=["docs/decisions/records/ci/x.md: parsed to 0 records, expected exactly 1"],
|
||||
)
|
||||
assert any("x.md" in e for e in errs), errs
|
||||
|
||||
|
||||
@@ -826,7 +830,7 @@ def test_wing_faults_sees_a_DEEPER_nested_archive_record(tmp_path):
|
||||
one level down'."""
|
||||
records, archive = _wing(tmp_path)
|
||||
(records / "ci" / "good.md").write_text(_GOOD)
|
||||
(archive / "api.md").write_text("# api\n\n## Records formerly in this file\n") # still exempt
|
||||
(archive / "api.md").write_text("# api\n\n## Records formerly in this file\n") # still exempt
|
||||
deep = archive / "ci" / "sub"
|
||||
deep.mkdir(parents=True)
|
||||
(deep / "broken.md").write_text("# not a record\n")
|
||||
@@ -887,8 +891,7 @@ def test_junk_frontmatter_key_from_a_split_value_is_faulted(tmp_path):
|
||||
PyYAML rejects this input, so the hand reader is more permissive than the writer."""
|
||||
records, archive = _wing(tmp_path)
|
||||
bad = records / "ci" / "corrupt.md"
|
||||
bad.write_text(_GOOD.replace("rule: 'a rule on one quoted line'\n",
|
||||
"rule: >-\nthe real rule: with a colon\n"))
|
||||
bad.write_text(_GOOD.replace("rule: 'a rule on one quoted line'\n", "rule: >-\nthe real rule: with a colon\n"))
|
||||
recs = dl.parse_file(bad)
|
||||
assert len(recs) == 1 and recs[0].key, "precondition: this parses to one KEYED record"
|
||||
assert recs[0].rule == ">-", f"precondition: the real value was truncated, got {recs[0].rule!r}"
|
||||
@@ -903,7 +906,7 @@ def test_an_empty_or_missing_record_wing_is_LOUD(tmp_path):
|
||||
missing = dv.record_wing_faults(tmp_path / "nope" / "records", tmp_path / "nope" / "archive")
|
||||
assert missing and "missing or contains no" in missing[0], missing
|
||||
|
||||
records, archive = _wing(tmp_path) # exists but holds no *.md
|
||||
records, archive = _wing(tmp_path) # exists but holds no *.md
|
||||
empty = dv.record_wing_faults(records, archive)
|
||||
assert empty and "missing or contains no" in empty[0], empty
|
||||
|
||||
@@ -1437,7 +1440,6 @@ def test_no_budget_flag_means_no_retirement_warning(capsys):
|
||||
assert "is RETIRED and was IGNORED" not in capsys.readouterr().err
|
||||
|
||||
|
||||
|
||||
def test_main_reports_ceiling_drift_as_a_NOTICE_and_still_exits_0(capsys):
|
||||
"""The fine claim's live wiring (#688): the drift notice must fire, and must NOT turn the run
|
||||
red — the entire point of the v5 split.
|
||||
@@ -1511,8 +1513,7 @@ def test_main_actually_REPORTS_the_ceiling_and_the_trend(capsys):
|
||||
err = capsys.readouterr().err
|
||||
assert "prose lines across" in err, "the aggregate trend notice must always print"
|
||||
|
||||
over = [r.key for r in dl.all_active_records()
|
||||
if r.key and dv.record_prose_lines(r) > dv.RECORD_CEILING_DEFAULT]
|
||||
over = [r.key for r in dl.all_active_records() if r.key and dv.record_prose_lines(r) > dv.RECORD_CEILING_DEFAULT]
|
||||
warned = "exceed the" in err and "prose ceiling" in err
|
||||
assert warned is bool(over), f"ceiling warning printed={warned} but {len(over)} record(s) are over it"
|
||||
if over:
|
||||
|
||||
@@ -0,0 +1,331 @@
|
||||
"""docs/guard-inventory.md covers exactly the guards that exist (ersatztv#774, #775).
|
||||
|
||||
THIS FILE IS THE ANSWER TO BOTH ISSUES' "is a mechanical check possible?" QUESTION, and the shape of
|
||||
the answer matters more than the code.
|
||||
|
||||
What is NOT possible, and must not be attempted: a lint that flags filter-shaped guards by matching
|
||||
`.filter(` / `.Where(` / `grep` inside guard code. #774 asks for this explicitly and the honest
|
||||
answer is no. The token is not the defect — `ToolCatalogTests.Every_Query_Parameter_Should_Be_A_
|
||||
Declared_Property` filters correctly eight lines from a completeness assertion that must not. A
|
||||
matcher would flag both, and a matcher that is wrong half the time is waved through until it is
|
||||
never read, which is the symptom-keyed-guard mistake this repo has now paid for at #644 and #650.
|
||||
Worse, it is a string predicate over source, and this repo's own record is that a string predicate
|
||||
takes three or more rounds to get right (#629, #633, #698). Building it would be #774 violating
|
||||
#774.
|
||||
|
||||
What IS possible, and is what this file does: you cannot mechanically detect that a guard reasons
|
||||
about a sample, but you CAN mechanically guarantee that every guard has been *classified by someone*
|
||||
and that its claimed proof exists. That converts both rules from "remember to do this" into "the
|
||||
suite goes red until you have". Specifically:
|
||||
|
||||
* the guard population is DERIVED — read out of the GIT INDEX for `.claude/hooks/` and
|
||||
`.husky/`, plus every `scripts/*.sh|py` referenced by a workflow or a hook — and compared for
|
||||
SET EQUALITY against the inventory's rows, in both directions. The index rather than the disk
|
||||
per ersatztv#806: a filesystem walk reports build output and editor droppings and differs per
|
||||
machine, so it cannot be the authoritative source a completeness claim needs;
|
||||
* every row's `Proof ref` is resolved to a real file and a real `def` in it;
|
||||
* every row's `Kind` and `Proof` come from a closed vocabulary, so a typo cannot invent a state.
|
||||
|
||||
The residue this leaves, named rather than papered over: nothing here checks that a row's
|
||||
`MUTATION` claim is TRUE. A test named in the table might merely exercise the guard. That judgement
|
||||
stays with review, and the table is the thing review reads.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from collections import Counter
|
||||
from pathlib import Path
|
||||
|
||||
from scripts.tests import tracked_files
|
||||
from scripts.tests.tracked_files import tracked_children, tracked_paths
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[2]
|
||||
INVENTORY = REPO_ROOT / "docs" / "guard-inventory.md"
|
||||
|
||||
# THE POPULATION SCOPES, resolved against the GIT INDEX rather than the filesystem (ersatztv#806).
|
||||
# Directory + patterns instead of `Path.glob`; `scripts/tests/tracked_files.py` carries why the disk
|
||||
# is not an authoritative source. `.husky` is the sharp case: it holds an untracked `_/` of 17
|
||||
# npm-generated shims, and the previous `iterdir() ... if p.is_file()` excluded them only because
|
||||
# `_` happens to be a directory — by accident, not by design, so the obvious "make it recursive"
|
||||
# edit would have reintroduced #778's third defect inside the repo's own model guard.
|
||||
HOOKS = (".claude/hooks", ("*.sh",))
|
||||
HUSKY = (".husky", ("*",))
|
||||
WORKFLOWS = (".gitea/workflows", ("*.yml", "*.yaml"))
|
||||
GUARD_TESTS = ("scripts/tests", ("test_*.py",))
|
||||
|
||||
KINDS = {"GUARD", "TOOLING", "PROOF"}
|
||||
PROOFS = {"MUTATION", "BEHAVIOUR-ONLY", "NONE"}
|
||||
|
||||
# `scripts/x.sh` AND `scripts/tests/x.py`. The first version omitted the `/`, and the consequence was
|
||||
# not theoretical: the three guard files this inventory shipped with were themselves outside the
|
||||
# population it checked for completeness, so they acquired no rows and the guard stayed green. Cold
|
||||
# review found it. A completeness guard blind to its own author's new guards is the defect this
|
||||
# whole change is about, so the miss is recorded here rather than quietly corrected.
|
||||
_SCRIPT_REF = re.compile(r"scripts/(?:[a-z0-9_.-]+/)?[a-z0-9_.-]+\.(?:sh|py)")
|
||||
_ROW = re.compile(r"^\|\s*`([^`]+)`\s*\|([^|]*)\|\s*([A-Z-]+)\s*\|\s*([A-Z-]+)\s*\|([^|]*)\|\s*$", re.M)
|
||||
|
||||
# The prose summary, parsed so it cannot drift from the table it summarises. It already had:
|
||||
# shipped as "28 guards, 4 tooling … 6 … 3 … 19" against a table holding 27/5/6/3/18, because it was
|
||||
# a hand-maintained mirror with no equality check — #773's Family C inside the deliverable arguing
|
||||
# against it. Both reviewers caught it independently.
|
||||
_SUMMARY = re.compile(
|
||||
r"(\d+)\s+guards?,\s+(\d+)\s+tooling\s+scripts?,\s+(\d+)\s+proof\s+files?\.\s+"
|
||||
r"\*\*(\d+)\s+guards?\s+carry\s+a\s+mutation\s+proof;\s+(\d+)\s+(?:are|is)\s+behaviour-only;\s+"
|
||||
r"(\d+)\s+have\s+none\.\*\*"
|
||||
)
|
||||
|
||||
|
||||
def derived_guard_files() -> set[str]:
|
||||
"""THE AUTHORITATIVE POPULATION, from the git index and the call sites — never a list.
|
||||
|
||||
Four contributors, unioned: three scope directories plus the paths those files REFERENCE. The
|
||||
hook and husky directories are taken whole, so a new hook is in the population the moment it is
|
||||
STAGED. The scripts half is discovered by scanning what the workflows and hooks actually
|
||||
INVOKE, rather than taking `scripts/` whole — a script nothing calls is not a guard, and taking
|
||||
it whole would drag in every helper and make the inventory a chore that gets rubber-stamped.
|
||||
Both halves are gated on the index: the callers by `tracked_paths`, the targets they name by the
|
||||
`tracked` set below.
|
||||
|
||||
"The moment it is STAGED" rather than "the moment it exists" is the ersatztv#806 change, and
|
||||
it is a strengthening: an untracked `foo.sh` dropped in `.claude/hooks/` used to enter this
|
||||
population and demand an inventory row for a file that is not part of the repo — red on that
|
||||
checkout, green in CI, which is #778's third shape. Nothing weakens, because a guard that is not
|
||||
staged is not on its way to anyone else either.
|
||||
"""
|
||||
found = (
|
||||
tracked_children(*HOOKS)
|
||||
| tracked_children(*HUSKY)
|
||||
# `pr-checks.yml` runs `pytest scripts/tests` as a directory, so every file in it is invoked
|
||||
# and none is individually named anywhere. Taking the directory whole is the only derivation
|
||||
# that matches how they actually run.
|
||||
| tracked_children(*GUARD_TESTS)
|
||||
)
|
||||
|
||||
# THE REFERENCED TARGETS ARE GATED ON THE INDEX, NOT ON `Path.exists()`. Converting the CALLERS
|
||||
# and leaving the members they contribute on a disk check would have left a quarter of this
|
||||
# population answering a question about the machine: a tracked workflow naming
|
||||
# `scripts/generated/helper.sh` that exists on one laptop only would enter there, demand an
|
||||
# inventory row for a file that is not in the repo, and go red on that checkout while CI stayed
|
||||
# green — #778's third shape, in the guard this file calls its model.
|
||||
#
|
||||
# A referenced path that git does not track is therefore dropped silently, and that is the right
|
||||
# residual rather than an assertion: `_SCRIPT_REF` matches any occurrence, including inside a
|
||||
# comment or an `::error::` string (limit 3 in `docs/guard-inventory.md`), so demanding that
|
||||
# every matched path be tracked would redden a correct tree on a prose mention.
|
||||
# Called through the MODULE, never `from … import _git_ls_files`. A direct name binding is
|
||||
# captured at import time, and the exhaustive proof in
|
||||
# `test_guard_populations_derive_from_git.py` then cannot narrow the index for this branch at
|
||||
# all — every referenced target reports as surviving removal, which is a red for the wrong
|
||||
# reason and, worse, means the branch is untested however the proof reads.
|
||||
tracked = set(tracked_files._git_ls_files())
|
||||
callers = tracked_paths(*WORKFLOWS) + tracked_paths(*HOOKS) + tracked_paths(*HUSKY)
|
||||
for caller in callers:
|
||||
for ref in _SCRIPT_REF.findall(caller.read_text()):
|
||||
if ref in tracked:
|
||||
found.add(ref)
|
||||
return found
|
||||
|
||||
|
||||
def wired_hook_files() -> set[str]:
|
||||
"""Hooks reachable from `.claude/settings.json` or a husky hook — WIRING, not existence.
|
||||
|
||||
Directory membership is not execution. A hook whose settings.json registration is deleted keeps
|
||||
its file, keeps its inventory row, and stops running — and the table would go on describing a
|
||||
working guard. That is #631 and #719's shape ("wired is not running") one level down, and it was
|
||||
the derivation's blind spot until cold review named it.
|
||||
|
||||
COMMENT LINES ARE STRIPPED from the husky hooks first, and that is not a refinement — the first
|
||||
version of this function counted a mention anywhere, and `.husky/pre-commit:7` reads
|
||||
|
||||
# CI where a base ref exists). Fail-open shim — see .claude/hooks/decisions-guard.sh.
|
||||
|
||||
one line above the real invocation. Delete line 8 and keep line 7 and the hook would still have
|
||||
read as wired, which is the exact substitution of mention for invocation this function exists to
|
||||
stop, reproduced inside the fix for it. `.claude/settings.json` needs no stripping: JSON has no
|
||||
comments, so every occurrence there is in a real command string.
|
||||
"""
|
||||
text = (REPO_ROOT / ".claude" / "settings.json").read_text()
|
||||
for husky in tracked_paths(*HUSKY):
|
||||
text += "\n".join(line for line in husky.read_text().splitlines() if not line.lstrip().startswith("#"))
|
||||
return {rel for rel in tracked_children(*HOOKS) if rel.rpartition("/")[2] in text}
|
||||
|
||||
|
||||
def inventory_rows() -> list[tuple[str, str, str, str]]:
|
||||
"""(guard, kind, proof, proof_ref) for each table row."""
|
||||
rows = []
|
||||
for guard, _blocks, kind, proof, ref in _ROW.findall(INVENTORY.read_text()):
|
||||
rows.append((guard.strip(), kind.strip(), proof.strip(), ref.strip()))
|
||||
return rows
|
||||
|
||||
|
||||
# ------------------------------------------------------------------------------------------------
|
||||
# ANTI-VACUITY FIRST — a row regex that stopped matching would make every assertion below compare
|
||||
# empty sets and report a fully-covered inventory. That is the failure this file exists to prevent,
|
||||
# so it is checked before anything depends on it.
|
||||
# ------------------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_the_table_actually_parsed():
|
||||
rows = inventory_rows()
|
||||
assert len(rows) >= 25, (
|
||||
f"only parsed {len(rows)} rows out of {INVENTORY.name} — the row pattern has stopped "
|
||||
"matching the table's markdown, so the coverage assertions below are vacuous."
|
||||
)
|
||||
assert len(derived_guard_files()) >= 25, "the guard discovery walk found almost nothing"
|
||||
|
||||
|
||||
def test_no_duplicate_rows():
|
||||
"""Two rows for one guard would let one satisfy the set comparison while the other says
|
||||
anything at all — including a fabricated proof."""
|
||||
guards = [g for g, _, _, _ in inventory_rows()]
|
||||
dupes = sorted({g for g in guards if guards.count(g) > 1})
|
||||
assert not dupes, f"{INVENTORY.name} has duplicate rows for {dupes}"
|
||||
|
||||
|
||||
# ------------------------------------------------------------------------------------------------
|
||||
# SET EQUALITY, BOTH DIRECTIONS
|
||||
# ------------------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_the_inventory_covers_exactly_the_guards_that_exist():
|
||||
listed = {g for g, _, _, _ in inventory_rows()}
|
||||
found = derived_guard_files()
|
||||
|
||||
missing = sorted(found - listed)
|
||||
assert not missing, (
|
||||
f"these guard files exist but have no row in {INVENTORY.name}: {missing}. Every guard must "
|
||||
"be classified — add a row giving what it blocks, whether it is a GUARD or TOOLING, and "
|
||||
"whether it ships a mutation proof. An unclassified guard is one nobody has decided is "
|
||||
"load-bearing, which is how #631's suite ran nowhere for months."
|
||||
)
|
||||
phantom = sorted(listed - found)
|
||||
assert not phantom, (
|
||||
f"{INVENTORY.name} lists {phantom}, which no longer exist or are no longer invoked by any "
|
||||
"workflow or hook. A row for a guard that does not run reads as coverage and is not."
|
||||
)
|
||||
|
||||
|
||||
# ------------------------------------------------------------------------------------------------
|
||||
# THE CLAIMS IN EACH ROW RESOLVE
|
||||
# ------------------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_every_row_uses_the_closed_vocabulary():
|
||||
for guard, kind, proof, _ref in inventory_rows():
|
||||
assert kind in KINDS, f"{guard}: Kind {kind!r} is not one of {sorted(KINDS)}"
|
||||
assert proof in PROOFS, f"{guard}: Proof {proof!r} is not one of {sorted(PROOFS)}"
|
||||
|
||||
|
||||
def test_every_claimed_proof_names_a_test_that_exists():
|
||||
"""The half that makes the table load-bearing rather than decorative.
|
||||
|
||||
A row claiming MUTATION with a `Proof ref` that no longer resolves is worse than a row claiming
|
||||
NONE: it tells the next reader this guard is covered. Renaming a test then silently converts a
|
||||
proven guard into an unproven one that still reads as proven, and nothing else in the repo
|
||||
would notice.
|
||||
"""
|
||||
for guard, _kind, proof, ref in inventory_rows():
|
||||
if proof == "NONE":
|
||||
assert ref in ("—", "-", ""), f"{guard}: Proof is NONE but a ref is given ({ref!r})"
|
||||
continue
|
||||
|
||||
assert "::" in ref, f"{guard}: Proof is {proof} but the ref {ref!r} is not file::function"
|
||||
filename, func = ref.strip("`").split("::", 1)
|
||||
path = REPO_ROOT / "scripts" / "tests" / filename
|
||||
assert path.exists(), f"{guard}: proof ref names {filename}, which does not exist"
|
||||
assert re.search(rf"^def {re.escape(func)}\(", path.read_text(), re.M), (
|
||||
f"{guard}: {filename} has no `def {func}(`. The proof ref is stale — either the test "
|
||||
"was renamed (update the row) or it was deleted (this guard is now unproven, and the "
|
||||
"row must say NONE)."
|
||||
)
|
||||
|
||||
|
||||
def test_every_hook_file_is_actually_WIRED():
|
||||
"""A hook file nothing registers is dead code holding an inventory row that reads as coverage."""
|
||||
staged = tracked_children(*HOOKS)
|
||||
unwired = sorted(staged - wired_hook_files())
|
||||
assert not unwired, (
|
||||
f"these hook files exist and have inventory rows but are referenced by neither "
|
||||
f".claude/settings.json nor any .husky/ hook: {unwired}. They do not run. Either wire them "
|
||||
"or delete them — a row for a hook that never fires is the coverage claim #631 paid for."
|
||||
)
|
||||
|
||||
|
||||
def test_proof_rows_do_not_themselves_claim_a_proof():
|
||||
"""`PROOF` exists to stop an infinite regress, and the regress is not hypothetical.
|
||||
|
||||
Once `scripts/tests/*.py` entered the population, every mutation proof became a row needing a
|
||||
proof of its own, and so on. `PROOF` marks a file whose job IS to prove another guard; grading
|
||||
it would ask what proves the prover, forever. Files under `scripts/tests/` that enforce a repo
|
||||
invariant with no separate guard file behind them are `GUARD`, not `PROOF`, and are graded
|
||||
normally — that is the honest place to draw the line.
|
||||
"""
|
||||
for guard, kind, proof, ref in inventory_rows():
|
||||
if kind == "PROOF":
|
||||
assert proof == "NONE", f"{guard} is PROOF but claims Proof {proof}"
|
||||
assert guard.startswith("scripts/tests/"), f"{guard} is marked PROOF but does not live in scripts/tests/"
|
||||
assert ref in ("—", "-", ""), f"{guard}: PROOF rows carry no proof ref"
|
||||
|
||||
|
||||
def test_every_proof_ref_points_at_a_row_marked_PROOF():
|
||||
"""Ties the two halves of the table together. A guard citing a test that the table does not
|
||||
classify as a PROOF means the population and the refs disagree about what that file is."""
|
||||
rows = inventory_rows()
|
||||
proof_files = {g for g, k, _, _ in rows if k == "PROOF"}
|
||||
for guard, _kind, proof, ref in rows:
|
||||
if proof == "NONE":
|
||||
continue
|
||||
filename = ref.strip("`").split("::", 1)[0]
|
||||
path = f"scripts/tests/{filename}"
|
||||
# A test-file GUARD may cite ITSELF: its mutation cases live in the same file, because the
|
||||
# thing it guards is a repo invariant rather than another script. Splitting those into a
|
||||
# separate file to satisfy the table would be bookkeeping driving the code.
|
||||
if path == guard:
|
||||
continue
|
||||
assert path in proof_files, (
|
||||
f"{guard} cites {filename} as its proof, but that file is not a PROOF row in this "
|
||||
"table. The citation and the classification must agree."
|
||||
)
|
||||
|
||||
|
||||
def test_the_summary_counts_match_the_table():
|
||||
"""The prose is DERIVED-checked, not hand-maintained.
|
||||
|
||||
It shipped wrong the first time — "28 guards, 4 tooling … 19 have none" against a table holding
|
||||
27/5/…/18 — because it was a mirror with no equality check. Both cold reviewers found it
|
||||
independently, which is the clearest possible evidence that a summary nobody checks is a summary
|
||||
nobody can trust.
|
||||
"""
|
||||
rows = inventory_rows()
|
||||
kinds = Counter(k for _, k, _, _ in rows)
|
||||
grades = Counter(p for _, k, p, _ in rows if k == "GUARD")
|
||||
|
||||
m = _SUMMARY.search(INVENTORY.read_text())
|
||||
assert m, (
|
||||
"could not find the summary sentence in the expected shape. It must read exactly like: "
|
||||
"`N guards, N tooling scripts, N proof files. **N guards carry a mutation proof; N are "
|
||||
"behaviour-only; N have none.**` — if you reword it, update `_SUMMARY` in the same commit, "
|
||||
"because an unparsed summary is an unchecked one."
|
||||
)
|
||||
claimed = tuple(int(g) for g in m.groups())
|
||||
actual = (
|
||||
kinds["GUARD"],
|
||||
kinds["TOOLING"],
|
||||
kinds["PROOF"],
|
||||
grades["MUTATION"],
|
||||
grades["BEHAVIOUR-ONLY"],
|
||||
grades["NONE"],
|
||||
)
|
||||
assert claimed == actual, (
|
||||
f"the summary claims (guards, tooling, proofs, mutation, behaviour-only, none) = {claimed} "
|
||||
f"but the table holds {actual}."
|
||||
)
|
||||
|
||||
|
||||
def test_tooling_rows_never_claim_a_proof():
|
||||
"""A TOOLING row asserting nothing cannot have a proof that it can go red, and letting one
|
||||
carry a ref would quietly inflate the coverage count at the bottom of the inventory."""
|
||||
for guard, kind, proof, _ref in inventory_rows():
|
||||
if kind == "TOOLING":
|
||||
assert proof == "NONE", f"{guard} is TOOLING but claims Proof {proof}"
|
||||
@@ -0,0 +1,604 @@
|
||||
"""No file population in this repo admits a file git does not track (ersatztv#806).
|
||||
|
||||
The regression for #778's third defect, hoisted to cover every guard that shares the mechanism
|
||||
rather than being copied into each of them. `Path.rglob` enumerated `.husky/_/` — 17 husky shims
|
||||
generated by `npm ci`, gitignored and untracked — so `test_remote_state_inventory.py` was RED on
|
||||
every developer checkout and GREEN in CI, whose `script-tests` job checks out and pip-installs but
|
||||
never runs `npm ci`. A guard that fails everywhere except where it runs trains its readers to ignore
|
||||
it, and it did so on the artifact whose entire thesis is population correctness.
|
||||
|
||||
TWO PROOFS, because they fail differently and either alone leaves a hole.
|
||||
|
||||
* `test_the_primitive_REALLY_excludes_an_untracked_file` builds a throwaway git repo, commits one
|
||||
file, leaves an identical sibling untracked, and runs the real derivation against it. It proves
|
||||
the mechanism by EXECUTING it rather than by recognising its shape — no monkeypatching, no
|
||||
stand-in for git. Nothing here is a claim about `git ls-files`; it is `git ls-files`.
|
||||
* `test_no_derivation_admits_an_untracked_file` narrows the tracked set under each real derivation
|
||||
and requires the dropped member to vanish from the population even though the file is still on
|
||||
disk and still matches the scope. That is the property stated over the ACTUAL guards, so a
|
||||
future refactor that quietly reintroduces a filesystem walk in any one of them fails here rather
|
||||
than on somebody's laptop.
|
||||
|
||||
`DERIVATIONS` is the reason this file is not one near-copy per derivation: a guard that starts deriving a file
|
||||
population registers here, and both proofs cover it for free. The register is hand-written and that
|
||||
is a SCOPE decision, not a population one — per `testing.guard-derives-population-from-source`, a
|
||||
scope mirroring an authoritative source needs its own equality check, and
|
||||
`test_every_index_derived_module_is_registered` is it: it reads which modules import the shared
|
||||
helper and demands each one appear below, so adding another derivation and forgetting this file is
|
||||
red rather than silently uncovered.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import ast
|
||||
import glob
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from scripts.tests import test_ci_image_pin_population as image_pin
|
||||
from scripts.tests import test_guard_inventory as guard_inventory
|
||||
from scripts.tests import test_hook_fire_log as hook_fire
|
||||
from scripts.tests import test_pr_changed_files as pr_changed
|
||||
from scripts.tests import test_remote_state_inventory as remote_state
|
||||
from scripts.tests import tracked_files
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[2]
|
||||
TESTS_DIR = REPO_ROOT / "scripts" / "tests"
|
||||
|
||||
|
||||
class _EmptyScan:
|
||||
"""An exhausted ITERATOR that is also a context manager, standing in for `os.scandir`.
|
||||
|
||||
Both halves are load-bearing and each was missing in turn. `os.scandir` is used as
|
||||
`with os.scandir(...) as it`, so a bare iterator broke the context-manager protocol; and
|
||||
`os.walk` does `entry = next(scandir_it)` on the result, so an ITERABLE defining only
|
||||
`__iter__` broke that. Either way the enumeration assertion still fired with the right message,
|
||||
but the report also carried a TypeError about the harness — and a finding that arrives beside a
|
||||
harness error invites doubting the finding rather than the code.
|
||||
"""
|
||||
|
||||
def __iter__(self):
|
||||
return self
|
||||
|
||||
def __next__(self):
|
||||
raise StopIteration
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, *_exc):
|
||||
return False
|
||||
|
||||
def close(self):
|
||||
return None
|
||||
|
||||
|
||||
def _as_relative_strings(members) -> set[str]:
|
||||
"""Derivations return either repo-relative strings or absolute `Path`s; compare on one form.
|
||||
|
||||
Iterating into a set is also what DRAINS a derivation that returns a generator, which the
|
||||
enumeration proof depends on — see its call site.
|
||||
|
||||
An unexpected member type is REPORTED rather than stringified. `str(m)` on anything at all meant
|
||||
a derivation yielding, say, nested generators produced plausible-looking members and compared
|
||||
equal to nothing, which is a population check passing over data it did not understand.
|
||||
"""
|
||||
out = set()
|
||||
for m in members:
|
||||
assert isinstance(m, str | Path), (
|
||||
f"a derivation yielded {type(m).__name__} ({m!r}); populations here are repo-relative "
|
||||
"strings or absolute Paths, and stringifying anything else would compare a plausible "
|
||||
"value against a set that can never contain it."
|
||||
)
|
||||
out.add(str(Path(m).relative_to(REPO_ROOT)) if isinstance(m, Path) else str(m))
|
||||
return out
|
||||
|
||||
|
||||
# (label, callable, a floor below which the derivation has plainly broken). `test_hook_fire_log`
|
||||
# floors the same population at the same number for its own coverage assertions; that is not a
|
||||
# duplicate guard masking another, because the two protect different consumers from going
|
||||
# vacuous — delete this one and THIS file's proofs iterate over nothing while reporting success.
|
||||
DERIVATIONS = (
|
||||
("test_guard_inventory.derived_guard_files", guard_inventory.derived_guard_files, 25),
|
||||
("test_hook_fire_log.hook_scripts", hook_fire.hook_scripts, 10),
|
||||
("test_ci_image_pin_population.workflow_files", image_pin.workflow_files, 5),
|
||||
("test_remote_state_inventory.derived_population", remote_state.derived_population, 40),
|
||||
("test_pr_changed_files._workflow_files", pr_changed._workflow_files, 5),
|
||||
)
|
||||
|
||||
# The floor matters only to the anti-vacuity test; the two property tests take the pair, so an
|
||||
# unused parameter cannot drift into looking like an assertion they make.
|
||||
_IDS = [d[0] for d in DERIVATIONS]
|
||||
_PAIRS = [(label, derive) for label, derive, _ in DERIVATIONS]
|
||||
|
||||
# Modules that import the shared derivation WITHOUT deriving a guard population. Kept here, beside
|
||||
# DERIVATIONS, so adding one is an edit to this file that a reviewer sees.
|
||||
POPULATION_EXEMPT = {
|
||||
# Uses the index to assemble a HERMETIC tmp fixture copy; nothing in it asserts membership.
|
||||
"test_ci_release_path_scan_job.py": "index-derived fixture copy, not a population",
|
||||
}
|
||||
|
||||
_HELPER = "scripts.tests.tracked_files"
|
||||
_PACKAGE = ["scripts", "tests"]
|
||||
|
||||
|
||||
# ------------------------------------------------------------------------------------------------
|
||||
# ANTI-VACUITY FIRST — every assertion below compares sets, and a derivation that collapsed to
|
||||
# nothing would satisfy all of them while proving nothing.
|
||||
# ------------------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("label", "derive", "floor"), DERIVATIONS, ids=_IDS)
|
||||
def test_each_derivation_found_something(label, derive, floor):
|
||||
members = _as_relative_strings(derive())
|
||||
assert len(members) >= floor, (
|
||||
f"{label} derived only {len(members)} members, below its floor of {floor} — the derivation "
|
||||
"is broken, not the repo, and every set comparison built on it is vacuous."
|
||||
)
|
||||
|
||||
|
||||
# ------------------------------------------------------------------------------------------------
|
||||
# PROOF 1 — the primitive, executed against a real git repo rather than described
|
||||
# ------------------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_the_primitive_REALLY_excludes_an_untracked_file(tmp_path, monkeypatch):
|
||||
"""A tracked and an untracked file, identical in name shape and both on disk. Only one is in.
|
||||
|
||||
Run rather than reasoned about. This class of defect is produced by arguments about what a
|
||||
traversal WOULD enumerate, and such arguments are locally convincing whether or not they are
|
||||
right; only executing the traversal distinguishes the two.
|
||||
"""
|
||||
repo = tmp_path / "repo"
|
||||
(repo / ".claude" / "hooks").mkdir(parents=True)
|
||||
(repo / ".claude" / "hooks" / "committed.sh").write_text("#!/bin/sh\n")
|
||||
(repo / ".claude" / "hooks" / "untracked.sh").write_text("#!/bin/sh\n")
|
||||
|
||||
def git(*args):
|
||||
subprocess.run(["git", "-C", str(repo), *args], check=True, capture_output=True)
|
||||
|
||||
git("init", "-q")
|
||||
git("config", "user.email", "guard@example.invalid")
|
||||
git("config", "user.name", "guard")
|
||||
git("add", ".claude/hooks/committed.sh")
|
||||
git("commit", "-qm", "one tracked hook")
|
||||
|
||||
monkeypatch.setattr(tracked_files, "REPO_ROOT", repo)
|
||||
found = tracked_files.tracked_children(".claude/hooks", ("*.sh",))
|
||||
|
||||
assert (repo / ".claude" / "hooks" / "untracked.sh").is_file(), (
|
||||
"the untracked file must still be on disk, or this proves nothing about the index winning over the filesystem"
|
||||
)
|
||||
assert found == {".claude/hooks/committed.sh"}, (
|
||||
f"the derivation returned {sorted(found)}. A filesystem walk returns both files here; only "
|
||||
"the index distinguishes them, and that difference is the entire point of ersatztv#806."
|
||||
)
|
||||
|
||||
|
||||
def test_the_primitive_does_not_recurse_into_an_untracked_subdirectory(tmp_path, monkeypatch):
|
||||
"""`.husky/_/` in miniature — the shape that made #778 red on every checkout.
|
||||
|
||||
Even a TRACKED nested file must stay out: `tracked_children` is direct-children-only by design,
|
||||
and recursion is what dragged the shims in. Proving it with a tracked file makes the assertion
|
||||
about the traversal rather than about the index, so the two properties cannot mask each other.
|
||||
"""
|
||||
repo = tmp_path / "repo"
|
||||
(repo / ".husky" / "_").mkdir(parents=True)
|
||||
(repo / ".husky" / "pre-commit").write_text("#!/bin/sh\n")
|
||||
(repo / ".husky" / "_" / "husky.sh").write_text("#!/bin/sh\n")
|
||||
|
||||
def git(*args):
|
||||
subprocess.run(["git", "-C", str(repo), *args], check=True, capture_output=True)
|
||||
|
||||
git("init", "-q")
|
||||
git("config", "user.email", "guard@example.invalid")
|
||||
git("config", "user.name", "guard")
|
||||
git("add", "-A")
|
||||
git("commit", "-qm", "husky plus a nested shim, both tracked")
|
||||
|
||||
monkeypatch.setattr(tracked_files, "REPO_ROOT", repo)
|
||||
assert tracked_files.tracked_children(".husky", ("*",)) == {".husky/pre-commit"}, (
|
||||
"a nested file entered a flat population — this is the `.husky/_/` shape, and it was red on "
|
||||
"every developer checkout the last time it shipped"
|
||||
)
|
||||
|
||||
|
||||
def test_an_empty_index_FAILS_LOUDLY_rather_than_reporting_an_empty_population(tmp_path, monkeypatch):
|
||||
"""The floor under every floor. A silent empty population is how a completeness guard reports
|
||||
total coverage having examined nothing, which is the failure mode this repo has shipped twice
|
||||
(#631, #751)."""
|
||||
repo = tmp_path / "repo"
|
||||
repo.mkdir()
|
||||
subprocess.run(["git", "-C", str(repo), "init", "-q"], check=True, capture_output=True)
|
||||
|
||||
monkeypatch.setattr(tracked_files, "REPO_ROOT", repo)
|
||||
with pytest.raises(AssertionError, match="reported nothing"):
|
||||
tracked_files.tracked_children(".claude/hooks", ("*.sh",))
|
||||
|
||||
|
||||
# ------------------------------------------------------------------------------------------------
|
||||
# PROOF 2 — the property, over the real guards
|
||||
# ------------------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("label", "derive"), _PAIRS, ids=_IDS)
|
||||
def test_no_derivation_admits_an_untracked_file(label, derive):
|
||||
"""Narrow the index, leave the disk alone, and require the member to disappear — for EVERY
|
||||
member, one at a time.
|
||||
|
||||
EXHAUSTIVE RATHER THAN ONE VICTIM, and the difference is not thoroughness for its own sake.
|
||||
`derived_guard_files` unions four sources; one victim is always drawn from whichever sorts
|
||||
first, so a mutant putting only the third source back on a filesystem walk passes while the
|
||||
proof reports on all four. A sample cannot see the source it did not draw from — this file
|
||||
applying `testing.guard-derives-population-from-source` to itself.
|
||||
|
||||
The floors cannot substitute, and the numbers say why. Reproduce with:
|
||||
|
||||
PYTHONPATH=. python3 -c "from scripts.tests import test_guard_inventory as g; \
|
||||
print(len(g.derived_guard_files()))"
|
||||
|
||||
61 members on 2026-08-22; suppressing a single contributor leaves 39 (`scripts/tests`), 48
|
||||
(hooks) or 58 (husky), all far above the anti-vacuity floor of 25. The figures move whenever a
|
||||
guard is added — they were 60/39/47/57 one commit earlier — so read them as an illustration of
|
||||
the GAP, not as values to assert against. A floor tight enough to catch a lost source
|
||||
would go red every time a guard is legitimately deleted, which is the wrong instrument.
|
||||
|
||||
Its own `monkeypatch` context, never the shared fixture instance: the function-scoped fixture is
|
||||
the same object `conftest.py`'s autouse `isolate_hook_fire_log` patched, so calling `undo()` on
|
||||
it here also unsets `ETV_HOOK_FIRE_LOG_DIR` and silently re-points a later hook-driving test at
|
||||
the PRODUCTION log — the #776 isolation disarmed from inside the file that argues for proofs.
|
||||
|
||||
WHAT REMOVAL CANNOT SEE, so it is not read as more than it is: a source contributing ONLY
|
||||
untracked members has nothing here to remove, and #778's defect was exactly that shape (an
|
||||
`rglob` over `.husky/_/` adds 17 untracked members and removes none). That direction is
|
||||
`test_no_derivation_ENUMERATES_the_filesystem` below; the two are complements, not duplicates.
|
||||
"""
|
||||
before = _as_relative_strings(derive())
|
||||
assert before, f"{label} derived nothing; there is no victim to remove"
|
||||
|
||||
real = tracked_files._git_ls_files()
|
||||
survivors = []
|
||||
absent = []
|
||||
with pytest.MonkeyPatch.context() as m:
|
||||
for victim in sorted(before):
|
||||
if not (REPO_ROOT / victim).is_file():
|
||||
absent.append(victim)
|
||||
continue
|
||||
m.setattr(tracked_files, "_git_ls_files", lambda v=victim: [p for p in real if p != v])
|
||||
if victim in _as_relative_strings(derive()):
|
||||
survivors.append(victim)
|
||||
|
||||
assert not absent, (
|
||||
f"{label} contains {absent}, which git tracks but are not on disk. The proof below asserts "
|
||||
"that the INDEX decides while the file is still present; it cannot mean that for a member "
|
||||
"that is missing, so this is reported rather than skipped."
|
||||
)
|
||||
assert not survivors, (
|
||||
f"{label} still contains {survivors} after git stopped tracking them. Every one of those is "
|
||||
"still on disk, so the derivation is reading the filesystem for that member and untracked "
|
||||
"build output can redden it on a developer checkout while CI stays green (ersatztv#778)."
|
||||
)
|
||||
|
||||
|
||||
# The directory-listing APIs a Python file population is realistically written with. NOT every way a
|
||||
# process can list a directory — `subprocess.run(["ls"])`, a module-level alias captured before the
|
||||
# patch, and any C-level call all walk straight past this, all three verified by cold review. That
|
||||
# bounds what the check below can claim, and the docstring says so rather than implying a sandbox.
|
||||
# Reading a file stays allowed: `derived_guard_files` must read workflow bodies.
|
||||
_ENUMERATORS = (
|
||||
(Path, "glob"),
|
||||
(Path, "rglob"),
|
||||
(Path, "iterdir"),
|
||||
(Path, "walk"),
|
||||
(os, "listdir"),
|
||||
(os, "walk"),
|
||||
(os, "scandir"),
|
||||
(glob, "glob"),
|
||||
(glob, "iglob"),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("label", "derive"), _PAIRS, ids=_IDS)
|
||||
def test_no_derivation_ENUMERATES_the_filesystem(label, derive):
|
||||
"""The ADD direction, without arranging any state: a derivation may READ files, but it may not
|
||||
LIST a directory while it runs.
|
||||
|
||||
WHY THE ADD DIRECTION NEEDS ITS OWN TEST. Removing members from the index cannot see a source
|
||||
that contributes ONLY untracked members: it adds and never takes away, so nothing of its is
|
||||
available to remove. `.husky/_/` is that shape exactly — 17 shims `npm ci` writes, none ever in
|
||||
the index — and it is the shape #778 shipped. Scope the claim precisely: a source appending
|
||||
`(REPO_ROOT / ".husky" / "_").rglob("*")` leaves the removal proof green ON A MACHINE WHERE THAT
|
||||
DIRECTORY IS ABSENT, which is the `script-tests` checkout. Where the shims exist, the appended
|
||||
members are present and removal reddens too. The blind spot is an append-only source that yields
|
||||
nothing HERE — which is exactly the CI shape, and exactly where a guard going quiet matters.
|
||||
|
||||
WHY THE PROPERTY IS "DOES NOT ENUMERATE" RATHER THAN "DOES NOT RETURN AN UNTRACKED FILE". Both
|
||||
obvious formulations of the latter are machine-dependent, which is the very fault #806 exists to
|
||||
remove:
|
||||
|
||||
* creating real probe files in the checkout and requiring they not enter. That needs a
|
||||
`test_*.py` probe to reach the `scripts/tests` scope — a file pytest COLLECTS mid-session —
|
||||
and its parametrised names collide across `-n auto` workers and concurrent sessions,
|
||||
`finally` does not survive SIGKILL, and a concurrent `git add -A` can stage one. Defects in
|
||||
the test, not in the thing tested.
|
||||
* neutralising the shared primitive and requiring the derivation to go empty. That misses the
|
||||
`.husky/_/` source on any machine where `.husky/_/` does not exist — which is every CI
|
||||
checkout of `script-tests`, which never runs `npm ci`. Green where it runs, red only on a
|
||||
laptop: the inverted asymmetry again, inside the proof written to abolish it.
|
||||
|
||||
Watching for the CALL needs no arranged state: an `rglob` issued while deriving is caught even
|
||||
where the directory it walks is empty, because the evidence is the call rather than what it
|
||||
returned. Reading is untouched, so a derivation may still parse the workflow bodies it scrapes
|
||||
for referenced scripts.
|
||||
|
||||
WHAT IT DOES NOT COVER. The boundary is not "synchronous", which is what two earlier drafts
|
||||
said and what measurement disproved — a thread that outlives the `derive()` call but finishes
|
||||
while its result is being drained IS caught, as is a `__del__` firing during that drain. The
|
||||
boundary is mechanical rather than temporal: a call to one of the SPIES is observed, wherever and
|
||||
whenever it happens in this process before the assertion below. "While the patch is active"
|
||||
under-claims it — the spy appends to a list that outlives the patch, so a reference captured
|
||||
during the window and invoked after it still records. What decides observation is whether the
|
||||
call goes through a spy, not when. The result is drained while the patch is installed, so a lazy
|
||||
generator is reached.
|
||||
|
||||
NOT REACHED, because no spy was ever installed on that path. Enumeration HOISTED TO MODULE SCOPE
|
||||
runs at import, before this test exists — the likeliest instance rather than a contrivance, since
|
||||
`test_ci_image_pin_population.py` already precomputes `_DOC` that way — as does an `atexit` hook,
|
||||
and a cached property warmed by the baseline call below. `from os import listdir` binds the real
|
||||
function before any patch; `from os import walk` IS caught (it routes through the patched
|
||||
`os.scandir`) and so is `from glob import glob` (through the patched `glob.iglob`). And anything
|
||||
listing in ANOTHER PROCESS — a deliberate `subprocess.run(["ls"])`, or a forked child. This is a
|
||||
regression guard against the shapes that arrive by accident, not a sandbox.
|
||||
|
||||
It cuts the other way too: any spy call at all reddens this test, so unrelated background thread
|
||||
activity touching a patched name during the window would too. Nothing in this suite does that
|
||||
today, and the report names the call, so a false red would be diagnosable rather than
|
||||
mysterious.
|
||||
|
||||
It also cannot see a derivation that admits a HARDCODED path without listing anything
|
||||
(`if (REPO_ROOT / "x.sh").exists(): add`) — listing is the commonest way to discover an untracked
|
||||
member, not the only one. The removal proof above catches that shape, and catches memoisation,
|
||||
which this one cannot: the baseline call below warms any cache outside the patch. The two are
|
||||
complements.
|
||||
"""
|
||||
assert derive(), f"{label} derived nothing; this proof needs a baseline"
|
||||
|
||||
calls: list[str] = []
|
||||
|
||||
def _spy(what):
|
||||
# RECORDS and returns empty rather than raising. Raising made the assertion "did an
|
||||
# exception reach us", which a derivation defeats by catching it: a `try: ... except
|
||||
# Exception: return set()` around an `rglob` enumerated the filesystem and this test passed,
|
||||
# measured. The evidence is the CALL, so the call is what is asserted on.
|
||||
def spy(*_args, **_kwargs):
|
||||
calls.append(what)
|
||||
# Iterator AND context manager: `os.scandir` is used as `with os.scandir(...) as it`,
|
||||
# and a bare iterator made the failure report carry a test-induced TypeError about the
|
||||
# context manager protocol alongside the real finding. The assertion fired correctly
|
||||
# either way, but a report that blames the harness invites doubting the finding.
|
||||
return _EmptyScan()
|
||||
|
||||
return spy
|
||||
|
||||
failure = None
|
||||
with pytest.MonkeyPatch.context() as m:
|
||||
for owner, name in _ENUMERATORS:
|
||||
m.setattr(owner, name, _spy(f"{getattr(owner, '__name__', owner)}.{name}"), raising=False)
|
||||
try:
|
||||
# DRAINED inside the context, never `derive()` discarded. A derivation returning a lazy
|
||||
# generator does its work when the caller drains it, so discarding the result moved the
|
||||
# whole walk outside the patch: a generator yielding the index population and then
|
||||
# appending `.husky/_` passed here and admitted 17 untracked shims on a checkout where
|
||||
# that directory exists. `_as_relative_strings` is what drains it — it iterates into a
|
||||
# set — so the call must stay here rather than being hoisted out or wrapped in something
|
||||
# lazier.
|
||||
_as_relative_strings(derive())
|
||||
except BaseException as exc: # re-raised below, after the evidence has been judged
|
||||
failure = exc
|
||||
|
||||
assert not calls, (
|
||||
f"{label} enumerated the filesystem via {sorted(set(calls))} while deriving its population. "
|
||||
"Directory listings report build output, generated shims and editor droppings, and differ "
|
||||
"between the CI checkout and a developer's, so the member set stops being a property of the "
|
||||
f"repo (ersatztv#778, #806)." + (f" It also raised: {failure!r}" if failure is not None else "")
|
||||
)
|
||||
if failure is not None:
|
||||
raise failure
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("label", "derive"), _PAIRS, ids=_IDS)
|
||||
def test_every_derived_member_is_tracked(label, derive):
|
||||
"""The same property as an invariant over the real tree, which is the form that catches a
|
||||
refactor going back to a filesystem walk without also touching this file."""
|
||||
tracked = set(tracked_files._git_ls_files())
|
||||
stray = sorted(m for m in _as_relative_strings(derive()) if m not in tracked)
|
||||
assert not stray, f"{label} contains untracked path(s): {stray}"
|
||||
|
||||
|
||||
# ------------------------------------------------------------------------------------------------
|
||||
# THE SCOPE MIRROR ABOVE IS ITSELF CHECKED
|
||||
# ------------------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _modules_importing_the_helper() -> set[str]:
|
||||
"""Which `scripts/tests/test_*.py` import the shared derivation, by PARSING them.
|
||||
|
||||
`ast` rather than a substring scan, and the distinction is the point rather than tidiness. A
|
||||
substring scan over source both EVADES and FALSELY FIRES here: `import scripts.tests.
|
||||
tracked_files as tf` and `from scripts.tests import tracked_files as tf` escape a scan for
|
||||
`"tracked_files import"`, while a comment merely citing `scripts/tests/tracked_files.py` matches
|
||||
a scan for `"tracked_files."` and would redden a correct tree over prose.
|
||||
`docs/decisions/records/testing/guard-derives-population-from-source.md` records why patching
|
||||
such a predicate does not converge: it is not a parser. Python ships the parser, and a comment
|
||||
is not a node at all.
|
||||
|
||||
WHAT THE PARSE DOES NOT REACH, stated rather than implied by the word "parsing": STATIC import
|
||||
statements naming the helper. `importlib.import_module("scripts.tests.tracked_files")`, a
|
||||
re-export through `scripts/tests/__init__.py`, and `from scripts.tests import *` are invisible,
|
||||
verified by executing each. (`from scripts.tests.tracked_files import *` IS seen — it names the
|
||||
module.) Those sit inside the same residual as a module that derives a population without the
|
||||
helper at all — the residual named below — and no mechanical check closes it.
|
||||
|
||||
The FILE LIST is a filesystem walk on purpose, and it is not the defect this file forbids: it is
|
||||
a superset check over what pytest itself collects, so an untracked stray `test_x.py` here makes
|
||||
the guard MORE demanding, never blind. Using the index would let an unstaged new guard escape
|
||||
registration, which is the wrong direction for a check about coverage.
|
||||
"""
|
||||
found = set()
|
||||
for path in sorted(TESTS_DIR.glob("test_*.py")):
|
||||
for node in ast.walk(ast.parse(path.read_text())):
|
||||
# RESOLVE the name to an absolute module and compare exactly. Testing `base ==
|
||||
# "tracked_files"` handled `from .tracked_files import x` but silently missed
|
||||
# `from ..tests.tracked_files import x`, which resolves to the same helper — a false
|
||||
# NEGATIVE, the direction that lets a module adopt the helper and escape registration.
|
||||
# A suffix match instead over-accepts `from unrelated.package import tracked_files`,
|
||||
# reddening a correct tree over a module this repo does not own. Resolution is the only
|
||||
# form with neither failure: these files live in `scripts.tests`, so level 1 resolves to
|
||||
# `scripts.tests` and level 2 to `scripts` — see the guard below for why there is no
|
||||
# level 3.
|
||||
if isinstance(node, ast.Import):
|
||||
hit = any(a.name == _HELPER for a in node.names)
|
||||
elif isinstance(node, ast.ImportFrom):
|
||||
if node.level > len(_PACKAGE):
|
||||
# Beyond the top-level package: Python raises ImportError for this, so it cannot
|
||||
# be an import of the helper. Guarded explicitly because `_PACKAGE[:negative]`
|
||||
# silently WRAPS — level 4 produced the same prefix as level 2 — which pinned an
|
||||
# unimportable form as a valid detection. Valid levels here are exactly 1 and 2:
|
||||
# 1 resolves to `scripts.tests`, 2 to `scripts`, and 3 or more is beyond the
|
||||
# top-level package, which Python refuses.
|
||||
continue
|
||||
prefix = _PACKAGE[: len(_PACKAGE) - (node.level - 1)] if node.level else []
|
||||
base_parts = prefix + ([node.module] if node.module else [])
|
||||
base = ".".join(base_parts)
|
||||
hit = base == _HELPER or any(f"{base}.{a.name}" == _HELPER for a in node.names)
|
||||
else:
|
||||
continue
|
||||
if hit:
|
||||
found.add(path.name)
|
||||
break
|
||||
return found
|
||||
|
||||
|
||||
# (source, should the matcher see it). Every row is a form that has actually been mis-classified; the
|
||||
# table is here so the next edit to `_modules_importing_the_helper` cannot re-open one silently.
|
||||
# A false NEGATIVE lets a module adopt the helper and escape registration; a false POSITIVE reddens
|
||||
# a correct tree over a module this repo does not own. Both directions are pinned.
|
||||
_IMPORT_FORMS = (
|
||||
("import scripts.tests.tracked_files as tf", True),
|
||||
("from scripts.tests import tracked_files as tf", True),
|
||||
("from scripts.tests.tracked_files import tracked_paths", True),
|
||||
("from scripts.tests.tracked_files import *", True),
|
||||
("from . import tracked_files", True),
|
||||
("from .tracked_files import tracked_paths", True),
|
||||
("from ..tests.tracked_files import tracked_paths", True),
|
||||
# Beyond the top-level package from `scripts.tests`: Python raises ImportError, so there is
|
||||
# nothing to detect. Pinned False so the negative-slicing wrap that once made it look
|
||||
# detectable cannot come back.
|
||||
("from ...scripts.tests.tracked_files import tracked_paths", False),
|
||||
# Witnesses the negative-slice wrap specifically: without the level guard this one
|
||||
# resolves through `scripts` and matches.
|
||||
("from ....tests.tracked_files import tracked_paths", False),
|
||||
("def f():\n from scripts.tests import tracked_files\n return tracked_files", True),
|
||||
("from unrelated.package import tracked_files", False),
|
||||
("from ..something import tracked_files", False),
|
||||
("from .something import tracked_files", False),
|
||||
("from ..other.tracked_files import x", False),
|
||||
("# see scripts/tests/tracked_files.py for the rationale\nimport re", False),
|
||||
# Pins PARSER VISIBILITY, not importability: executing this line really does import the helper.
|
||||
# The row records that a static parse cannot see it — a known gap, pinned so it is not a
|
||||
# surprise — and closing it would mean updating this row, which is the intended friction.
|
||||
("import importlib\nimportlib.import_module('scripts.tests.tracked_files')", False),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("source", "expected"), _IMPORT_FORMS, ids=[s.splitlines()[0][:48] for s, _ in _IMPORT_FORMS])
|
||||
def test_the_import_matcher_classifies_every_reviewed_form(source, expected, monkeypatch):
|
||||
class _Fake:
|
||||
name = "test_probe.py"
|
||||
|
||||
def read_text(self):
|
||||
return source
|
||||
|
||||
def __lt__(self, other):
|
||||
return True
|
||||
|
||||
class _Dir:
|
||||
def glob(self, _pattern):
|
||||
return [_Fake()]
|
||||
|
||||
monkeypatch.setattr(sys.modules[__name__], "TESTS_DIR", _Dir())
|
||||
seen = "test_probe.py" in _modules_importing_the_helper()
|
||||
assert seen is expected, f"the import matcher {'missed' if expected else 'falsely matched'} this form:\n{source}"
|
||||
|
||||
|
||||
def test_every_index_derived_module_is_registered():
|
||||
"""`DERIVATIONS` is a hand-written mirror, so it gets an equality check rather than a promise.
|
||||
|
||||
Without this, adding another index-derived guard and forgetting to register it leaves that guard
|
||||
unproven while this file reads as covering them all — a completeness claim standing behind a
|
||||
hand-maintained list, which is the defect one altitude up (#773 Family C, and the shape that put
|
||||
`MARKED_JOBS` in the record as a residual gap).
|
||||
|
||||
THE SCOPE THIS CANNOT SEE, stated because a check described as complete stops being re-examined:
|
||||
it detects modules that IMPORT the shared helper. A module deriving a file population some other
|
||||
way — shelling out to `git ls-files` itself, or going back to `Path.rglob` — is invisible to it,
|
||||
and no mechanical check can close that. `test_guard_inventory.py`'s own header argues the same
|
||||
point about flagging filter-shaped guards by token, and #774 concluded there that the honest
|
||||
answer is no. What is mechanised here is the case that actually recurs: someone adopts the
|
||||
helper and forgets this file.
|
||||
|
||||
Registration is MODULE-level, not derivation-level, so a second population added inside an
|
||||
already-registered module is covered only if it is registered too.
|
||||
|
||||
`POPULATION_EXEMPT` is the opt-out, and it lives HERE rather than as a marker comment in the
|
||||
exempt file because the two directions are not symmetric: a false import-match only reddens,
|
||||
while a false EXEMPTION is silent. A marker a file grants itself by containing a token is
|
||||
trippable from that file's prose — this file's own error message names the token — so it would
|
||||
be a one-line silent kill switch, the shape `test_ci_release_path_scan_job.py` argues against
|
||||
for its recursion fence. Listing exemptions beside the registrations makes adding one a visible
|
||||
edit here.
|
||||
|
||||
WHAT NO ASSERTION CAN DECIDE, dated so it is re-examined rather than assumed: whether an
|
||||
exemption is still WARRANTED. A stale key and an unexplained one are both caught below, but an
|
||||
exempt module that later grows a real derived population stays uncovered and silent. Reviewed
|
||||
2026-08-22 — the single entry uses the helper only to assemble a tmp fixture copy and asserts
|
||||
nothing about membership.
|
||||
"""
|
||||
exempt = {Path(__file__).name} | set(POPULATION_EXEMPT)
|
||||
importers = _modules_importing_the_helper()
|
||||
registered = {label.split(".", 1)[0] + ".py" for label, _, _ in DERIVATIONS}
|
||||
|
||||
# ANTI-VACUITY, and only that. A broken parse is caught loudly by `phantom` below — every
|
||||
# registered module would go missing at once — so this is the cheaper, more specific signal, not
|
||||
# the thing standing between a broken parse and a green run.
|
||||
assert len(importers) >= len(DERIVATIONS), (
|
||||
f"the import parse found only {sorted(importers)}, fewer modules than DERIVATIONS registers "
|
||||
f"({sorted(registered)}) — the parse has broken."
|
||||
)
|
||||
|
||||
stale = sorted(name for name in POPULATION_EXEMPT if name not in importers)
|
||||
assert not stale, (
|
||||
f"POPULATION_EXEMPT lists {stale}, which the parser no longer sees importing the shared "
|
||||
"helper (renamed, "
|
||||
"deleted, or the import removed). A stale exemption is worse than none: if the filename is "
|
||||
"ever reused, the new module is exempt from birth without anyone deciding that."
|
||||
)
|
||||
thin = sorted(name for name, why in POPULATION_EXEMPT.items() if not str(why).strip())
|
||||
assert not thin, f"POPULATION_EXEMPT entries with no stated reason: {thin}"
|
||||
|
||||
unregistered = sorted(importers - registered - exempt)
|
||||
assert not unregistered, (
|
||||
f"{unregistered} import the shared index derivation but are not in DERIVATIONS, so neither "
|
||||
"proof in this file covers them. Add a named derivation function and register it, or add "
|
||||
"the module to POPULATION_EXEMPT in this file if it imports the helper without deriving a "
|
||||
"population."
|
||||
)
|
||||
phantom = sorted(registered - importers)
|
||||
assert not phantom, (
|
||||
f"DERIVATIONS registers {phantom}, which no longer import the shared helper. A row for "
|
||||
"a derivation that is not there reads as coverage and is not."
|
||||
)
|
||||
File diff suppressed because it is too large
Load Diff
@@ -51,12 +51,12 @@ def preflight(tmp_path):
|
||||
so every test passed.
|
||||
"""
|
||||
shim = bindir / "jq"
|
||||
body = "#!/bin/sh\nif [ \"$1\" = \"--version\" ]; then\n"
|
||||
body = '#!/bin/sh\nif [ "$1" = "--version" ]; then\n'
|
||||
if version_line:
|
||||
body += ' printf "%%s\\n" %s\n' % _shq(version_line)
|
||||
body += f' printf "%s\\n" {_shq(version_line)}\n'
|
||||
if stderr:
|
||||
body += ' printf "%%s\\n" %s >&2\n' % _shq(stderr)
|
||||
body += " exit %d\nfi\nexit 0\n" % exit_code
|
||||
body += f' printf "%s\\n" {_shq(stderr)} >&2\n'
|
||||
body += f" exit {exit_code}\nfi\nexit 0\n"
|
||||
shim.write_text(body)
|
||||
shim.chmod(0o755)
|
||||
|
||||
@@ -73,8 +73,7 @@ def preflight(tmp_path):
|
||||
# but jq itself (`command -v` is a builtin, and bash is invoked by absolute path), so
|
||||
# there is nothing to keep.
|
||||
env["PATH"] = str(bindir)
|
||||
return subprocess.run([BASH, str(SCRIPT), *args],
|
||||
env=env, capture_output=True, text=True)
|
||||
return subprocess.run([BASH, str(SCRIPT), *args], env=env, capture_output=True, text=True)
|
||||
|
||||
def run_bytes(self, *args):
|
||||
"""Same, but WITHOUT text mode.
|
||||
@@ -87,8 +86,7 @@ def preflight(tmp_path):
|
||||
"""
|
||||
env = dict(os.environ)
|
||||
env["PATH"] = str(bindir)
|
||||
return subprocess.run([BASH, str(SCRIPT), *args],
|
||||
env=env, capture_output=True)
|
||||
return subprocess.run([BASH, str(SCRIPT), *args], env=env, capture_output=True)
|
||||
|
||||
return Handle()
|
||||
|
||||
@@ -172,11 +170,15 @@ def test_expect_without_a_value_is_a_usage_error_WITH_output(preflight):
|
||||
# the script exited 0 having asserted NOTHING. That is this script's own stated failure mode,
|
||||
# reproduced inside itself, which is why these cases are pinned rather than left to inspection.
|
||||
|
||||
@pytest.mark.parametrize("version_line", [
|
||||
"jq version 1.6", # some distro wrappers print this form
|
||||
"JQ-1.6",
|
||||
"jq-1.6-dirty",
|
||||
])
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"version_line",
|
||||
[
|
||||
"jq version 1.6", # some distro wrappers print this form
|
||||
"JQ-1.6",
|
||||
"jq-1.6-dirty",
|
||||
],
|
||||
)
|
||||
def test_unusual_but_parseable_version_forms_are_accepted(preflight, version_line):
|
||||
preflight.with_jq(version_line)
|
||||
r = preflight.run()
|
||||
@@ -190,7 +192,8 @@ def test_unparseable_version_fails_CLOSED_rather_than_asserting_nothing(prefligh
|
||||
r = preflight.run()
|
||||
assert r.returncode == 1, (
|
||||
f"{version_line!r} exited {r.returncode}: an unparsed version must never reach — or "
|
||||
"silently skip — the floor assertion")
|
||||
"silently skip — the floor assertion"
|
||||
)
|
||||
assert "could not parse" in r.stderr
|
||||
|
||||
|
||||
@@ -202,20 +205,21 @@ def test_a_jq_that_cannot_START_fails_closed(preflight):
|
||||
discarding the exit status, so that message became the parse input, `2.34` matched, and the floor
|
||||
was certified green on a jq that cannot run at all.
|
||||
"""
|
||||
preflight.with_jq(
|
||||
"", stderr="jq: /lib/x86_64-linux-gnu/libc.so.6: version `GLIBC_2.34' not found",
|
||||
exit_code=127)
|
||||
preflight.with_jq("", stderr="jq: /lib/x86_64-linux-gnu/libc.so.6: version `GLIBC_2.34' not found", exit_code=127)
|
||||
r = preflight.run()
|
||||
assert r.returncode == 1
|
||||
assert "cannot run" in r.stderr
|
||||
assert "parsed 2.34" not in r.stdout, "stderr must never be parsed as a version"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("version_line", [
|
||||
"warning: something 3.14", # a noise line carrying a plausible number
|
||||
"2026.07.26 jq-1.6", # a date prefix, which outranks the real version if unanchored
|
||||
"jq-master-v0.0.0-1.6",
|
||||
])
|
||||
@pytest.mark.parametrize(
|
||||
"version_line",
|
||||
[
|
||||
"warning: something 3.14", # a noise line carrying a plausible number
|
||||
"2026.07.26 jq-1.6", # a date prefix, which outranks the real version if unanchored
|
||||
"jq-master-v0.0.0-1.6",
|
||||
],
|
||||
)
|
||||
def test_a_number_that_is_not_the_VERSION_is_not_accepted_as_one(preflight, version_line):
|
||||
"""Matching the first `<digits>.<digits>` ANYWHERE let a prefix win over the real version.
|
||||
`2026.07.26 jq-1.6` parsed as 2026.07 and sailed over the floor. The pattern is anchored to the
|
||||
@@ -226,10 +230,13 @@ def test_a_number_that_is_not_the_VERSION_is_not_accepted_as_one(preflight, vers
|
||||
assert "could not parse" in r.stderr
|
||||
|
||||
|
||||
@pytest.mark.parametrize("version_line", [
|
||||
"jq-99999999999999999999999.0",
|
||||
"jq-1.99999999999999999999999",
|
||||
])
|
||||
@pytest.mark.parametrize(
|
||||
"version_line",
|
||||
[
|
||||
"jq-99999999999999999999999.0",
|
||||
"jq-1.99999999999999999999999",
|
||||
],
|
||||
)
|
||||
def test_an_OUT_OF_RANGE_digit_run_fails_closed(preflight, version_line):
|
||||
"""The round-1 fail-open mechanism, resurrected via an over-long number.
|
||||
|
||||
@@ -247,25 +254,28 @@ def test_an_OUT_OF_RANGE_digit_run_fails_closed(preflight, version_line):
|
||||
assert r.returncode == 1, f"{version_line!r} exited 0 — the floor was not asserted"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("version_line", [
|
||||
# Killed by the SEPARATOR restriction (a blank separator must be followed by `version`).
|
||||
"jq\n2.34: cannot load shared library",
|
||||
"jq\n\n\n99.9",
|
||||
"jq -- 2.34 (real jq-1.6)",
|
||||
"jq\t\t9.9",
|
||||
# Killed ONLY by the first-line slice + `[[:blank:]]`. These carry the literal word `version`,
|
||||
# so the separator restriction is satisfied and cannot save us — the newline must be excluded
|
||||
# from the separator class AND the parse confined to line one.
|
||||
#
|
||||
# Without these, a round-5 mutation check found that reverting BOTH of those changes together
|
||||
# (`[[:blank:]]`→`[[:space:]]` and parsing `$raw` instead of `$first`) left the whole suite
|
||||
# GREEN: the four cases above are all killed by the separator alone, so they attributed the fix
|
||||
# to the wrong layer. A test that passes for the wrong reason is how the previous three rounds
|
||||
# each shipped a defect.
|
||||
"jq\nversion\n9.9",
|
||||
"jq\nversion 9.9",
|
||||
"jq \n version \n 9.9",
|
||||
])
|
||||
@pytest.mark.parametrize(
|
||||
"version_line",
|
||||
[
|
||||
# Killed by the SEPARATOR restriction (a blank separator must be followed by `version`).
|
||||
"jq\n2.34: cannot load shared library",
|
||||
"jq\n\n\n99.9",
|
||||
"jq -- 2.34 (real jq-1.6)",
|
||||
"jq\t\t9.9",
|
||||
# Killed ONLY by the first-line slice + `[[:blank:]]`. These carry the literal word `version`,
|
||||
# so the separator restriction is satisfied and cannot save us — the newline must be excluded
|
||||
# from the separator class AND the parse confined to line one.
|
||||
#
|
||||
# Without these, a round-5 mutation check found that reverting BOTH of those changes together
|
||||
# (`[[:blank:]]`→`[[:space:]]` and parsing `$raw` instead of `$first`) left the whole suite
|
||||
# GREEN: the four cases above are all killed by the separator alone, so they attributed the fix
|
||||
# to the wrong layer. A test that passes for the wrong reason is how the previous three rounds
|
||||
# each shipped a defect.
|
||||
"jq\nversion\n9.9",
|
||||
"jq\nversion 9.9",
|
||||
"jq \n version \n 9.9",
|
||||
],
|
||||
)
|
||||
def test_a_number_AFTER_the_jq_token_is_not_reachable_across_filler(preflight, version_line):
|
||||
"""Two independent layers keep a stray number from being read as the version, and both are
|
||||
pinned here: the separator must be one of the forms real jq emits (`jq-1.6` / `jq version 1.6`),
|
||||
@@ -306,14 +316,17 @@ def test_the_observability_line_stays_on_ONE_line(preflight):
|
||||
assert "trailing noise" not in r.stdout
|
||||
|
||||
|
||||
@pytest.mark.parametrize("version_line,expected", [
|
||||
("jq-1.6 (Debian 1.6-2.1)", "1.6"), # distro packaging suffix
|
||||
("jq-1.10", "1.10"), # two-digit minor: must compare numerically, not lexically
|
||||
("jq-1.7.1", "1.7"),
|
||||
("jq-1.6.0", "1.6"),
|
||||
("jq-v1.6", "1.6"),
|
||||
("JQ-1.6", "1.6"),
|
||||
])
|
||||
@pytest.mark.parametrize(
|
||||
"version_line,expected",
|
||||
[
|
||||
("jq-1.6 (Debian 1.6-2.1)", "1.6"), # distro packaging suffix
|
||||
("jq-1.10", "1.10"), # two-digit minor: must compare numerically, not lexically
|
||||
("jq-1.7.1", "1.7"),
|
||||
("jq-1.6.0", "1.6"),
|
||||
("jq-v1.6", "1.6"),
|
||||
("JQ-1.6", "1.6"),
|
||||
],
|
||||
)
|
||||
def test_legitimate_forms_still_parse_to_the_right_version(preflight, version_line, expected):
|
||||
preflight.with_jq(version_line)
|
||||
r = preflight.run()
|
||||
@@ -323,6 +336,7 @@ def test_legitimate_forms_still_parse_to_the_right_version(preflight, version_li
|
||||
|
||||
# --- Wiring guards: the preflight is worthless if a caller silently stops running it ------------
|
||||
|
||||
|
||||
def test_script_tests_pins_the_jq_version():
|
||||
"""The pin is the tripwire, so its presence is asserted rather than merely commented.
|
||||
|
||||
@@ -333,8 +347,7 @@ def test_script_tests_pins_the_jq_version():
|
||||
is `main`, which does not yet contain `scripts/jq-preflight.sh`.
|
||||
"""
|
||||
pr_checks = (WORKFLOWS / "pr-checks.yml").read_text()
|
||||
assert "jq-preflight.sh --expect" in pr_checks, \
|
||||
"script-tests must pin the jq version — that pin is the tripwire"
|
||||
assert "jq-preflight.sh --expect" in pr_checks, "script-tests must pin the jq version — that pin is the tripwire"
|
||||
|
||||
|
||||
def test_review_verdict_never_pins_a_jq_version():
|
||||
@@ -344,6 +357,7 @@ def test_review_verdict_never_pins_a_jq_version():
|
||||
follow-up PR adds the floor-only call — rather than being a comment someone can miss.
|
||||
"""
|
||||
review_verdict = (WORKFLOWS / "review-verdict.yml").read_text()
|
||||
assert "jq-preflight.sh --expect" not in review_verdict, \
|
||||
("review-verdict.yml must NOT pin a jq version: it writes the required review-verdict/h10 "
|
||||
"status, so a pin would deadlock every merge on a jq bump (ersatztv#648)")
|
||||
assert "jq-preflight.sh --expect" not in review_verdict, (
|
||||
"review-verdict.yml must NOT pin a jq version: it writes the required review-verdict/h10 "
|
||||
"status, so a pin would deadlock every merge on a jq bump (ersatztv#648)"
|
||||
)
|
||||
|
||||
@@ -33,7 +33,7 @@ SHA = "a9e3e23abf337980ca4c05854f5b1e210099d08b"
|
||||
|
||||
# The PR is deliberately NOT docs-only: the docs-only exemption short-circuits the whole gate, so a
|
||||
# docs PR would never reach the base check and the tests would pass without exercising it.
|
||||
CURL_SHIM = r'''#!/usr/bin/env python3
|
||||
CURL_SHIM = r"""#!/usr/bin/env python3
|
||||
import json, os, sys, pathlib, urllib.parse
|
||||
|
||||
state = pathlib.Path(os.environ["STUB_DIR"])
|
||||
@@ -75,14 +75,18 @@ if "/pulls/" in url:
|
||||
sys.exit(0)
|
||||
|
||||
print("{}")
|
||||
'''
|
||||
"""
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def hook(tmp_path):
|
||||
bindir = tmp_path / "bin"; bindir.mkdir()
|
||||
curl = bindir / "curl"; curl.write_text(CURL_SHIM); curl.chmod(0o755)
|
||||
state = tmp_path / "state"; state.mkdir()
|
||||
bindir = tmp_path / "bin"
|
||||
bindir.mkdir()
|
||||
curl = bindir / "curl"
|
||||
curl.write_text(CURL_SHIM)
|
||||
curl.chmod(0o755)
|
||||
state = tmp_path / "state"
|
||||
state.mkdir()
|
||||
(state / "live_base").write_text("main")
|
||||
(state / "verdict_desc").write_text("Review-verdict: MERGEABLE @ a9e3e23 (base: main)")
|
||||
|
||||
@@ -90,7 +94,7 @@ def hook(tmp_path):
|
||||
env["PATH"] = f"{bindir}{os.pathsep}{env['PATH']}"
|
||||
env["STUB_DIR"] = str(state)
|
||||
env["STUB_SHA"] = SHA
|
||||
env["ETV_GITEA_TOKEN"] = "stub"
|
||||
env["ETV_GITEA_TOKEN"] = "stub" # noqa: S105 - deliberately fake; the real credential comes from the environment
|
||||
env["ETV_GITEA_URL"] = "http://gitea.example"
|
||||
env.pop("ETV_GITEA_BASICAUTH", None)
|
||||
|
||||
@@ -103,10 +107,8 @@ def hook(tmp_path):
|
||||
(state / "verdict_desc").write_text(desc)
|
||||
|
||||
def decision(self):
|
||||
payload = {"tool_input": {"method": "merge", "owner": "timothy",
|
||||
"repo": "ersatztv", "pull_number": 42}}
|
||||
r = subprocess.run(["bash", str(HOOK)], input=json.dumps(payload),
|
||||
env=env, capture_output=True, text=True)
|
||||
payload = {"tool_input": {"method": "merge", "owner": "timothy", "repo": "ersatztv", "pull_number": 42}}
|
||||
r = subprocess.run(["bash", str(HOOK)], input=json.dumps(payload), env=env, capture_output=True, text=True)
|
||||
assert r.returncode == 0, r.stderr
|
||||
if not r.stdout.strip():
|
||||
return None
|
||||
@@ -124,21 +126,24 @@ def test_a_retargeted_base_denies_a_verdict_formed_against_the_old_one(hook):
|
||||
reason = hook.reason()
|
||||
assert "deny" in reason, "a verdict formed against a different base was allowed to stand"
|
||||
assert "release/26.4" in reason and "main" in reason, (
|
||||
"the deny must name both bases; a reader cannot act on 'the base changed'")
|
||||
"the deny must name both bases; a reader cannot act on 'the base changed'"
|
||||
)
|
||||
|
||||
|
||||
def test_positive_control_an_unchanged_base_does_not_trigger_the_base_deny(hook):
|
||||
"""Without this, the test above could pass because the hook denies on every path — which it
|
||||
very nearly does, since this PR is non-docs and the rest of the gate is unstubbed."""
|
||||
reason = hook.reason()
|
||||
assert "ersatztv#632" not in reason, (
|
||||
"the base check fired on a PR whose base never moved")
|
||||
assert "ersatztv#632" not in reason, "the base check fired on a PR whose base never moved"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("desc", [
|
||||
"Review-verdict: MERGEABLE @ a9e3e23", # posted before #632
|
||||
"NONE", # no verdict status on this head at all
|
||||
])
|
||||
@pytest.mark.parametrize(
|
||||
"desc",
|
||||
[
|
||||
"Review-verdict: MERGEABLE @ a9e3e23", # posted before #632
|
||||
"NONE", # no verdict status on this head at all
|
||||
],
|
||||
)
|
||||
def test_a_verdict_with_no_recorded_base_gets_no_opinion(hook, desc):
|
||||
"""Graceful adoption. Denying here would block every in-flight PR the day this lands, and the
|
||||
window closes on its own: verdicts are per-head and short-lived, so every verdict posted after
|
||||
@@ -151,7 +156,8 @@ def test_a_verdict_with_no_recorded_base_gets_no_opinion(hook, desc):
|
||||
hook.set_live_base("release/26.4")
|
||||
hook.set_verdict_description(desc)
|
||||
assert "base" not in hook.reason(), (
|
||||
"a pre-#632 verdict drew a base-related decision for a field it could not have carried")
|
||||
"a pre-#632 verdict drew a base-related decision for a field it could not have carried"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("failure", ["SCALAR-ROW", "NONSTRING-DESC"])
|
||||
@@ -173,7 +179,7 @@ def test_a_malformed_status_MEMBER_asks_too(hook, failure):
|
||||
|
||||
@pytest.mark.parametrize("failure", ["TRANSPORT-ERROR", "GARBAGE"])
|
||||
def test_an_UNREADABLE_status_response_asks_rather_than_skipping_the_check(hook, failure):
|
||||
""""Could not check" is a third outcome, not a quiet synonym for "no base recorded".
|
||||
""" "Could not check" is a third outcome, not a quiet synonym for "no base recorded".
|
||||
|
||||
The first draft collapsed the two: an unreadable status response produced an empty
|
||||
`recorded_base`, took the graceful-adoption path, and skipped validation in silence — after
|
||||
@@ -202,5 +208,4 @@ def test_the_comparator_is_the_base_REF_not_its_tip_sha():
|
||||
A base branch that merely ADVANCES must be silent here; rebasing onto it moves the head sha,
|
||||
which the per-sha binding already covers."""
|
||||
assert ".base.ref" in HOOK.read_text(), "the hook must compare the base BRANCH, not its tip sha"
|
||||
assert ".base.sha" not in HOOK.read_text(), (
|
||||
"comparing base.sha deadlocks every open PR whenever main advances")
|
||||
assert ".base.sha" not in HOOK.read_text(), "comparing base.sha deadlocks every open PR whenever main advances"
|
||||
|
||||
@@ -33,7 +33,7 @@ HOOK = REPO_ROOT / ".claude" / "hooks" / "pretooluse-merge-consent.sh"
|
||||
SHA = "a9e3e23abf337980ca4c05854f5b1e210099d08b"
|
||||
|
||||
# Serves paged `pulls/N/files`, plus the minimal PR object the hook reads first.
|
||||
CURL_SHIM = r'''#!/usr/bin/env python3
|
||||
CURL_SHIM = r"""#!/usr/bin/env python3
|
||||
import json, os, sys, pathlib, urllib.parse
|
||||
|
||||
state = pathlib.Path(os.environ["STUB_DIR"])
|
||||
@@ -88,7 +88,7 @@ if "/pulls/" in url:
|
||||
sys.exit(0)
|
||||
|
||||
print("{}")
|
||||
'''
|
||||
"""
|
||||
|
||||
|
||||
def _rows(paths):
|
||||
@@ -97,15 +97,19 @@ def _rows(paths):
|
||||
|
||||
@pytest.fixture
|
||||
def hook(tmp_path):
|
||||
bindir = tmp_path / "bin"; bindir.mkdir()
|
||||
shim = bindir / "curl"; shim.write_text(CURL_SHIM); shim.chmod(0o755)
|
||||
state = tmp_path / "state"; state.mkdir()
|
||||
bindir = tmp_path / "bin"
|
||||
bindir.mkdir()
|
||||
shim = bindir / "curl"
|
||||
shim.write_text(CURL_SHIM)
|
||||
shim.chmod(0o755)
|
||||
state = tmp_path / "state"
|
||||
state.mkdir()
|
||||
|
||||
env = dict(os.environ)
|
||||
env["PATH"] = f"{bindir}{os.pathsep}{env['PATH']}"
|
||||
env["STUB_DIR"] = str(state)
|
||||
env["STUB_SHA"] = SHA
|
||||
env["ETV_GITEA_TOKEN"] = "stub"
|
||||
env["ETV_GITEA_TOKEN"] = "stub" # noqa: S105 - deliberately fake; the real credential comes from the environment
|
||||
env["ETV_GITEA_URL"] = "http://gitea.example"
|
||||
env.pop("ETV_GITEA_BASICAUTH", None)
|
||||
|
||||
@@ -114,10 +118,10 @@ def hook(tmp_path):
|
||||
(state / "pages.json").write_text(json.dumps(list(pages)))
|
||||
|
||||
def run(self):
|
||||
payload = {"tool_input": {"method": "merge", "owner": "timothy",
|
||||
"repo": "ersatztv", "pull_number": 42}}
|
||||
return subprocess.run(["bash", str(HOOK)], input=json.dumps(payload),
|
||||
env=env, capture_output=True, text=True)
|
||||
payload = {"tool_input": {"method": "merge", "owner": "timothy", "repo": "ersatztv", "pull_number": 42}}
|
||||
return subprocess.run(
|
||||
["bash", str(HOOK)], input=json.dumps(payload), env=env, capture_output=True, text=True
|
||||
)
|
||||
|
||||
def exempted(self):
|
||||
"""Exempt == passthrough == exit 0 with no decision JSON."""
|
||||
@@ -140,16 +144,14 @@ def test_code_pr_is_not_exempt(hook):
|
||||
|
||||
def test_protected_path_on_a_LATER_page_is_still_seen(hook):
|
||||
"""The #619 shape: 50 docs files on page 1, code hiding on page 2."""
|
||||
hook.set_pages(_rows([f"docs/f{i}.md" for i in range(50)]),
|
||||
_rows(["scripts/decisions_lib.py"]))
|
||||
hook.set_pages(_rows([f"docs/f{i}.md" for i in range(50)]), _rows(["scripts/decisions_lib.py"]))
|
||||
assert hook.exempted() is False
|
||||
|
||||
|
||||
def test_full_first_page_alone_does_not_end_enumeration(hook):
|
||||
"""A full 50-row page must trigger a second fetch, not terminate the loop."""
|
||||
hook.set_pages(_rows([f"docs/f{i}.md" for i in range(50)]),
|
||||
_rows(["docs/tail.md"]))
|
||||
assert hook.exempted() is True # genuinely all docs, but only provable by reading page 2
|
||||
hook.set_pages(_rows([f"docs/f{i}.md" for i in range(50)]), _rows(["docs/tail.md"]))
|
||||
assert hook.exempted() is True # genuinely all docs, but only provable by reading page 2
|
||||
|
||||
|
||||
def test_rename_of_code_into_docs_is_not_exempt(hook):
|
||||
@@ -161,15 +163,15 @@ def test_rename_of_code_into_docs_is_not_exempt(hook):
|
||||
exempt. That is the hook's contract and differs from `review-verdict.yml`'s stricter
|
||||
PROTECTED list, which must never auto-post a green status for those paths.
|
||||
"""
|
||||
hook.set_pages([{"filename": "docs/innocuous-note.md", "status": "renamed",
|
||||
"previous_filename": "ErsatzTV/Program.cs"}])
|
||||
hook.set_pages(
|
||||
[{"filename": "docs/innocuous-note.md", "status": "renamed", "previous_filename": "ErsatzTV/Program.cs"}]
|
||||
)
|
||||
assert hook.exempted() is False
|
||||
|
||||
|
||||
def test_rename_between_two_exempt_paths_stays_exempt(hook):
|
||||
"""Guards the above: reading previous_filename must not over-trigger on legitimate moves."""
|
||||
hook.set_pages([{"filename": "docs/b.md", "status": "renamed",
|
||||
"previous_filename": "docs/a.md"}])
|
||||
hook.set_pages([{"filename": "docs/b.md", "status": "renamed", "previous_filename": "docs/a.md"}])
|
||||
assert hook.exempted() is True
|
||||
|
||||
|
||||
@@ -232,14 +234,15 @@ def test_malformed_rename_row_withholds_the_exemption(hook):
|
||||
Real Gitea always populates it (verified by constructing a rename), so this is the
|
||||
malformed-2xx class the guard claims to fail closed on; the claim should match the behaviour.
|
||||
"""
|
||||
hook.set_pages(_rows([f"docs/f{i}.md" for i in range(50)]),
|
||||
[{"filename": "docs/moved.md", "status": "renamed"}])
|
||||
hook.set_pages(_rows([f"docs/f{i}.md" for i in range(50)]), [{"filename": "docs/moved.md", "status": "renamed"}])
|
||||
assert hook.exempted() is False
|
||||
|
||||
|
||||
def test_rename_row_with_empty_previous_filename_withholds_the_exemption(hook):
|
||||
hook.set_pages(_rows([f"docs/f{i}.md" for i in range(50)]),
|
||||
[{"filename": "docs/moved.md", "status": "renamed", "previous_filename": ""}])
|
||||
hook.set_pages(
|
||||
_rows([f"docs/f{i}.md" for i in range(50)]),
|
||||
[{"filename": "docs/moved.md", "status": "renamed", "previous_filename": ""}],
|
||||
)
|
||||
assert hook.exempted() is False
|
||||
|
||||
|
||||
@@ -254,9 +257,13 @@ def test_ordinary_row_without_previous_filename_is_still_valid(hook):
|
||||
`test_a_rename_disguised_by_an_unknown_status_is_rejected[None]`. The statusless row this test
|
||||
used to carry was incidental to what it is actually pinning.
|
||||
"""
|
||||
hook.set_pages([{"filename": "docs/a.md", "status": "modified"},
|
||||
{"filename": "docs/b.md", "status": "added"},
|
||||
{"filename": "docs/c.md", "status": "changed"}])
|
||||
hook.set_pages(
|
||||
[
|
||||
{"filename": "docs/a.md", "status": "modified"},
|
||||
{"filename": "docs/b.md", "status": "added"},
|
||||
{"filename": "docs/c.md", "status": "changed"},
|
||||
]
|
||||
)
|
||||
assert hook.exempted() is True
|
||||
|
||||
|
||||
@@ -312,16 +319,20 @@ sys.exit(127)
|
||||
@pytest.fixture
|
||||
def hook_jq16(tmp_path):
|
||||
"""Same harness as `hook`, plus a jq shim emulating jq 1.6's empty-input exit status."""
|
||||
bindir = tmp_path / "bin"; bindir.mkdir()
|
||||
(bindir / "curl").write_text(CURL_SHIM); (bindir / "curl").chmod(0o755)
|
||||
(bindir / "jq").write_text(_JQ16_SHIM); (bindir / "jq").chmod(0o755)
|
||||
state = tmp_path / "state"; state.mkdir()
|
||||
bindir = tmp_path / "bin"
|
||||
bindir.mkdir()
|
||||
(bindir / "curl").write_text(CURL_SHIM)
|
||||
(bindir / "curl").chmod(0o755)
|
||||
(bindir / "jq").write_text(_JQ16_SHIM)
|
||||
(bindir / "jq").chmod(0o755)
|
||||
state = tmp_path / "state"
|
||||
state.mkdir()
|
||||
|
||||
env = dict(os.environ)
|
||||
env["PATH"] = f"{bindir}{os.pathsep}{env['PATH']}"
|
||||
env["STUB_DIR"] = str(state)
|
||||
env["STUB_SHA"] = SHA
|
||||
env["ETV_GITEA_TOKEN"] = "stub"
|
||||
env["ETV_GITEA_TOKEN"] = "stub" # noqa: S105 - deliberately fake; the real credential comes from the environment
|
||||
env["ETV_GITEA_URL"] = "http://gitea.example"
|
||||
env.pop("ETV_GITEA_BASICAUTH", None)
|
||||
|
||||
@@ -330,10 +341,8 @@ def hook_jq16(tmp_path):
|
||||
(state / "pages.json").write_text(json.dumps(list(pages)))
|
||||
|
||||
def exempted(self):
|
||||
payload = {"tool_input": {"method": "merge", "owner": "timothy",
|
||||
"repo": "ersatztv", "pull_number": 42}}
|
||||
r = subprocess.run(["bash", str(HOOK)], input=json.dumps(payload),
|
||||
env=env, capture_output=True, text=True)
|
||||
payload = {"tool_input": {"method": "merge", "owner": "timothy", "repo": "ersatztv", "pull_number": 42}}
|
||||
r = subprocess.run(["bash", str(HOOK)], input=json.dumps(payload), env=env, capture_output=True, text=True)
|
||||
assert r.returncode == 0, r.stderr
|
||||
return r.stdout.strip() == ""
|
||||
|
||||
@@ -394,8 +403,9 @@ def test_newline_in_previous_filename_is_also_rejected(hook):
|
||||
rejects on its own merits, so the test passed with the newline guard entirely removed — it
|
||||
asserted the outcome without ever exercising the mechanism. That is the same
|
||||
filter-hides-the-defect trap the guard itself is about."""
|
||||
hook.set_pages([{"filename": "docs/ok.md", "previous_filename": "safe.md\ndocs/Program.cs",
|
||||
"status": "renamed"}], [])
|
||||
hook.set_pages(
|
||||
[{"filename": "docs/ok.md", "previous_filename": "safe.md\ndocs/Program.cs", "status": "renamed"}], []
|
||||
)
|
||||
assert hook.exempted() is False
|
||||
|
||||
|
||||
@@ -406,8 +416,7 @@ def test_previous_filename_is_validated_on_NON_renamed_rows_too(hook, status):
|
||||
`chunk` emits `(.previous_filename // empty)` for EVERY row regardless of `.status`, but the
|
||||
field was validated only when `.status == "renamed"`. A row marked `modified` (or Gitea's
|
||||
distinct `copied`) carrying a newline in `previous_filename` was reproducibly exempted."""
|
||||
hook.set_pages([{"filename": "docs/ok.md", "status": status,
|
||||
"previous_filename": "safe.md\ndocs/Program.cs"}], [])
|
||||
hook.set_pages([{"filename": "docs/ok.md", "status": status, "previous_filename": "safe.md\ndocs/Program.cs"}], [])
|
||||
assert hook.exempted() is False
|
||||
|
||||
|
||||
@@ -421,21 +430,18 @@ def test_dotdot_path_component_is_rejected(hook):
|
||||
|
||||
def test_legitimate_rename_within_docs_still_exempts(hook):
|
||||
"""Positive control: the tightened row schema must not break a real docs-only rename."""
|
||||
hook.set_pages([{"filename": "docs/b.md", "status": "renamed",
|
||||
"previous_filename": "docs/a.md"}], [])
|
||||
hook.set_pages([{"filename": "docs/b.md", "status": "renamed", "previous_filename": "docs/a.md"}], [])
|
||||
assert hook.exempted() is True
|
||||
|
||||
|
||||
def test_short_NONTERMINAL_page_does_not_end_the_enumeration(hook):
|
||||
""""Fewer rows than we asked for" must not be read as "last page".
|
||||
""" "Fewer rows than we asked for" must not be read as "last page".
|
||||
|
||||
Gitea caps `limit` at the server-wide MAX_RESPONSE_ITEMS (default 50, configurable) and may
|
||||
return fewer rows than requested. A 30-row docs page followed by a page of code would otherwise
|
||||
complete the enumeration over a PARTIAL list — the same fail-open, reached with no transport
|
||||
error at all. Only a validated EMPTY page may terminate it."""
|
||||
hook.set_pages(_rows([f"docs/f{i}.md" for i in range(30)]),
|
||||
_rows(["ErsatzTV/Program.cs"]),
|
||||
[])
|
||||
hook.set_pages(_rows([f"docs/f{i}.md" for i in range(30)]), _rows(["ErsatzTV/Program.cs"]), [])
|
||||
assert hook.exempted() is False
|
||||
|
||||
|
||||
@@ -473,9 +479,14 @@ def test_gitea_real_status_values_are_accepted(hook):
|
||||
"""Positive control for the closed set. The real Gitea 1.25.4 value for an edit is `changed`,
|
||||
NOT `modified` — a closed allow-list built from the wrong vocabulary would reject every real
|
||||
docs-only PR, which is a far worse failure than the hole it closes."""
|
||||
hook.set_pages([{"filename": "docs/a.md", "status": "changed"},
|
||||
{"filename": "docs/b.md", "status": "added"},
|
||||
{"filename": "docs/c.md", "status": "deleted"}], [])
|
||||
hook.set_pages(
|
||||
[
|
||||
{"filename": "docs/a.md", "status": "changed"},
|
||||
{"filename": "docs/b.md", "status": "added"},
|
||||
{"filename": "docs/c.md", "status": "deleted"},
|
||||
],
|
||||
[],
|
||||
)
|
||||
assert hook.exempted() is True
|
||||
|
||||
|
||||
@@ -483,6 +494,7 @@ def test_gitea_real_status_values_are_accepted(hook):
|
||||
# The round-3 `..` finding was an anchor subversion, and mutating the anchors showed no test
|
||||
# covered them: dropping `^` from the docs/ alternative, or `$` from `.md`, both survived.
|
||||
|
||||
|
||||
def test_docs_must_be_a_PREFIX_not_a_substring(hook):
|
||||
"""Dropping `^` would exempt `ErsatzTV/docs/Program.cs`."""
|
||||
hook.set_pages([{"filename": "ErsatzTV/docs/Program.cs", "status": "changed"}], [])
|
||||
@@ -517,7 +529,6 @@ def test_object_valued_status_is_also_rejected(hook):
|
||||
assert hook.exempted() is False
|
||||
|
||||
|
||||
|
||||
# --- The `grep -q` / pipefail inversion, on the ADVISORY side (ersatztv#698) --------------------
|
||||
#
|
||||
# Round-2 cross-family review noted the enforced gate gained large-input regression tests while the
|
||||
@@ -530,6 +541,7 @@ def test_object_valued_status_is_also_rejected(hook):
|
||||
# negated docs-only test. ~171KB is needed to cross the threshold; every other test in this file uses a
|
||||
# handful of short paths, which is exactly why the class was invisible here.
|
||||
|
||||
|
||||
def _many_docs(n=1900):
|
||||
return [f"docs/{'d' * 40}-{i:040d}.md" for i in range(n)]
|
||||
|
||||
@@ -539,12 +551,12 @@ def test_a_LARGE_pr_containing_a_code_file_is_NOT_exempt(hook):
|
||||
bulk of ~171KB still to write."""
|
||||
hook.set_pages(_rows(["A.cs", *_many_docs()]))
|
||||
assert hook.exempted() is False, (
|
||||
"a large PR containing A.cs was granted the docs-only exemption — the predicate inverted")
|
||||
"a large PR containing A.cs was granted the docs-only exemption — the predicate inverted"
|
||||
)
|
||||
|
||||
|
||||
def test_positive_control_a_LARGE_genuinely_docs_only_pr_IS_still_exempt(hook):
|
||||
"""Guards the opposite failure: if large lists merely errored, the test above would pass while the
|
||||
hook prompted on every big docs PR. Without this, 'fixed' and 'broken' are indistinguishable."""
|
||||
hook.set_pages(_rows(_many_docs()))
|
||||
assert hook.exempted() is True, (
|
||||
"a large but genuinely docs-only PR lost its exemption")
|
||||
assert hook.exempted() is True, "a large but genuinely docs-only PR lost its exemption"
|
||||
|
||||
@@ -0,0 +1,843 @@
|
||||
"""The scheduled-auto-merge path verifies the protection it rests on (ersatztv#778).
|
||||
|
||||
`merge_when_checks_succeed` hands the actual merge to Gitea, to be performed later against whatever
|
||||
head is green at that moment. Everything the hook proves is therefore a SNAPSHOT. What makes that
|
||||
safe is stated in the hook and in #622: `review-verdict/h10` is a REQUIRED status check on the base
|
||||
branch, a commit status belongs to exactly one sha, so a commit pushed after scheduling cannot
|
||||
inherit the verdict and Gitea's own gate refuses the merge.
|
||||
|
||||
That guarantee is branch-protection CONFIG. It lives outside this repo, and before #778 nothing
|
||||
compared the two — the hook asserted it in a comment and in the reason string a human reads, which
|
||||
is a claim about the past, not a check. These tests pin the conversion of that assumption into a
|
||||
precondition.
|
||||
|
||||
The outcome set is the contract, and each arm is asserted separately because collapsing any two of
|
||||
them is how this class of guard has failed here before:
|
||||
|
||||
* required check PRESENT -> proceed (no opinion drawn from this check)
|
||||
* branch protection UNREADABLE -> ask (a transient failure is not evidence of safety)
|
||||
* required check ABSENT -> deny (this is #622's hole reopened, not a degraded read)
|
||||
* a GLOB rule COULD govern the base -> ask, and distinctly from the unreadable case. This hook
|
||||
does not reimplement Gitea's glob dialect, so "some rule might apply and we cannot tell" is a
|
||||
fourth answer, not a flavour of the third. Tests that assert only `"ask" in reason` cannot tell
|
||||
the two apart — and a crashed classifier also produces an ask — so each arm is pinned on the
|
||||
text unique to it.
|
||||
|
||||
Observable contract: the hook exits 0 with EMPTY stdout when it has no opinion, and emits a JSON
|
||||
`permissionDecision` otherwise.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[2]
|
||||
HOOK = REPO_ROOT / ".claude" / "hooks" / "pretooluse-merge-consent.sh"
|
||||
|
||||
SHA = "a9e3e23abf337980ca4c05854f5b1e210099d08b"
|
||||
SHORT = SHA[:7]
|
||||
|
||||
# The PR is deliberately NOT docs-only: the docs-only exemption short-circuits the whole gate before
|
||||
# the scheduled-merge branch is reached, so a docs PR would pass these tests without ever running
|
||||
# the code under test.
|
||||
CURL_SHIM = r"""#!/usr/bin/env python3
|
||||
import json, os, sys, pathlib, urllib.parse
|
||||
|
||||
state = pathlib.Path(os.environ["STUB_DIR"])
|
||||
args = sys.argv[1:]
|
||||
url = [a for a in args if a.startswith("http")][-1]
|
||||
|
||||
# The branch-protection call uses `-o <file> -w '%{http_code}'` rather than `curl -sf`, precisely so
|
||||
# it can tell a 200 from every other outcome (`curl -sf` collapses every HTTP error into exit 22
|
||||
# with empty output, hiding the difference between an empty list and a failed request). It does
|
||||
# NOT treat 404 as a finding. The shim must therefore behave like real curl for those
|
||||
# flags: body to the -o file, status code to stdout. A shim that ignored them would make the hook
|
||||
# read an empty body and a blank code on EVERY path, and the tests would pass by accident against a
|
||||
# guard that never ran — the "test double's fidelity claim" failure this repo has on record.
|
||||
def respond(body, code="200"):
|
||||
if "-o" in args:
|
||||
pathlib.Path(args[args.index("-o") + 1]).write_text(body)
|
||||
else:
|
||||
sys.stdout.write(body)
|
||||
if "-w" in args:
|
||||
sys.stdout.write(code)
|
||||
sys.exit(0)
|
||||
|
||||
# RECORD BEFORE FILTERING. This recorder used to live inside the `endswith` branch below, which
|
||||
# made the "no ref reaches the URL" assertion unfalsifiable: the only URLs it could record were ones
|
||||
# that already satisfied it, so a by-name request was invisible to the very test written to forbid
|
||||
# it. Cold review reintroduced a by-name lookup in the hook and the suite stayed 33/33 green. That is
|
||||
# the filter-on-the-asserted-property defect this PR's sibling record is about, committed inside the
|
||||
# guard against it — so the recorder now sees EVERY branch-protection URL, whatever its shape.
|
||||
if "/branch_protections" in url:
|
||||
with (state / "bp_urls").open("a") as fh:
|
||||
fh.write(url + "\n")
|
||||
|
||||
if url.rstrip("/").endswith("/branch_protections"):
|
||||
# The hook reads ONLY this endpoint now — the by-name lookup was deleted because it performs no
|
||||
# matching and knows nothing about rule precedence, so a 200 from it proved less than it looked.
|
||||
mode = (state / "bp").read_text().strip()
|
||||
|
||||
if mode == "TRANSPORT-ERROR":
|
||||
respond("", "000")
|
||||
if mode == "FORBIDDEN":
|
||||
respond('{"message":"token does not have at least one of required scope(s)"}', "403")
|
||||
if mode == "GARBAGE":
|
||||
respond('{"message":"not an array"}')
|
||||
if mode == "EMPTY":
|
||||
respond('')
|
||||
if mode == "UNPARSEABLE-RULES":
|
||||
# A 200 whose rule NAME is a number: `//` fires only on null/false, so the classifier's
|
||||
# `explode`/`match` throws and the program dies on a read that plainly succeeded.
|
||||
respond(json.dumps([{"branch_name": 7, "enable_status_check": True,
|
||||
"status_check_contexts": ["review-verdict/h10"]}]))
|
||||
if mode == "LIST-404":
|
||||
# An HTTP 404 from the LIST endpoint: the repo is absent, or invisible to this credential.
|
||||
# Gitea answers 404 for both, and it says NOTHING about whether the base is protected.
|
||||
respond('{"message":"Not Found"}', "404")
|
||||
if mode == "EMPTY-LIST":
|
||||
# The list WAS read and holds no rule — the only shape that establishes absence.
|
||||
respond("[]")
|
||||
if mode == "LIST-UNREADABLE":
|
||||
respond('{"message":"internal error"}', "500")
|
||||
if mode == "GLOB-RULE":
|
||||
respond(json.dumps([{"branch_name": "m*", "enable_status_check": True,
|
||||
"status_check_contexts": ["review-verdict/h10"]}]))
|
||||
if mode == "REGEX-META-RULE":
|
||||
respond(json.dumps([{"branch_name": "mai.", "enable_status_check": True,
|
||||
"status_check_contexts": ["review-verdict/h10"]}]))
|
||||
if mode == "GLOB-WITH-DOT-RULE":
|
||||
respond(json.dumps([{"branch_name": "release/26.*", "enable_status_check": True,
|
||||
"status_check_contexts": ["review-verdict/h10"]}]))
|
||||
if mode == "CHARCLASS-RULE":
|
||||
respond(json.dumps([{"branch_name": "a[b", "enable_status_check": True,
|
||||
"status_check_contexts": ["review-verdict/h10"]},
|
||||
{"branch_name": "main", "enable_status_check": True,
|
||||
"status_check_contexts": ["review-verdict/h10"]}]))
|
||||
if mode == "ESCAPED-META-RULE":
|
||||
respond(json.dumps([{"branch_name": "a\\{b", "enable_status_check": True,
|
||||
"status_check_contexts": ["review-verdict/h10"]}]))
|
||||
if mode == "CASEFOLD-RULE":
|
||||
respond(json.dumps([{"branch_name": "MAIN", "enable_status_check": True,
|
||||
"status_check_contexts": ["review-verdict/h10"]}]))
|
||||
if mode == "TWO-CASE-VARIANT-RULES":
|
||||
respond(json.dumps([{"branch_name": "MAIN", "enable_status_check": True,
|
||||
"status_check_contexts": ["review-verdict/h10"]},
|
||||
{"branch_name": "main", "enable_status_check": True,
|
||||
"status_check_contexts": ["Build ErsatzTV Image / Build & test (.NET)"]}]))
|
||||
if mode == "NONASCII-RULE":
|
||||
respond(json.dumps([{"branch_name": "\u00fcnstable", "enable_status_check": True,
|
||||
"status_check_contexts": ["review-verdict/h10"]}]))
|
||||
if mode == "EXACT-PLUS-GLOB-RULE":
|
||||
respond(json.dumps([{"branch_name": "main", "enable_status_check": True,
|
||||
"status_check_contexts": ["review-verdict/h10"]},
|
||||
{"branch_name": "m*", "enable_status_check": True,
|
||||
"status_check_contexts": ["Build ErsatzTV Image / Build & test (.NET)"]}]))
|
||||
if mode == "MALFORMED-MEMBER":
|
||||
respond(json.dumps([{"branch_name": "main", "enable_status_check": True,
|
||||
"status_check_contexts": 7}]))
|
||||
if mode == "SUBSTRING-STRING":
|
||||
respond(json.dumps([{"branch_name": "main", "enable_status_check": True,
|
||||
"status_check_contexts": "prefix-review-verdict/h10-suffix"}]))
|
||||
if mode == "FALSE-CONTEXTS":
|
||||
respond(json.dumps([{"branch_name": "main", "enable_status_check": True,
|
||||
"status_check_contexts": False}]))
|
||||
if mode == "STRING-ENABLE":
|
||||
respond(json.dumps([{"branch_name": "main", "enable_status_check": "true",
|
||||
"status_check_contexts": ["review-verdict/h10"]}]))
|
||||
if mode == "NON-STRING-MEMBER":
|
||||
respond(json.dumps([{"branch_name": "main", "enable_status_check": True,
|
||||
"status_check_contexts": [7, "review-verdict/h10"]}]))
|
||||
if mode == "EMPTY-CONTEXTS":
|
||||
respond(json.dumps([{"branch_name": "main", "enable_status_check": True,
|
||||
"status_check_contexts": []}]))
|
||||
if mode == "NULL-CONTEXTS":
|
||||
respond(json.dumps([{"branch_name": "main", "enable_status_check": True,
|
||||
"status_check_contexts": None}]))
|
||||
if mode == "LONGER-STRING-MEMBER":
|
||||
respond(json.dumps([{"branch_name": "main", "enable_status_check": True,
|
||||
"status_check_contexts": ["xx-review-verdict/h10-yy"]}]))
|
||||
if mode == "STATUS-CHECK-OFF":
|
||||
respond(json.dumps([{"branch_name": "main", "enable_status_check": False,
|
||||
"status_check_contexts": ["review-verdict/h10"]}]))
|
||||
if mode == "MISSING-CONTEXT":
|
||||
respond(json.dumps([{"branch_name": "main", "enable_status_check": True,
|
||||
"status_check_contexts":
|
||||
["Build ErsatzTV Image / Build & test (.NET) (pull_request)"]}]))
|
||||
# GUARDED
|
||||
respond(json.dumps([{"branch_name": "main", "enable_status_check": True,
|
||||
"status_check_contexts":
|
||||
["Build ErsatzTV Image / Build & test (.NET) (pull_request)",
|
||||
"review-verdict/h10"]}]))
|
||||
|
||||
if "/pulls/" in url and "/files" in url:
|
||||
q = urllib.parse.parse_qs(urllib.parse.urlparse(url).query)
|
||||
page = int(q.get("page", ["1"])[0])
|
||||
if page == 1:
|
||||
print(json.dumps([{"filename": "ErsatzTV/Program.cs", "status": "modified"}]))
|
||||
else:
|
||||
print("[]")
|
||||
sys.exit(0)
|
||||
|
||||
if "/status" in url and (state / "no_recorded_base").exists():
|
||||
# A verdict posted before ersatztv#632 carries no `(base: …)` marker, so the #632 detection
|
||||
# takes its graceful-adoption path and forms no opinion. That isolates the hoisted retarget
|
||||
# check as the ONLY guard that can deny.
|
||||
print(json.dumps({"state": "success", "statuses": [
|
||||
{"context": "review-verdict/h10", "status": "success",
|
||||
"description": "Review-verdict: MERGEABLE @ %s" % os.environ["STUB_SHORT"]}]}))
|
||||
sys.exit(0)
|
||||
|
||||
if "/status" in url:
|
||||
# A 2xx body whose `.statuses` IS an array but whose members are scalars. `.statuses | type ==
|
||||
# "array"` passes; indexing a number then makes jq exit 5 and, under `set -e`, kills the hook
|
||||
# with no JSON at all.
|
||||
_m = (state / "bp").read_text().strip()
|
||||
if _m == "SCALAR-STATUS-ROW":
|
||||
print('{"state":"success","statuses":[1]}')
|
||||
sys.exit(0)
|
||||
if _m == "NONSTRING-STATUS-ROW":
|
||||
# Object shape, string context, valid description — passes the #632 block, which does NOT
|
||||
# validate `.status` — but a NUMERIC status. This is the shape that actually reaches the
|
||||
# scheduled branch's validator, i.e. the new clause's reachable contribution.
|
||||
print(json.dumps({"state": "success", "statuses": [
|
||||
{"context": "review-verdict/h10", "status": 7,
|
||||
"description": "Review-verdict: MERGEABLE @ %s (base: main)"
|
||||
% os.environ["STUB_SHORT"]}]}))
|
||||
sys.exit(0)
|
||||
print(json.dumps({"state": "success", "statuses": [
|
||||
{"context": "review-verdict/h10", "status": "success",
|
||||
"description": "Review-verdict: MERGEABLE @ %s (base: main)" % os.environ["STUB_SHORT"]}]}))
|
||||
sys.exit(0)
|
||||
|
||||
if "/issues/" in url and "/comments" in url:
|
||||
print(json.dumps([{"body": "Review-verdict: MERGEABLE @ %s" % os.environ["STUB_SHORT"]}]))
|
||||
sys.exit(0)
|
||||
|
||||
if "/issues/" in url:
|
||||
print(json.dumps({"body": "## Done-when\n- [x] everything\n"}))
|
||||
sys.exit(0)
|
||||
|
||||
if "/pulls/" in url:
|
||||
# The base is served per-GET so a PERSISTENT retarget mid-run can be modelled: the first read
|
||||
# (top of the hook) sees `main`, a later one sees whatever `retarget` names.
|
||||
counter = state / "pr_get_count"
|
||||
n = int(counter.read_text()) if counter.exists() else 0
|
||||
counter.write_text(str(n + 1))
|
||||
base = "main"
|
||||
bo = state / "base_override"
|
||||
if bo.exists():
|
||||
base = bo.read_text().strip()
|
||||
rt = state / "retarget"
|
||||
if rt.exists() and n >= 1:
|
||||
base = rt.read_text().strip()
|
||||
print(json.dumps({"head": {"sha": os.environ["STUB_SHA"]},
|
||||
"base": {"ref": base, "sha": "b" * 40},
|
||||
"body": "fixes #1"}))
|
||||
sys.exit(0)
|
||||
|
||||
print("{}")
|
||||
"""
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def hook(tmp_path):
|
||||
bindir = tmp_path / "bin"
|
||||
bindir.mkdir()
|
||||
curl = bindir / "curl"
|
||||
curl.write_text(CURL_SHIM)
|
||||
curl.chmod(0o755)
|
||||
state = tmp_path / "state"
|
||||
state.mkdir()
|
||||
(state / "bp").write_text("GUARDED")
|
||||
|
||||
env = dict(os.environ)
|
||||
env["PATH"] = f"{bindir}{os.pathsep}{env['PATH']}"
|
||||
env["STUB_DIR"] = str(state)
|
||||
env["STUB_SHA"] = SHA
|
||||
env["STUB_SHORT"] = SHORT
|
||||
env["ETV_GITEA_TOKEN"] = "stub" # noqa: S105 - deliberately fake; the real credential comes from the environment
|
||||
env["ETV_GITEA_URL"] = "http://gitea.example"
|
||||
env["CLAUDE_PROJECT_DIR"] = str(REPO_ROOT)
|
||||
env.pop("ETV_GITEA_BASICAUTH", None)
|
||||
|
||||
class Handle:
|
||||
def set_branch_protection(self, mode):
|
||||
(state / "bp").write_text(mode)
|
||||
|
||||
def set_base(self, ref):
|
||||
"""The base reported by EVERY PR read — a stable target, not a retarget."""
|
||||
(state / "base_override").write_text(ref)
|
||||
|
||||
def drop_recorded_base(self):
|
||||
"""Serve a pre-#632 verdict (no `(base: …)`), so only the hoisted check can deny."""
|
||||
(state / "no_recorded_base").write_text("1")
|
||||
|
||||
def set_retarget(self, ref):
|
||||
"""Persistent retarget: every PR read after the first reports `ref`."""
|
||||
(state / "retarget").write_text(ref)
|
||||
|
||||
def branch_protection_urls(self):
|
||||
f = state / "bp_urls"
|
||||
return f.read_text().splitlines() if f.exists() else []
|
||||
|
||||
def decision(self, scheduled=True):
|
||||
payload = {
|
||||
"tool_input": {
|
||||
"method": "merge",
|
||||
"owner": "timothy",
|
||||
"repo": "ersatztv",
|
||||
"pull_number": 42,
|
||||
"merge_when_checks_succeed": scheduled,
|
||||
}
|
||||
}
|
||||
r = subprocess.run(["bash", str(HOOK)], input=json.dumps(payload), env=env, capture_output=True, text=True)
|
||||
assert r.returncode == 0, r.stderr
|
||||
if not r.stdout.strip():
|
||||
return None
|
||||
return json.loads(r.stdout)
|
||||
|
||||
def reason(self, scheduled=True):
|
||||
d = self.decision(scheduled=scheduled)
|
||||
return "" if d is None else json.dumps(d)
|
||||
|
||||
return Handle()
|
||||
|
||||
|
||||
def test_a_base_without_the_required_check_denies_a_SCHEDULED_merge(hook):
|
||||
"""The defect #778 closes: arming an auto-merge while the per-sha gate that makes it safe is
|
||||
absent. Nothing else in the flow notices, which is what makes it worth a guard."""
|
||||
hook.set_branch_protection("MISSING-CONTEXT")
|
||||
reason = hook.reason()
|
||||
assert "deny" in reason, (
|
||||
"a scheduled auto-merge was armed with no 'review-verdict/h10' required check on the base — "
|
||||
"that is ersatztv#622's hole reopened"
|
||||
)
|
||||
assert "review-verdict/h10" in reason, (
|
||||
"the deny must name the missing context; a reader cannot act on 'branch protection is wrong'"
|
||||
)
|
||||
|
||||
|
||||
def test_status_checks_disabled_wholesale_also_denies(hook):
|
||||
"""The context can be listed while `enable_status_check` is false, in which case Gitea enforces
|
||||
none of them. Reading only the list would report the protection as present — the same
|
||||
check-the-label-not-the-capability shape (#697/#698) this repo has paid for twice."""
|
||||
hook.set_branch_protection("STATUS-CHECK-OFF")
|
||||
reason = hook.reason()
|
||||
assert "deny" in reason, "status checks were disabled entirely and the listed context was read as protection anyway"
|
||||
|
||||
|
||||
def test_positive_control_a_guarded_base_REACHES_the_check_and_still_auto_grants(hook):
|
||||
"""Without this, every test above passes if the hook denies on all paths — which it very nearly
|
||||
does, since this PR is non-docs and several later conditions are stubbed only loosely.
|
||||
|
||||
Asserting the absence of one phrase was not enough (cold review): an unrelated early `ask`, or a
|
||||
differently-worded deny, would satisfy it while proving nothing. So this pins all three of the
|
||||
things that must be true — the branch-protection endpoint was actually CALLED, the decision is
|
||||
`allow`, and the reason is the satisfied-gate message rather than any refusal.
|
||||
"""
|
||||
decision = hook.decision()
|
||||
assert hook.branch_protection_urls(), (
|
||||
"the guarded case never reached the branch-protection endpoint, so the other tests are not "
|
||||
"exercising the code they claim to"
|
||||
)
|
||||
assert decision is not None, "the hook passed through instead of auto-granting"
|
||||
verdict = decision["hookSpecificOutput"]["permissionDecision"]
|
||||
assert verdict == "allow", f"a fully-satisfied gate did not auto-grant (got {verdict!r})"
|
||||
assert "satisfied" in decision["hookSpecificOutput"]["permissionDecisionReason"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("shape", ["SUBSTRING-STRING", "LONGER-STRING-MEMBER"])
|
||||
def test_a_context_name_that_merely_CONTAINS_the_required_one_does_not_satisfy_it(hook, shape):
|
||||
"""The false-OPEN this guard must not have.
|
||||
|
||||
jq's `index()` on a STRING is substring search, so `"prefix-review-verdict/h10-suffix"` answers
|
||||
yes to a naive membership test — auto-granting a scheduled merge on a base where the context is
|
||||
not required at all. `LONGER-STRING-MEMBER` covers the same confusion inside a real array.
|
||||
A false-closed here costs one prompt; a false-open costs an unreviewed merge, so the membership
|
||||
test is exact equality over a value first proven to be an array of strings.
|
||||
"""
|
||||
hook.set_branch_protection(shape)
|
||||
reason = hook.reason()
|
||||
assert "allow" not in reason or "deny" in reason or "ask" in reason, (
|
||||
f"payload shape {shape} auto-granted a scheduled merge"
|
||||
)
|
||||
assert "satisfied" not in reason, (
|
||||
f"a context name that merely contains 'review-verdict/h10' ({shape}) was accepted as it"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("shape", ["EMPTY-CONTEXTS", "NULL-CONTEXTS"])
|
||||
def test_an_empty_or_null_contexts_list_DENIES_rather_than_asking(hook, shape):
|
||||
"""Absent is the finding, not a read failure. An empty or null list is a well-formed answer
|
||||
meaning "nothing is required here", so it must take the deny arm and not be swept into the
|
||||
unknown-shape ask alongside genuinely unreadable payloads."""
|
||||
hook.set_branch_protection(shape)
|
||||
reason = hook.reason()
|
||||
assert "deny" in reason, f"{shape} was treated as unreadable rather than as a confirmed absent required check"
|
||||
|
||||
|
||||
def test_a_FALSE_contexts_value_asks_rather_than_denying(hook):
|
||||
"""jq's `//` alternative fires on `false`, not only on null, so `// []` mapped this malformed
|
||||
payload to an empty list and answered "no" — a confident deny derived from a shape that was
|
||||
never understood. Absent and null are defaulted explicitly; everything else is unknown."""
|
||||
hook.set_branch_protection("FALSE-CONTEXTS")
|
||||
reason = hook.reason()
|
||||
assert "ask" in reason, "a false contexts value was defaulted to [] and produced a deny"
|
||||
|
||||
|
||||
def test_a_non_string_MEMBER_inside_the_array_asks(hook):
|
||||
"""`[7, "review-verdict/h10"]` contains the context, but the payload is not the shape this
|
||||
guard knows how to reason about. Answering "yes" would mean trusting a structure we cannot
|
||||
validate; the honest answer is that we could not tell."""
|
||||
hook.set_branch_protection("NON-STRING-MEMBER")
|
||||
reason = hook.reason()
|
||||
assert "ask" in reason, "an array with a non-string member produced a decision anyway"
|
||||
|
||||
|
||||
def test_a_malformed_contexts_MEMBER_asks_rather_than_denying_with_the_wrong_reason(hook):
|
||||
"""One level below the response-shape check, and it survives it.
|
||||
|
||||
`{"status_check_contexts": 7}` is a perfectly good object, so the top-level type guard passes;
|
||||
jq then errors on the member, `|| true` turns that into an empty string, and a two-way test
|
||||
would report "NOT a required status check" — a confident, specific, wrong diagnosis of a payload
|
||||
that was never read. The same swallow one level down as the #632 base-change guard's second fix.
|
||||
"""
|
||||
hook.set_branch_protection("MALFORMED-MEMBER")
|
||||
reason = hook.reason()
|
||||
assert "ask" in reason, "a malformed contexts member produced a decision instead of a question"
|
||||
assert "NOT a required status check" not in reason, (
|
||||
"an unreadable payload was reported as a confirmed missing required check"
|
||||
)
|
||||
|
||||
|
||||
def test_a_base_with_NO_branch_protection_at_all_denies_rather_than_asking(hook):
|
||||
"""The strongest form of the thing being checked, and the likeliest real trigger.
|
||||
|
||||
A rule list that is READABLE and EMPTY has nothing that can govern any base, so
|
||||
`review-verdict/h10` is definitively not required and scheduling an auto-merge is #622's hole.
|
||||
That must DENY, not ask: routing the most likely real-world trigger — branch protection removed
|
||||
— to a human prompt would make it read like a transient hiccup.
|
||||
|
||||
Absence is established by the LIST, never by a status code; this fixture returns 200 with `[]`.
|
||||
An HTTP 404 means the repo was absent or invisible to the credential and is a read failure,
|
||||
covered by `test_an_HTTP_404_on_the_LIST_read_asks_and_does_not_claim_the_list_was_read`.
|
||||
"""
|
||||
hook.set_branch_protection("EMPTY-LIST")
|
||||
reason = hook.reason()
|
||||
assert "deny" in reason, "a base with no branch protection at all did not deny a scheduled auto-merge"
|
||||
assert "none matches" in reason, (
|
||||
"the deny must distinguish 'the list was read and nothing governs this base' from 'could not read'"
|
||||
)
|
||||
|
||||
|
||||
def test_a_403_asks_because_it_says_only_that_we_could_not_look(hook):
|
||||
"""A credential without the repo-admin scope this endpoint needs proves nothing about the
|
||||
protection, so it must NOT deny — otherwise the guard strands every scheduled merge run made
|
||||
with a narrower token."""
|
||||
hook.set_branch_protection("FORBIDDEN")
|
||||
reason = hook.reason()
|
||||
assert "ask" in reason, "a 403 was treated as evidence about the protection"
|
||||
assert "none matches" not in reason, "a 403 was reported as a confirmed absence of branch protection"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("failure", ["TRANSPORT-ERROR", "GARBAGE", "EMPTY", "LIST-404"])
|
||||
def test_an_UNREADABLE_branch_protection_asks_rather_than_denying_or_passing(hook, failure):
|
||||
""" "Could not check" is a third outcome, not a synonym for either neighbour.
|
||||
|
||||
Denying would strand every scheduled merge on a Gitea hiccup or on credentials without the
|
||||
repo-admin scope this endpoint needs. Passing would be worse: it would restore the exact
|
||||
unverified assumption #778 exists to remove, while now printing a reason string claiming the
|
||||
protection was confirmed.
|
||||
"""
|
||||
hook.set_branch_protection(failure)
|
||||
reason = hook.reason()
|
||||
assert "ask" in reason, f"an unreadable branch-protection response ({failure}) did not fall through to a human"
|
||||
assert "branch-protection rules" in reason, "the ask must name what could not be checked"
|
||||
|
||||
|
||||
def test_an_IMMEDIATE_merge_is_not_subjected_to_this_check(hook):
|
||||
"""Scope, deliberately narrow — and stated without the overclaim cold review removed.
|
||||
|
||||
An immediate merge is not window-FREE: the hook returns `allow` and a separate call performs the
|
||||
merge, so a push can still land in between. What it lacks is a SCHEDULER — nothing waits on
|
||||
pending checks, so the gap is one tool call rather than however long CI takes. The required
|
||||
branch-protection check is what protects the scheduled path specifically, so extending this deny
|
||||
to immediate merges would block a materially safer operation and invite the whole guard being
|
||||
switched off. The residual on this path is carried in docs/remote-state-inventory.md.
|
||||
"""
|
||||
hook.set_branch_protection("MISSING-CONTEXT")
|
||||
reason = hook.reason(scheduled=False)
|
||||
assert "NOT a required status check" not in reason, (
|
||||
"the required-check deny fired on an immediate merge, which has no post-scheduling window"
|
||||
)
|
||||
|
||||
|
||||
def test_a_NON_BOOLEAN_enable_status_check_asks(hook):
|
||||
"""`"true"` is not `true`. Comparing the string to `true` yields a confident "no" -> deny from a
|
||||
payload never understood, which collapses the documented tri-state into two states. Every
|
||||
malformed shape on this endpoint has to reach the same ask arm."""
|
||||
hook.set_branch_protection("STRING-ENABLE")
|
||||
reason = hook.reason()
|
||||
assert "ask" in reason, "a string enable_status_check produced a decision instead of a question"
|
||||
assert "NOT a required status check" not in reason
|
||||
|
||||
|
||||
def test_a_SCALAR_status_row_asks_instead_of_killing_the_hook(hook):
|
||||
"""The consent hook's contract is that it always emits exactly one of grant/deny/ask, and
|
||||
`{"statuses":[1]}` is the payload that can break it: it passes an `.statuses | type == "array"`
|
||||
check, after which indexing a number errors and exits 5, which under `set -e` aborts the hook
|
||||
with NO JSON at all. A gate that emits nothing has not failed closed; it has failed to decide.
|
||||
|
||||
WHAT THIS TEST DOES *NOT* PROVE, stated because the mutation showed it. Restoring the
|
||||
scheduled-branch validation to its predecessor leaves this test GREEN, because the #632
|
||||
base-retarget block runs FIRST and validates the members it consumes — so it catches THIS
|
||||
payload and asks before the scheduled branch is reached. The two guards overlap, which is the
|
||||
masking shape `duplicate guards mask each other` describes.
|
||||
|
||||
The caveat is scoped to this payload, NOT to the clause. The earlier block validates `.context`
|
||||
and `.description` but not `.status`, so an object row with a numeric status passes it and does
|
||||
reach the new validator — `test_a_NON_STRING_status_reaches_the_scheduled_validator` covers that
|
||||
and goes red when the clause is removed. So the clause is masked for scalar rows and load-bearing
|
||||
for that one. An earlier draft called the whole clause defence-in-depth, understating it in the
|
||||
opposite direction from this repo's usual error.
|
||||
|
||||
So this asserts the OBSERVABLE contract — a decision is always emitted for this payload — which
|
||||
is true and worth pinning whichever guard supplies it. It is deliberately not offered as a
|
||||
mutation proof of the newer clause, because it is not one.
|
||||
"""
|
||||
hook.set_branch_protection("SCALAR-STATUS-ROW")
|
||||
decision = hook.decision()
|
||||
assert decision is not None, (
|
||||
"the hook emitted no decision at all for a malformed statuses payload — it neither granted, denied nor asked"
|
||||
)
|
||||
assert decision["hookSpecificOutput"]["permissionDecision"] == "ask"
|
||||
|
||||
|
||||
def test_a_PERSISTENT_retarget_denies_on_the_IMMEDIATE_path_too(hook):
|
||||
"""The twin. The re-read first landed inside the scheduled branch only, so this exact case —
|
||||
same fixture, `merge_when_checks_succeed` absent — AUTO-GRANTED while its sibling denied.
|
||||
|
||||
Cold review demonstrated it side by side, and it is the shape this repo has on record as
|
||||
"fix one path, then check its TWIN": the fix was applied where the defect was noticed, and the
|
||||
other consumer of the same stale value kept it. The re-read is now hoisted above every
|
||||
base-dependent decision rather than duplicated into the branch that happened to be under review.
|
||||
"""
|
||||
hook.drop_recorded_base()
|
||||
hook.set_retarget("scratch")
|
||||
reason = hook.reason(scheduled=False)
|
||||
assert "deny" in reason, "an immediate merge was auto-granted after the PR was retargeted mid-evaluation"
|
||||
assert "scratch" in reason and "main" in reason
|
||||
|
||||
|
||||
def test_a_PERSISTENT_retarget_after_the_first_read_denies(hook):
|
||||
"""The defect this guard had itself, found in the fifth cold-review round.
|
||||
|
||||
`$base_ref` is captured from the PR snapshot at the top of the hook, and everything between
|
||||
then and the branch-protection lookup is round trips — the file enumeration alone can be forty
|
||||
pages. A retarget in that gap needs no ABA and no force-push: the lookup would name the OLD
|
||||
base, confirm `review-verdict/h10` on a branch the PR no longer targets, and grant a scheduled
|
||||
merge onto one that may require nothing. Checking a stale identifier is not checking, which is
|
||||
the whole of `process.check-and-use-pins-a-version` — so the guard enforcing that rule had to
|
||||
stop breaking it.
|
||||
"""
|
||||
hook.drop_recorded_base()
|
||||
hook.set_retarget("scratch")
|
||||
reason = hook.reason()
|
||||
assert "deny" in reason, "the PR was retargeted mid-evaluation and the gate still granted on the original base"
|
||||
assert "scratch" in reason and "main" in reason, (
|
||||
"the deny must name both branches; a reader cannot act on 'the base changed'"
|
||||
)
|
||||
|
||||
|
||||
def test_a_NON_STRING_status_reaches_the_scheduled_validator(hook):
|
||||
"""The reachable contribution of the scheduled-branch member validation, which the previous
|
||||
caveat understated.
|
||||
|
||||
The #632 block validates `.context` and `.description` but NOT `.status`, so an object row with
|
||||
a numeric status passes it and arrives here. Without this clause it becomes `vstate=7` and falls
|
||||
to the catch-all deny arm — fail-closed, but reported as "the verdict is '7'" rather than as a
|
||||
payload that could not be read. So the clause is masked for scalar rows and load-bearing for
|
||||
this one; the caveat on the scalar test is scoped accordingly.
|
||||
"""
|
||||
hook.set_branch_protection("NONSTRING-STATUS-ROW")
|
||||
reason = hook.reason()
|
||||
assert "ask" in reason, "a non-string .status produced a verdict-shaped decision"
|
||||
|
||||
|
||||
def test_a_base_that_a_GLOB_rule_could_govern_ASKS(hook):
|
||||
"""A base covered only by a glob rule has no rule bearing its own name, and the deleted by-name
|
||||
endpoint answered 404 for exactly that — read as "unprotected", producing a hard DENY with a
|
||||
specific, false cause.
|
||||
|
||||
But the opposite error is worse: deciding the glob DOES match would auto-grant on a base whose
|
||||
protection was never established. This hook does not reimplement Gitea's glob dialect (its `*`
|
||||
does not cross `/`, and `?`/`[…]`/`{a,b}` are wildcards), so a glob rule that COULD govern the
|
||||
base is undecidable and asks — the only answer honest in both directions.
|
||||
"""
|
||||
hook.set_branch_protection("GLOB-RULE")
|
||||
reason = hook.reason()
|
||||
assert "ask" in reason, "a base a glob rule could govern was decided rather than referred to a human"
|
||||
assert "could govern it" in reason, (
|
||||
"the ask came from the generic could-not-read arm, not the undecidable-glob arm — those "
|
||||
"are different outcomes and a crashed classifier must not pass as a correct classification"
|
||||
)
|
||||
assert "none matches" not in reason, "a base a glob rule could govern was reported as having no protection at all"
|
||||
|
||||
|
||||
def test_an_unreadable_rule_LIST_asks_rather_than_denying(hook):
|
||||
"""A read failure must not convert 'could not confirm' into 'confirmed absent'. Absence is
|
||||
established only by the classifier returning `nomatch` over a list that WAS read."""
|
||||
hook.set_branch_protection("LIST-UNREADABLE")
|
||||
reason = hook.reason()
|
||||
assert "ask" in reason, "an unreadable rule list was treated as proof of absence"
|
||||
assert "none matches" not in reason
|
||||
|
||||
|
||||
def test_the_protection_lookup_puts_NO_ref_in_the_url(hook):
|
||||
"""The successor to a URL-encoding test, and the reason it could be retired.
|
||||
|
||||
The ref used to be interpolated into `branch_protections/{name}`, where a base like
|
||||
`release/26.4` injected a path separator and 404'd — read as "unprotected". That endpoint is
|
||||
gone: it performed no matching and knew nothing about rule precedence, so a 200 from it proved
|
||||
less than it looked. Only the LIST endpoint is read now, which takes no ref at all, so the whole
|
||||
encoding hazard is removed by construction rather than escaped.
|
||||
"""
|
||||
hook.set_base("release/26.4")
|
||||
hook.drop_recorded_base()
|
||||
hook.reason()
|
||||
urls = hook.branch_protection_urls()
|
||||
assert urls, "the branch-protection endpoint was never requested"
|
||||
for u in urls:
|
||||
assert u.rstrip("/").endswith("/branch_protections"), f"a ref reached the branch-protection URL: {u}"
|
||||
|
||||
|
||||
def test_a_rule_name_with_REGEX_METACHARACTERS_does_not_match_a_different_base(hook):
|
||||
"""The false-open in the glob fallback: `*` must be the only wildcard.
|
||||
|
||||
Substituting `*` into a raw regex left every other metacharacter live, so a rule named `mai.`
|
||||
matched the base `main` (and `a+b` matched `aab`). A spurious match to some OTHER rule that
|
||||
happens to require `review-verdict/h10` reports this base as protected when nothing governs it —
|
||||
a consent gate answering yes on evidence about a different branch. Verified directly before the
|
||||
fix: `main.x` matched `mainax`.
|
||||
|
||||
Here the only rule is `mai.`, which governs a branch that is not `main`, so nothing protects the
|
||||
base and the gate must deny rather than grant.
|
||||
"""
|
||||
hook.set_branch_protection("REGEX-META-RULE")
|
||||
reason = hook.reason()
|
||||
assert "deny" in reason, "a rule named 'mai.' was regex-matched against base 'main' and read as protection"
|
||||
assert "none matches" in reason, (
|
||||
"'mai.' contains no GLOB metacharacter, so it is decidable: it simply does not govern "
|
||||
"'main', and the base is genuinely unprotected"
|
||||
)
|
||||
|
||||
|
||||
def test_a_GLOB_rule_whose_literal_part_has_a_metacharacter_still_MATCHES(hook):
|
||||
"""The positive control the first escaping attempt lacked, and the reason it looked green.
|
||||
|
||||
Escaping is only half the property: `*` must still span. The first version emitted TWO
|
||||
backslashes (`\\.` = "a literal backslash, then any character"), which made every rule
|
||||
containing a metacharacter UNMATCHABLE — so the fallback found nothing and hard-denied with the
|
||||
stated cause that no rule can govern the base — a false-open converted into a false DENY.
|
||||
|
||||
A negative-only assertion cannot see that: a rule matched literally and a rule made unmatchable
|
||||
both fail to match the wrong base. Only a rule that SHOULD match distinguishes them. Here the rule
|
||||
`release/26.*` could govern the base `release/26.4`, so the gate must reach a decision about it
|
||||
rather than reporting the base as unprotected.
|
||||
"""
|
||||
hook.set_base("release/26.4")
|
||||
hook.drop_recorded_base() # keep the #632 comparison out of this test's way
|
||||
hook.set_branch_protection("GLOB-WITH-DOT-RULE")
|
||||
reason = hook.reason()
|
||||
assert "ask" in reason, (
|
||||
"the glob rule 'release/26.*' could govern base 'release/26.4', which is undecidable here and must ask"
|
||||
)
|
||||
assert "could govern it" in reason, (
|
||||
"the ask must come from the undecidable-glob arm, not from the classifier failing"
|
||||
)
|
||||
assert "none matches" not in reason, (
|
||||
"a base a glob rule could govern was reported as entirely unprotected — the over-escaping "
|
||||
"failure this test exists to catch"
|
||||
)
|
||||
|
||||
|
||||
def test_a_rule_name_containing_a_CHAR_CLASS_bracket_does_not_crash_the_matcher(hook):
|
||||
"""`a[b` built the pattern `a\\[b`, which is a premature end of char-class: jq exits 5, the
|
||||
`|| true` swallows it, and the whole list is discarded — so an unrelated rule's NAME could
|
||||
poison the lookup and deny a base that a later rule in the same list protects."""
|
||||
hook.set_branch_protection("CHARCLASS-RULE")
|
||||
reason = hook.reason()
|
||||
assert reason, "the hook emitted no decision at all"
|
||||
assert "ask" not in reason, (
|
||||
"the exact-name rule was decidable and must have been honoured; an ask here means the "
|
||||
"classifier failed rather than classified"
|
||||
)
|
||||
assert "none matches" not in reason, (
|
||||
"one rule with a bracket in its name discarded the whole list, including the exact-name "
|
||||
"rule that actually protects this base"
|
||||
)
|
||||
assert "deny" not in reason, "an exact-name rule requiring review-verdict/h10 was present and was not honoured"
|
||||
|
||||
|
||||
def test_a_plain_rule_name_is_matched_CASE_INSENSITIVELY(hook):
|
||||
"""Gitea compares a rule name with no glob metacharacter using `EqualFold`, so a rule named
|
||||
`MAIN` governs the base `main`. Comparing case-sensitively here would find no rule, conclude the
|
||||
base is unprotected, and deny with a false stated cause."""
|
||||
hook.set_branch_protection("CASEFOLD-RULE")
|
||||
reason = hook.reason()
|
||||
assert "ask" not in reason, (
|
||||
"the case-folded exact rule was decidable and must have been honoured; an ask means the "
|
||||
"classifier failed rather than classified"
|
||||
)
|
||||
assert "none matches" not in reason, (
|
||||
"a rule named 'MAIN' governs base 'main' in Gitea but was missed by a case-sensitive compare"
|
||||
)
|
||||
assert "deny" not in reason
|
||||
|
||||
|
||||
def test_a_BACKSLASH_ESCAPED_metacharacter_in_a_rule_name_is_undecidable_not_absent(hook):
|
||||
"""The one case that breaks the superset proof the `none` arm rests on.
|
||||
|
||||
`none` authorises a DENY on the stated grounds that nothing can possibly govern this base, so
|
||||
its premise must hold unconditionally. gobwas/glob reads `\\{` as a LITERAL brace, so the rule
|
||||
`a\\{b` governs the base `a{b`; a superset that treated `\\` as an ordinary character would build
|
||||
`a\\.*b`, fail to match, and deny a base that is in fact protected. Treating backslash as a
|
||||
metacharacter restores the property.
|
||||
|
||||
Git ref rules make this nearly unreachable — a branch name may not contain `*`, `?`, `[` or `\\`
|
||||
— but `{` IS legal in a branch name, and "nearly unreachable" is not the standard for the arm
|
||||
that issues a deny.
|
||||
"""
|
||||
hook.set_base("a{b")
|
||||
hook.drop_recorded_base()
|
||||
hook.set_branch_protection("ESCAPED-META-RULE")
|
||||
reason = hook.reason()
|
||||
assert "none matches" not in reason, (
|
||||
"a rule whose escaped brace governs this base was reported as unable to govern it"
|
||||
)
|
||||
assert "ask" in reason, "an escaped-metacharacter rule is undecidable here and must ask"
|
||||
assert "could govern it" in reason, (
|
||||
"the ask must come from the undecidable-glob arm, not from the classifier failing — this "
|
||||
"test hits the same arm as its two siblings and needs the same pin"
|
||||
)
|
||||
|
||||
|
||||
def test_the_precedence_check_runs_even_when_an_exactly_named_rule_EXISTS(hook):
|
||||
"""The twin the restructure deletes, pinned so it cannot come back.
|
||||
|
||||
The hook used to look the rule up by NAME first and only enumerate the list on a 404. That
|
||||
by-name endpoint is an exact DB lookup that performs no matching and knows nothing about
|
||||
precedence, so on a 200 — the path this repo actually takes, since its rule IS named `main` —
|
||||
the gate granted having consulted one rule and never asked which rule Gitea would apply. The
|
||||
precedence argument guarded the 404 path only: hardened code that was dead, next to live code
|
||||
that was not.
|
||||
|
||||
`EXACT-PLUS-GLOB-RULE` is exactly that configuration: a rule NAMED `main` that requires
|
||||
`review-verdict/h10`, plus `m*` that does not. Under the old flow the by-name hit returned the
|
||||
`main` rule, saw h10 and granted. Now there is one path, so the classifier sees both rules and
|
||||
refuses to guess which one Gitea applies.
|
||||
"""
|
||||
hook.set_branch_protection("EXACT-PLUS-GLOB-RULE")
|
||||
reason = hook.reason()
|
||||
assert "ask" in reason, "an exactly-named rule was trusted without asking which rule Gitea would actually apply"
|
||||
assert "could govern it" in reason
|
||||
# And the by-name endpoint must not be consulted at all — its existence is what split the paths.
|
||||
for u in hook.branch_protection_urls():
|
||||
assert u.rstrip("/").endswith("/branch_protections"), (
|
||||
f"the by-name lookup is back, and with it the unguarded path: {u}"
|
||||
)
|
||||
|
||||
|
||||
def test_a_GLOB_rule_that_could_outrank_an_exact_one_wins_and_ASKS(hook):
|
||||
"""The arm ORDER, pinned. Without this the reorder is invisible to the suite — swapping the arms
|
||||
back left all 29 tests green, which is how an unproven change ships.
|
||||
|
||||
Gitea picks the governing rule with `GetFirstMatched` over a list sorted by Priority and THEN by
|
||||
plain-name-ness, so a glob rule can outrank an exactly-named one. Here `main` requires
|
||||
`review-verdict/h10` and `m*` does not. Evaluating `exact` first inspects the rule that requires
|
||||
h10, concludes the base is protected, and AUTO-GRANTS a scheduled merge onto a base where the
|
||||
check may not be enforced at all — #622's hole, reached through the block written to close it.
|
||||
|
||||
Evaluating `undecidable` first is sound without knowing Gitea's precedence rules, which is the
|
||||
only claim this code is entitled to make about somebody else's resolver.
|
||||
"""
|
||||
hook.set_branch_protection("EXACT-PLUS-GLOB-RULE")
|
||||
reason = hook.reason()
|
||||
assert "ask" in reason, (
|
||||
"an exact rule was trusted while a glob rule could outrank it — the gate granted on a base "
|
||||
"whose enforced rule it never identified"
|
||||
)
|
||||
assert "could govern it" in reason, (
|
||||
"the ask must come from the undecidable-glob arm, not from the classifier failing"
|
||||
)
|
||||
|
||||
|
||||
def test_two_rules_differing_only_in_CASE_are_undecidable(hook):
|
||||
"""`first` picks list order; Gitea picks by Priority. With `MAIN` requiring `review-verdict/h10`
|
||||
and `main` not, inspecting whichever the API happened to list first would auto-grant on a base
|
||||
whose enforced rule was never identified — the same defect as the arm order, one level down."""
|
||||
hook.set_branch_protection("TWO-CASE-VARIANT-RULES")
|
||||
reason = hook.reason()
|
||||
assert "ask" in reason, "two fold-equal rules disagree about review-verdict/h10 and one was picked by list order"
|
||||
assert "could govern it" in reason
|
||||
|
||||
|
||||
def test_a_NON_ASCII_rule_or_base_is_undecidable_rather_than_fold_compared(hook):
|
||||
"""`ascii_downcase` is not Gitea's Unicode-aware `EqualFold`, so a rule `ünstable` and a base
|
||||
`Ünstable` fold equal there and not here. The miss lands on `none`, which DENIES with the stated
|
||||
cause "none matches" — and the backslash arm already rejects "nearly unreachable"
|
||||
as a standard for the arm that issues a deny, so the same standard applies here."""
|
||||
hook.set_base("\u00dcnstable")
|
||||
hook.drop_recorded_base()
|
||||
hook.set_branch_protection("NONASCII-RULE")
|
||||
reason = hook.reason()
|
||||
assert "none matches" not in reason, (
|
||||
"a rule that folds equal to this base under EqualFold was reported as unable to govern it"
|
||||
)
|
||||
assert "ask" in reason
|
||||
|
||||
|
||||
def test_an_HTTP_404_on_the_LIST_read_asks_and_does_not_claim_the_list_was_read(hook):
|
||||
"""The `nomatch` sentinel, pinned — it shipped UNPINNED, and a full revert left the suite green.
|
||||
|
||||
Absence must be established by the CLASSIFIER over a list that was actually read, never by an
|
||||
HTTP status. Gitea answers 404 on this endpoint when the repo is absent or invisible to the
|
||||
credential, which says nothing about the base. Reusing 404 for the classifier's own
|
||||
nothing-can-govern verdict let that read reach the deny whose reason states "the full rule list
|
||||
was read and none matches" — a claim about a read that never happened.
|
||||
|
||||
No fixture emitted an HTTP 404 on the list before this test, which is exactly why reverting the
|
||||
sentinel to `bp_code=404` changed nothing observable.
|
||||
"""
|
||||
hook.set_branch_protection("LIST-404")
|
||||
reason = hook.reason()
|
||||
assert "ask" in reason, "an unreadable repo was treated as evidence about the base"
|
||||
assert "none matches" not in reason, "a 404 read claimed the full rule list had been read and matched nothing"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("shape", ["UNPARSEABLE-RULES", "GARBAGE", "EMPTY"])
|
||||
def test_a_200_the_classifier_cannot_PARSE_asks_without_blaming_the_transport(hook, shape):
|
||||
"""The twin of the `nomatch` fix, on the other arm — and pinned this time rather than assumed.
|
||||
|
||||
A rule whose `branch_name` is a number makes the classifier throw on a read that plainly
|
||||
succeeded. Mapping that to `bp_code=000` produced "could not read … (HTTP '000' — Gitea
|
||||
unreachable…)", stating a transport cause for a 200. The decision (ask) was always safe; only
|
||||
the reason lied, which is precisely the defect corrected one arm over for the deny.
|
||||
|
||||
Parametrised over all three shapes that reach a 200 the hook cannot use, because the first fix
|
||||
covered only `UNPARSEABLE-RULES` — the arm where it was noticed. `GARBAGE` (an object, not an
|
||||
array) and `EMPTY` are diverted one branch EARLIER, by the array gate, and kept `bp_code=200`,
|
||||
so they reported "HTTP '200' — Gitea unreachable" about a successful read. Fixing one arm and
|
||||
leaving its twin is the shape this PR is largely about.
|
||||
"""
|
||||
hook.set_branch_protection(shape)
|
||||
reason = hook.reason()
|
||||
assert "ask" in reason, "an unparseable rule list produced a decision instead of a question"
|
||||
assert "could not parse" in reason, "the ask blamed the transport for a 200 the classifier simply could not read"
|
||||
assert "unreachable" not in reason
|
||||
@@ -0,0 +1,464 @@
|
||||
"""Every `MUTATION` row of `docs/guard-inventory.md` is EXECUTED, not asserted (ersatztv#790).
|
||||
|
||||
The `MUTATION` grade means "a clause-level mutation was executed and this named test was witnessed
|
||||
red". Witnessed once, by hand, that is evidence about the day the row was written and nothing else:
|
||||
it decays as soon as the guard is edited, and a wrong grade has no way to announce itself.
|
||||
|
||||
So this file re-runs every one of them: for each declared mutation in `mutation_manifest.py`, apply
|
||||
it to an isolated copy of this repository and require the row's OWN named test to go red.
|
||||
|
||||
WHAT A GREEN RUN HERE DOES AND DOES NOT PROVE — stated because a mutation harness that overclaims is
|
||||
the same defect one level up:
|
||||
|
||||
* It proves the recorded proof ref names a test that EXISTS, still collects, and still reacts to
|
||||
the declared clause. That is precisely the decay #790 was filed about.
|
||||
* It proves the declared clause still occurs, exactly once, in the entry's declared `target` —
|
||||
which is not always the guard's own file. A clause that has been reworded fails here rather than
|
||||
silently mutating nothing.
|
||||
* Combined with the positive control below, it proves every named proof test is GREEN on the
|
||||
unmutated sandbox — so "the mutation was noticed" cannot be confused with "the test was already
|
||||
red". For the proof tests that perform a disarm of their own, that control also runs the disarm;
|
||||
the two named tests that are plain production set-equality checks have no disarm of their own,
|
||||
and this harness supplies theirs.
|
||||
* It does NOT prove the declared clause is the ONLY thing the guard hangs on. Some entries redden
|
||||
through the proof test's own "the clause has moved, RETARGET this" assertion rather than through
|
||||
changed behaviour. That is the intended reading, not a hole: those tests perform their
|
||||
behavioural disarm themselves on every green run, and what they could not do is notice their own
|
||||
clause reference going stale. Which entries those are is a dated measurement and lives in
|
||||
`docs/decisions/records/testing/mutation-claims-are-executed.md`, not here.
|
||||
|
||||
The `granularity` column is where this file refuses to flatter itself. See `Mutation` in
|
||||
`mutation_harness_lib.py`: all but one guard admits a single-clause mutation, and the one that does
|
||||
not CARRIES the finer mutation that survived, which is re-run and required to keep surviving.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib.util
|
||||
import re
|
||||
import shutil
|
||||
import sys
|
||||
from dataclasses import replace
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from scripts.tests.mutation_harness_lib import (
|
||||
_BASELINES,
|
||||
Mutation,
|
||||
_git,
|
||||
build_sandbox,
|
||||
reset_sandbox,
|
||||
run_pytest,
|
||||
verify_mutation,
|
||||
)
|
||||
from scripts.tests.mutation_manifest import MUTATIONS, UNDECLARED
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[2]
|
||||
INVENTORY = REPO_ROOT / "docs" / "guard-inventory.md"
|
||||
|
||||
# `| `guard` | blocks | kind | proof | proof ref |` — the five-column inventory row. The column
|
||||
# count is what excludes the two-column tables elsewhere in the file; `test_the_inventory_rows_parse`
|
||||
# below is the floor that catches the shape changing under it.
|
||||
_ROW = re.compile(r"^\|\s*`([^`]+)`\s*\|([^|]*)\|\s*(\w[\w-]*)\s*\|\s*([A-Z-]+)\s*\|\s*(.*?)\s*\|\s*$", re.M)
|
||||
|
||||
|
||||
def _inventory_rows() -> list[tuple[str, str, str, str]]:
|
||||
"""(guard, kind, proof grade, proof ref) for every inventory row.
|
||||
|
||||
Derived from the document the grades live in, never hand-listed: the whole point is that a row
|
||||
cannot change its grade without this file noticing.
|
||||
"""
|
||||
rows = [(m.group(1), m.group(3), m.group(4), m.group(5).strip("`")) for m in _ROW.finditer(INVENTORY.read_text())]
|
||||
guards = [g for g, _k, _p, _r in rows]
|
||||
duplicates = sorted({g for g in guards if guards.count(g) > 1})
|
||||
assert not duplicates, (
|
||||
f"the inventory lists {duplicates} more than once. Every comparison below reduces rows "
|
||||
"through a set or a dict, so a duplicate row is INVISIBLE to this file — it would report all "
|
||||
"entries verified over a table that contradicts itself. `test_guard_inventory.py` catches "
|
||||
"this too, but only when the whole suite runs, and this file is routinely run alone."
|
||||
)
|
||||
assert len(rows) >= 50, (
|
||||
f"only parsed {len(rows)} inventory rows — the table's shape has changed and this regex now "
|
||||
"reads a fraction of it. Every set comparison below would then pass over a population that "
|
||||
"is mostly missing, which is the vacuous-completeness failure the inventory itself exists to "
|
||||
"prevent."
|
||||
)
|
||||
return rows
|
||||
|
||||
|
||||
# ------------------------------------------------------------------------------------------------
|
||||
# THE MANIFEST IS PINNED TO THE INVENTORY — a grade cannot change without this file changing
|
||||
# ------------------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_the_manifest_covers_exactly_the_MUTATION_rows():
|
||||
"""Set equality, both directions, because they are different defects.
|
||||
|
||||
A `MUTATION` row with no manifest entry is a claim nothing checks — the state #790 was filed
|
||||
about. A manifest entry for a row that is no longer graded `MUTATION` is a check whose subject
|
||||
has moved out from under it, and it would keep passing.
|
||||
"""
|
||||
graded = {guard for guard, _kind, grade, _ref in _inventory_rows() if grade == "MUTATION"}
|
||||
declared = {m.guard for m in MUTATIONS}
|
||||
assert graded, "no row in the inventory is graded MUTATION, so this whole file would prove nothing"
|
||||
assert declared - graded == set(), (
|
||||
f"declared mutations for rows that are not graded MUTATION: {sorted(declared - graded)}"
|
||||
)
|
||||
assert graded - declared == set(), (
|
||||
f"these rows claim a MUTATION proof with nothing executing it: {sorted(graded - declared)}. "
|
||||
"Declare the clause in mutation_manifest.py, or regrade the row."
|
||||
)
|
||||
assert len(MUTATIONS) == len(declared), "two manifest entries name the same guard"
|
||||
|
||||
|
||||
def test_every_declared_mutation_names_the_row_s_OWN_proof_ref():
|
||||
"""The manifest may not point at a different test than the row does.
|
||||
|
||||
Without this, the inventory could keep citing a stale proof while the harness quietly exercised
|
||||
a healthier one, and the row would read as verified.
|
||||
"""
|
||||
refs = {guard: ref for guard, _kind, grade, ref in _inventory_rows() if grade == "MUTATION"}
|
||||
wrong = [(m.guard, m.proof, refs.get(m.guard)) for m in MUTATIONS if refs.get(m.guard) != m.proof]
|
||||
assert not wrong, f"manifest proof ref disagrees with the inventory row: {wrong}"
|
||||
|
||||
|
||||
def test_every_GUARD_row_is_either_DECLARED_or_STATED_here():
|
||||
"""`Done-when`: guards whose mutation cannot be declared are STATED, not silently skipped.
|
||||
|
||||
Set equality against the inventory's GUARD rows, both directions, keyed on the guard. Keying on
|
||||
the row's GRADE instead would be cheaper and tautological — a new guard graded NONE would inherit
|
||||
a reason automatically and nobody would look at it — and a COUNT moves only on net change, so one
|
||||
guard arriving as another is promoted leaves it unchanged. This is the same hand-maintained,
|
||||
machine-checked shape as `docs/guard-inventory.md` itself, which is what makes it safe.
|
||||
|
||||
The partition covers `Kind == GUARD` rows only. `TOOLING` asserts nothing and `PROOF` files exist
|
||||
to prove other guards, so neither carries a mutation claim to verify — a rule about the Kind
|
||||
column rather than a list anyone maintains.
|
||||
"""
|
||||
guards = {guard for guard, kind, _grade, _ref in _inventory_rows() if kind == "GUARD"}
|
||||
declared = {m.guard for m in MUTATIONS}
|
||||
stated = set(UNDECLARED)
|
||||
|
||||
assert not (declared & stated), (
|
||||
f"these guards are both declared and stated as undeclared: {sorted(declared & stated)}"
|
||||
)
|
||||
unaccounted = sorted(guards - declared - stated)
|
||||
assert not unaccounted, (
|
||||
f"these guards are neither declared nor stated: {unaccounted}. Declare the clause in "
|
||||
"mutation_manifest.py, or write a line in UNDECLARED saying what a proof would need — "
|
||||
"silence is the one option that is not available."
|
||||
)
|
||||
orphaned = sorted(stated - guards)
|
||||
assert not orphaned, (
|
||||
f"UNDECLARED names rows that are not GUARD-kind rows any more: {orphaned}. A reason for a "
|
||||
"guard that no longer exists reads as coverage."
|
||||
)
|
||||
thin = [g for g, reason in UNDECLARED.items() if len(reason.strip()) < 60]
|
||||
assert not thin, f"these UNDECLARED entries say nothing a reader could act on: {thin}"
|
||||
|
||||
|
||||
def test_every_entry_declares_a_known_granularity_and_DETECTOR_entries_CARRY_their_survivor():
|
||||
"""`DETECTOR` is an admission, and an unevidenced one would be a grading curve.
|
||||
|
||||
#790's complaint about `pin_population_faults` — that neutering a whole helper is "coarse enough
|
||||
that a single surviving clause would not be noticed" — applies to every entry graded here. So an
|
||||
entry may only claim `DETECTOR` while carrying the finer mutation that was tried, as data rather
|
||||
than as a sentence: `test_every_SURVIVING_clause_mutation_still_does` then runs it.
|
||||
"""
|
||||
bad = [m.guard for m in MUTATIONS if m.granularity not in (Mutation.CLAUSE, Mutation.DETECTOR)]
|
||||
assert not bad, f"unknown granularity on {bad}; allowed: CLAUSE, DETECTOR"
|
||||
assert all(m.why.strip() for m in MUTATIONS), "every declared mutation must say what its clause does"
|
||||
|
||||
unevidenced = [m.guard for m in MUTATIONS if m.granularity == Mutation.DETECTOR and not m.survived_clause]
|
||||
assert not unevidenced, (
|
||||
f"these entries claim DETECTOR granularity while naming no finer mutation that was tried: "
|
||||
f"{unevidenced}. Carry the survivor, or declare the finer clause instead."
|
||||
)
|
||||
misplaced = [m.guard for m in MUTATIONS if m.granularity == Mutation.CLAUSE and m.survived_clause]
|
||||
assert not misplaced, (
|
||||
f"these entries are graded CLAUSE but carry a surviving finer mutation: {misplaced}. If a "
|
||||
"finer clause exists and survives, the grade is DETECTOR."
|
||||
)
|
||||
half = [m.guard for m in MUTATIONS if bool(m.survived_clause) != bool(m.survived_replacement)]
|
||||
assert not half, f"a survivor needs both a clause and a replacement: {half}"
|
||||
|
||||
|
||||
def test_every_entry_declares_a_SPECIFIC_diagnostic_it_must_redden_with():
|
||||
"""`expect` is what stops exit code 1 from being the whole verdict, so it cannot be a token.
|
||||
|
||||
An empty or near-empty expectation matches any output and hands the verdict straight back to the
|
||||
exit status — the state this field exists to leave. It must also be a substring of no other
|
||||
entry's, or two rows could be satisfied by one another's diagnostic.
|
||||
"""
|
||||
vague = [(m.guard, m.expect) for m in MUTATIONS if len(m.expect.strip()) < 20]
|
||||
assert not vague, f"these expectations are too weak to distinguish one red from another: {vague}"
|
||||
for m in MUTATIONS:
|
||||
clashes = [o.guard for o in MUTATIONS if o is not m and m.expect in o.expect]
|
||||
assert not clashes, f"{m.guard}'s expectation is contained in {clashes}'s — neither is specific"
|
||||
|
||||
|
||||
# ------------------------------------------------------------------------------------------------
|
||||
# THE EXECUTION — one isolated repository, reused, reset between mutations
|
||||
# ------------------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def sandbox(tmp_path_factory):
|
||||
"""An isolated copy of this repository, with the POSITIVE CONTROL already run in it.
|
||||
|
||||
The control is inside the fixture rather than in a test of its own so ordering is a dependency
|
||||
rather than a convention: no mutation can be judged before every proof test has been shown green
|
||||
on the unmutated tree. Without it, "the named test went red" is satisfied just as well by a proof
|
||||
test that was already broken, and reporting that as a verified mutation is the failure mode this
|
||||
whole file exists to remove.
|
||||
"""
|
||||
sb = build_sandbox(tmp_path_factory.mktemp("mutation-sandbox"))
|
||||
# The sandbox holds what GIT TRACKS, so a proof test in a file that has never been `git add`ed is
|
||||
# simply absent there and the run below reports "file or directory not found" — accurate, and
|
||||
# unreadable as a diagnosis. Named here instead, because this is what a developer adding a guard
|
||||
# hits first.
|
||||
missing = sorted({m.node_id.split("::")[0] for m in MUTATIONS if not (sb / m.node_id.split("::")[0]).is_file()})
|
||||
assert not missing, (
|
||||
f"these proof files are not in the sandbox: {missing}. The sandbox is derived from "
|
||||
"`git ls-files`, so an unstaged new file is not in it — `git add` it and re-run."
|
||||
)
|
||||
result = run_pytest(sb, [m.node_id for m in MUTATIONS])
|
||||
assert result.returncode == 0, (
|
||||
"the proof tests named by the inventory are NOT green on an unmutated copy of this "
|
||||
"repository, so nothing below can distinguish 'the mutation was noticed' from 'the test was "
|
||||
f"already red'. Fix them first.\n{result.stdout[-4000:]}{result.stderr[-2000:]}"
|
||||
)
|
||||
assert "passed" in result.stdout, f"the control run collected nothing: {result.stdout!r}"
|
||||
try:
|
||||
yield sb
|
||||
finally:
|
||||
# ~115 MiB of tracked files plus its own git objects. pytest keeps the last three sessions'
|
||||
# tmp dirs by default, so leaving it costs a third of a gigabyte on a developer machine that
|
||||
# runs this a few times.
|
||||
shutil.rmtree(sb, ignore_errors=True)
|
||||
|
||||
|
||||
def test_the_reset_restores_the_BASELINE_even_after_a_proof_COMMITS(sandbox):
|
||||
"""One sandbox serves every mutation, so the reset has to be a reset to a fixed point.
|
||||
|
||||
`git reset --hard` with no argument resets to whatever HEAD currently is. A proof test that
|
||||
commits inside the sandbox — several drive `git commit` for real — moves HEAD onto a commit
|
||||
carrying whatever was in the tree at the time, and every later reset would faithfully restore
|
||||
THAT. The contamination surfaces as an unrelated red several mutations further on, which is the
|
||||
hardest kind of harness defect to attribute.
|
||||
"""
|
||||
subject = sandbox / "docs" / "guard-inventory.md"
|
||||
baseline = subject.read_text()
|
||||
|
||||
subject.write_text(baseline + "\n<!-- planted, then COMMITTED -->\n")
|
||||
_git(sandbox, "add", "-A")
|
||||
_git(sandbox, "commit", "-qm", "a proof test committing inside the sandbox")
|
||||
assert subject.read_text() != baseline, "the planted change did not land, so this proves nothing"
|
||||
|
||||
reset_sandbox(sandbox)
|
||||
|
||||
assert subject.read_text() == baseline, (
|
||||
"the reset restored the sandbox to a commit made DURING a mutation rather than to the "
|
||||
"pristine baseline, so every later verdict is computed against a contaminated tree"
|
||||
)
|
||||
# Through `_git`, not a raw `subprocess.run`: an ambient `GIT_DIR` — which a git hook exports,
|
||||
# and this suite runs from one — would resolve this against the REAL repository and compare two
|
||||
# commits that have nothing to do with the sandbox. Asserting isolation with an unisolated call
|
||||
# is the defect this harness exists to catch, one level up.
|
||||
head = _git(sandbox, "rev-parse", "HEAD").stdout.decode().strip()
|
||||
assert head == _BASELINES[str(sandbox.resolve())], f"HEAD was left off the recorded baseline commit: {head}"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("mutation", MUTATIONS, ids=lambda m: m.guard)
|
||||
def test_MUTATION_the_declared_clause_reddens_the_named_proof(sandbox, mutation):
|
||||
reset_sandbox(sandbox)
|
||||
verdict = verify_mutation(sandbox, mutation)
|
||||
assert verdict.ok, f"{mutation.guard}: {verdict.reason}"
|
||||
|
||||
|
||||
_SURVIVORS = tuple(m for m in MUTATIONS if m.granularity == Mutation.DETECTOR)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("mutation", _SURVIVORS, ids=lambda m: m.guard)
|
||||
def test_every_SURVIVING_clause_mutation_still_does(sandbox, mutation):
|
||||
"""The DETECTOR grade, executed rather than recited.
|
||||
|
||||
A finer mutation that has since STARTED reddening the proof test means the guard now admits
|
||||
clause-level proof and the entry should be regraded — the coarse grade would otherwise persist as
|
||||
an excuse long after the reason for it went away. Failing here is therefore good news; it just
|
||||
has to be acted on.
|
||||
"""
|
||||
reset_sandbox(sandbox)
|
||||
finer = replace(
|
||||
mutation,
|
||||
clause=mutation.survived_clause,
|
||||
replacement=mutation.survived_replacement,
|
||||
survived_clause="",
|
||||
survived_replacement="",
|
||||
)
|
||||
verdict = verify_mutation(sandbox, finer)
|
||||
assert not verdict.ok, (
|
||||
f"{mutation.guard} is graded DETECTOR because {mutation.survived_clause!r} was tried and left "
|
||||
"the proof test green — but it reddens it now. Regrade the entry to CLAUSE with that mutation."
|
||||
)
|
||||
assert "still PASSED" in verdict.reason, (
|
||||
f"the survivor did not survive for the recorded reason — it failed with: {verdict.reason}"
|
||||
)
|
||||
|
||||
|
||||
# ------------------------------------------------------------------------------------------------
|
||||
# THIS GUARD'S OWN MUTATION PROOF — the redness clause, disarmed, on a sandbox of two files
|
||||
# ------------------------------------------------------------------------------------------------
|
||||
|
||||
EXIT_STATUS_CLAUSE = " if result.returncode != 1:"
|
||||
DIAGNOSTIC_CLAUSE = " if mutation.expect not in diagnostic:"
|
||||
LIB = REPO_ROOT / "scripts" / "tests" / "mutation_harness_lib.py"
|
||||
|
||||
|
||||
def _inert_sandbox(tmp_path: Path, body: str = " assert True\n") -> tuple[Path, Mutation]:
|
||||
"""A minimal synthetic sandbox holding one test file, whose outcome is fixed by `body`.
|
||||
|
||||
Deliberately not a copy of the repo: the subject here is `verify_mutation`'s verdict, and a real
|
||||
sandbox would cost four seconds to prove something about two lines of control flow.
|
||||
"""
|
||||
tests = tmp_path / "scripts" / "tests"
|
||||
tests.mkdir(parents=True)
|
||||
(tests / "test_inert.py").write_text(f"# INERT MARKER\n\n\ndef test_ok():\n{body}")
|
||||
return tmp_path, Mutation(
|
||||
guard="inert",
|
||||
target="scripts/tests/test_inert.py",
|
||||
clause="# INERT MARKER",
|
||||
replacement="# INERT MARKER, CHANGED",
|
||||
proof="test_inert.py::test_ok",
|
||||
granularity=Mutation.CLAUSE,
|
||||
# A legal expectation, not the empty string: an empty one is a shape the manifest forbids,
|
||||
# so a fixture relying on it would be proving something about a configuration that cannot
|
||||
# ship. Each caller that needs a different one passes it through `replace`.
|
||||
expect="a diagnostic no run of this fixture produces",
|
||||
why="an inert mutation: the named test cannot notice it",
|
||||
)
|
||||
|
||||
|
||||
def _lib_with(tmp_path: Path, clause: str, replacement: str, label: str):
|
||||
"""Import a copy of the library with one clause replaced.
|
||||
|
||||
The copy is registered in `sys.modules` before execution: `@dataclass` resolves a string
|
||||
annotation through `sys.modules[cls.__module__]`, so an unregistered module raises AttributeError
|
||||
on the first dataclass it defines rather than on anything to do with the clause.
|
||||
"""
|
||||
source = LIB.read_text()
|
||||
assert source.count(clause) == 1, (
|
||||
f"the {label} has moved or been reworded; RETARGET this mutation rather than loosening the "
|
||||
"match — and update the mutation_manifest entry for this file, which names the same string"
|
||||
)
|
||||
path = tmp_path / f"mutant_{label.replace(' ', '_')}.py"
|
||||
path.write_text(source.replace(clause, replacement, 1))
|
||||
|
||||
spec = importlib.util.spec_from_file_location(path.stem, path)
|
||||
mutant = importlib.util.module_from_spec(spec)
|
||||
sys.modules[spec.name] = mutant
|
||||
try:
|
||||
spec.loader.exec_module(mutant)
|
||||
finally:
|
||||
sys.modules.pop(spec.name, None)
|
||||
return mutant
|
||||
|
||||
|
||||
def test_an_INERT_mutation_is_REPORTED_rather_than_passed(tmp_path):
|
||||
"""The behavioural half. A harness that cannot tell a real disarm from a comment edit would
|
||||
report every row verified on any tree at all."""
|
||||
sb, inert = _inert_sandbox(tmp_path)
|
||||
verdict = verify_mutation(sb, inert)
|
||||
assert not verdict.ok, "a mutation the named test cannot possibly notice was reported as verified"
|
||||
assert "still PASSED" in verdict.reason, verdict.reason
|
||||
|
||||
|
||||
def test_a_red_for_the_WRONG_REASON_is_not_accepted(tmp_path):
|
||||
"""The gate `expect` exists for. Pytest reports an ordinary exception exactly as it reports a
|
||||
failed assertion, so a mutation that CRASHES the proof test looks identical to one it detected.
|
||||
A verdict that cannot tell those apart certifies rows on evidence about nothing."""
|
||||
sb, inert = _inert_sandbox(tmp_path, " raise RuntimeError('an unrelated crash')\n")
|
||||
crashing = replace(inert, expect="the diagnostic this row is supposed to produce")
|
||||
verdict = verify_mutation(sb, crashing)
|
||||
assert not verdict.ok, "a red with nothing to do with the declared diagnostic was accepted"
|
||||
assert "NOT with the declared diagnostic" in verdict.reason, verdict.reason
|
||||
|
||||
|
||||
def test_MUTATION_disarming_the_DIAGNOSTIC_gate_accepts_a_red_for_the_wrong_reason(tmp_path):
|
||||
"""The clause-level proof, on the newer of the two gates every verdict passes through.
|
||||
|
||||
Disarm the check that the failure carries the declared diagnostic, and the crashing case above
|
||||
must start reporting as verified. If it does not, that rejection is coming from somewhere other
|
||||
than the clause, and the test above proves nothing about it.
|
||||
"""
|
||||
sb, inert = _inert_sandbox(tmp_path, " raise RuntimeError('an unrelated crash')\n")
|
||||
crashing = replace(inert, expect="the diagnostic this row is supposed to produce")
|
||||
assert not verify_mutation(sb, crashing).ok, (
|
||||
"the UNMUTATED verdict already accepted it, so the mutant proves nothing"
|
||||
)
|
||||
|
||||
mutant = _lib_with(tmp_path, DIAGNOSTIC_CLAUSE, " if False:", "diagnostic gate")
|
||||
assert mutant.verify_mutation(sb, crashing).ok, (
|
||||
"the diagnostic gate was replaced with a constant and a red for an unrelated reason was "
|
||||
"STILL rejected, so the verdict does not hang on the clause that reads it"
|
||||
)
|
||||
|
||||
|
||||
def test_MUTATION_disarming_the_EXIT_STATUS_gate_accepts_a_run_that_NEVER_RAN_A_TEST(tmp_path):
|
||||
"""The same proof for the older gate, and it has to be built carefully to isolate it.
|
||||
|
||||
A GREEN run cannot serve: it produces no exception output, so only an EMPTY expectation would
|
||||
reach the status gate — and an empty expectation is a shape the manifest forbids, which would
|
||||
make this a proof about a configuration that cannot ship. Instead the sandbox's test file fails
|
||||
at IMPORT: pytest exits non-1 (nothing was collected, so nothing ran) while still printing the
|
||||
exception, so a legal non-empty expectation matches and the exit status is the ONLY thing
|
||||
rejecting it. That is the case this gate exists for — a proof ref that no longer names a
|
||||
collectable test must not read as a guard going red.
|
||||
"""
|
||||
boom = "a deliberate import-time failure, which is not a test result"
|
||||
sb, inert = _inert_sandbox(tmp_path)
|
||||
(sb / "scripts" / "tests" / "test_inert.py").write_text(f"# INERT MARKER\nraise RuntimeError({boom!r})\n")
|
||||
uncollectable = replace(inert, expect=boom)
|
||||
|
||||
verdict = verify_mutation(sb, uncollectable)
|
||||
assert not verdict.ok, "a run in which no test executed was accepted as a guard going red"
|
||||
assert len(uncollectable.expect) >= 20, "the expectation must be one the manifest would accept"
|
||||
|
||||
mutant = _lib_with(tmp_path, EXIT_STATUS_CLAUSE, " if False:", "exit status gate")
|
||||
assert mutant.verify_mutation(sb, uncollectable).ok, (
|
||||
"the exit-status gate was replaced with a constant and a run that never executed a test was "
|
||||
"still rejected, so the verdict does not hang on the status it reads"
|
||||
)
|
||||
|
||||
|
||||
def test_a_clause_that_no_longer_OCCURS_ONCE_is_reported_rather_than_applied(tmp_path):
|
||||
"""The failure paths, driven directly, because this is where a harness quietly stops harnessing.
|
||||
|
||||
A clause that has been reworded away, or that now matches a second site, must produce a verdict
|
||||
naming the problem. Silently replacing nothing — or replacing the wrong site — would leave every
|
||||
row in this file reporting verified while mutating something nobody declared.
|
||||
"""
|
||||
sb, inert = _inert_sandbox(tmp_path)
|
||||
|
||||
gone = replace(inert, clause="# A CLAUSE THAT IS NOT THERE")
|
||||
assert not verify_mutation(sb, gone).ok
|
||||
assert "occurs 0 times" in verify_mutation(sb, gone).reason
|
||||
|
||||
(sb / "scripts" / "tests" / "test_inert.py").write_text(
|
||||
"# INERT MARKER\n# INERT MARKER\n\n\ndef test_ok():\n assert True\n"
|
||||
)
|
||||
assert "occurs 2 times" in verify_mutation(sb, inert).reason
|
||||
|
||||
absent = replace(inert, target="scripts/tests/no_such_file.py")
|
||||
assert "does not exist in the sandbox" in verify_mutation(sb, absent).reason
|
||||
|
||||
|
||||
def test_the_sandbox_is_left_UNCHANGED_by_a_verdict(tmp_path):
|
||||
"""One sandbox serves every mutation, so a verdict that leaves its edit behind would make each
|
||||
result a function of the ones before it."""
|
||||
sb, inert = _inert_sandbox(tmp_path)
|
||||
subject = sb / "scripts" / "tests" / "test_inert.py"
|
||||
before = subject.read_text()
|
||||
verify_mutation(sb, inert)
|
||||
assert subject.read_text() == before, "verify_mutation left its mutation in the sandbox"
|
||||
@@ -73,6 +73,10 @@ if "/pulls/" in url and not url.endswith("/files"):
|
||||
"state": (state / "pr_state").read_text().strip(),
|
||||
"html_url": "http://gitea.example/timothy/ersatztv/pulls/42",
|
||||
}
|
||||
# A 2xx body that merely LOST the field, as distinct from an unreachable PR ('GONE' above).
|
||||
# This is the shape the `[ -n "$x" ] &&` conjunct used to wave through (ersatztv#778).
|
||||
if sha == "NOHEAD":
|
||||
body["head"] = {}
|
||||
if base != "MISSING":
|
||||
body["base"] = {"ref": base}
|
||||
print(json.dumps(body))
|
||||
@@ -100,7 +104,7 @@ def gitea(tmp_path):
|
||||
env = dict(os.environ)
|
||||
env["PATH"] = f"{bindir}{os.pathsep}{env['PATH']}"
|
||||
env["STUB_DIR"] = str(state)
|
||||
env["ETV_GITEA_TOKEN"] = "stub-token"
|
||||
env["ETV_GITEA_TOKEN"] = "stub-token" # noqa: S105 - deliberately fake; the real credential comes from the environment
|
||||
env["ETV_GITEA_URL"] = "http://gitea.example"
|
||||
env["ETV_GITEA_REPO"] = "timothy/ersatztv"
|
||||
env.pop("ETV_GITEA_BASICAUTH", None)
|
||||
@@ -123,7 +127,9 @@ def gitea(tmp_path):
|
||||
def run(self, *args):
|
||||
return subprocess.run(
|
||||
["bash", str(SCRIPT), *args],
|
||||
env=env, capture_output=True, text=True,
|
||||
env=env,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
|
||||
def posts(self):
|
||||
@@ -225,6 +231,7 @@ def test_unreachable_pr_is_an_error_not_a_silent_success(gitea):
|
||||
|
||||
# --- Cross-checks against the hook's own condition-(c) parser -----------------------------------
|
||||
|
||||
|
||||
def _classify(body: str, head: str) -> str:
|
||||
"""Run the REAL H10 classifier over a comment body — no Python mirror of the grammar.
|
||||
|
||||
@@ -235,9 +242,7 @@ def _classify(body: str, head: str) -> str:
|
||||
passing here while the shell drifted.
|
||||
"""
|
||||
payload = json.dumps([{"body": body}])
|
||||
p = subprocess.run(
|
||||
["bash", str(CLASSIFIER), "--head", head], input=payload, capture_output=True, text=True
|
||||
)
|
||||
p = subprocess.run(["bash", str(CLASSIFIER), "--head", head], input=payload, capture_output=True, text=True)
|
||||
assert p.returncode == 0, f"classifier errored: {p.stderr}"
|
||||
return p.stdout.strip()
|
||||
|
||||
@@ -276,6 +281,7 @@ def test_note_cannot_forge_a_second_verdict_line(gitea):
|
||||
# mirror case: the head sha and the status both hold still while the effective DIFF changes, so the
|
||||
# verdict keeps reading green for a review nobody performed against that base.
|
||||
|
||||
|
||||
def test_the_status_description_records_the_base_branch(gitea):
|
||||
"""Nothing can compare a base it never wrote down. This field is what the hook reads back."""
|
||||
assert gitea.run("42", "MERGEABLE").returncode == 0
|
||||
@@ -329,4 +335,80 @@ def test_a_failed_HEAD_RECHECK_writes_no_status(gitea):
|
||||
result = gitea.run("42", "MERGEABLE")
|
||||
assert result.returncode != 0
|
||||
assert gitea.statuses() == [], (
|
||||
"a status was written even though the head/base re-read failed — nothing was confirmed")
|
||||
"a status was written even though the head/base re-read failed — nothing was confirmed"
|
||||
)
|
||||
|
||||
|
||||
# --- write-side polarity, the half the #774 rescue initially missed ------------------------------
|
||||
|
||||
|
||||
# Kept deliberately in step with POSITIVE_WORDS/NEGATIVE_WORDS in test_check_review_verdict.py.
|
||||
# The two lists are NOT compared to each other, and this file does not claim they agree — proving
|
||||
# that needs one shared vocabulary both scripts read, which is ersatztv#788. What each side proves
|
||||
# is its OWN polarity: that these established tokens still map the way reviewers rely on.
|
||||
WRITE_POSITIVE = ["MERGEABLE", "APPROVED", "LGTM"]
|
||||
WRITE_NEGATIVE = ["BLOCKED", "NOT-MERGEABLE"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("word", "expected"),
|
||||
[(w, "success") for w in WRITE_POSITIVE] + [(w, "failure") for w in WRITE_NEGATIVE],
|
||||
)
|
||||
def test_each_verdict_word_posts_its_established_polarity(word, expected, gitea):
|
||||
"""The WRITE side of the polarity pair, added after cold review found only the read side.
|
||||
|
||||
ersatztv#774 withdrew a test that compared the two verdict vocabularies by parsing shell, and
|
||||
rescued its disjointness half into test_check_review_verdict.py. Review of that rescue found it
|
||||
covered only `check-review-verdict.sh`: an in-memory mutation adding `BLOCKED` to the SUCCESS
|
||||
arm here produced a write-side overlap the deleted test caught and the rescue did not, because
|
||||
the rescue never executes this script. That was a real, undisclosed loss and this closes it.
|
||||
|
||||
`case` takes the FIRST matching arm, so a token listed in both arms is not ambiguous — it
|
||||
resolves to whichever comes first, exactly as `is_pos` wins on the read side. The consequence is
|
||||
the same and it is the one that matters: a word a reviewer means as BLOCKED silently posting
|
||||
`success` writes a green `review-verdict/h10`, which is the required context branch protection
|
||||
honours. That flips this assertion.
|
||||
|
||||
What this does NOT prove, stated because the parent test was withdrawn for overclaiming: it is
|
||||
a polarity regression over five established tokens, not a universal disjointness property and
|
||||
not parity with the read side. A token added to only one script is untested here, not caught.
|
||||
"""
|
||||
result = gitea.run("42", word)
|
||||
assert result.returncode == 0, result.stderr
|
||||
payload = gitea.statuses()[0]["payload"]
|
||||
assert payload["state"] == expected, (
|
||||
f"'{word}' posted state {payload['state']!r}, expected {expected!r}. `case` takes the first "
|
||||
"matching arm, so a token that has appeared in the other arm resolves there silently — and "
|
||||
"a BLOCKED verdict posting `success` writes a green required context."
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("head_seq", "base_seq", "field"),
|
||||
[
|
||||
(("NOHEAD",), ("main", "main"), "head sha"),
|
||||
((SHA_A,), ("main", "MISSING"), "base ref"),
|
||||
],
|
||||
)
|
||||
def test_a_reread_that_LOSES_a_field_refuses_instead_of_posting(head_seq, base_seq, field, gitea):
|
||||
"""The fail-OPEN one level below the TOCTOU guard, found by cold review (ersatztv#778).
|
||||
|
||||
Both re-read checks were written as `[ -n "$x" ] && [ "$x" != "$want" ]`. That conjunct makes an
|
||||
EMPTY value a no-op: a well-formed 2xx response that merely omits `.head.sha` or `.base.ref`
|
||||
yields an empty variable, neither comparison runs, and the status is posted having confirmed
|
||||
NOTHING about the head or the base — while the script's whole purpose at that point is to refuse
|
||||
unless it can confirm. The transport failure one line above was already fatal, which is exactly
|
||||
what made this shape easy to miss: the loud case was handled and the quiet one was not.
|
||||
|
||||
Asserted on the OBSERVABLE outcome — no status written — rather than on message text, so it
|
||||
still holds if the wording changes.
|
||||
"""
|
||||
gitea.set_head_sequence(SHA_A, *head_seq)
|
||||
gitea.set_base_sequence(*base_seq)
|
||||
result = gitea.run("42", "MERGEABLE")
|
||||
assert result.returncode != 0, (
|
||||
f"a re-read missing its {field} was accepted; the script posted a verdict having confirmed nothing about it"
|
||||
)
|
||||
assert not gitea.statuses(), (
|
||||
f"a status was written despite the re-read carrying no {field} — this is the fail-open the -n conjunct created"
|
||||
)
|
||||
|
||||
+1261
-284
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,186 @@
|
||||
"""Tests for the tag-only-push exemption in `.claude/hooks/prepush-rebase-check.sh` (ersatztv#719).
|
||||
|
||||
H11 refuses to push a branch that is behind `origin/main`, to force a rebase instead of a merge.
|
||||
But the release cut tags a commit on `main` from a branch that is behind `origin/main`, so H11
|
||||
blocked every release -- and its "rebase first" advice did not even apply, because no branch was
|
||||
being pushed. (Observed while cutting v26.13.0; see #719. `docs/ci-cd.md` -> "Cutting a release"
|
||||
documents the tag step itself, not the release-notes-PR flow that puts the branch behind.) A tag
|
||||
push cannot revert anyone's merged work (the failure mode H11 exists to prevent), so the fix skips
|
||||
the freshness check when EVERY ref being pushed is under `refs/tags/`.
|
||||
|
||||
These tests use real local git repositories (a bare "origin" plus a work tree pushed one commit
|
||||
behind it) rather than stubbing `git`, because the hook's decision hinges on genuine
|
||||
`git fetch` / `merge-base` / `rev-list` behavior against an origin that has moved.
|
||||
|
||||
`test_zero_ref_lines_does_not_exempt` is the load-bearing negative case from the issue: "all pushed
|
||||
refs are tags" is vacuously true over zero ref lines, so a naive implementation would disable H11
|
||||
entirely whenever stdin is empty (hook run manually, or a caller that forgot to forward it). The fix
|
||||
must require at least one parsed ref line before granting the exemption.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[2]
|
||||
HOOK = REPO_ROOT / ".claude" / "hooks" / "prepush-rebase-check.sh"
|
||||
|
||||
DUMMY_SHA_A = "a" * 40
|
||||
DUMMY_SHA_B = "b" * 40
|
||||
|
||||
|
||||
def _git(args, cwd):
|
||||
r = subprocess.run(["git", *args], cwd=str(cwd), capture_output=True, text=True)
|
||||
assert r.returncode == 0, f"git {' '.join(args)} failed: {r.stderr}"
|
||||
return r.stdout
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def behind_repo(tmp_path):
|
||||
"""A work tree whose local `main` is exactly one commit behind `origin/main`."""
|
||||
origin = tmp_path / "origin.git"
|
||||
_git(["init", "--bare", "-q", str(origin)], cwd=tmp_path)
|
||||
|
||||
work = tmp_path / "work"
|
||||
_git(["init", "-q", "-b", "main", str(work)], cwd=tmp_path)
|
||||
_git(["config", "user.email", "test@example.com"], cwd=work)
|
||||
_git(["config", "user.name", "Test"], cwd=work)
|
||||
(work / "f.txt").write_text("one\n")
|
||||
_git(["add", "f.txt"], cwd=work)
|
||||
_git(["commit", "-q", "-m", "initial"], cwd=work)
|
||||
_git(["remote", "add", "origin", str(origin)], cwd=work)
|
||||
_git(["push", "-q", "-u", "origin", "main"], cwd=work)
|
||||
# The bare repo's HEAD symref still points at the (nonexistent) default branch until something
|
||||
# sets it explicitly; without this, `git clone` below checks out an unborn HEAD and "main" never
|
||||
# exists as a local branch in `advancer`.
|
||||
_git(["symbolic-ref", "HEAD", "refs/heads/main"], cwd=origin)
|
||||
|
||||
# Advance origin/main independently, via a second clone, so `work`'s local `main` falls behind.
|
||||
advancer = tmp_path / "advancer"
|
||||
_git(["clone", "-q", str(origin), str(advancer)], cwd=tmp_path)
|
||||
_git(["config", "user.email", "test@example.com"], cwd=advancer)
|
||||
_git(["config", "user.name", "Test"], cwd=advancer)
|
||||
(advancer / "f.txt").write_text("two\n")
|
||||
_git(["add", "f.txt"], cwd=advancer)
|
||||
_git(["commit", "-q", "-m", "advance"], cwd=advancer)
|
||||
_git(["push", "-q", "origin", "main"], cwd=advancer)
|
||||
|
||||
return work
|
||||
|
||||
|
||||
def _run_hook(cwd, stdin_text):
|
||||
env = dict(os.environ)
|
||||
for k in ("GIT_DIR", "GIT_WORK_TREE", "GIT_INDEX_FILE"):
|
||||
env.pop(k, None)
|
||||
return subprocess.run(
|
||||
["bash", str(HOOK)],
|
||||
cwd=str(cwd),
|
||||
input=stdin_text,
|
||||
env=env,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
|
||||
|
||||
def test_tag_only_push_from_a_behind_branch_is_allowed(behind_repo):
|
||||
"""The fix: a tag-only push must not be blocked by H11 even though the branch is behind."""
|
||||
stdin = f"refs/tags/v1.0.0 {DUMMY_SHA_A} refs/tags/v1.0.0 {DUMMY_SHA_B}\n"
|
||||
r = _run_hook(behind_repo, stdin)
|
||||
assert r.returncode == 0, f"tag-only push was blocked: {r.stdout}{r.stderr}"
|
||||
|
||||
|
||||
def test_tag_only_push_ignores_blank_lines(behind_repo):
|
||||
stdin = f"\nrefs/tags/v1.0.0 {DUMMY_SHA_A} refs/tags/v1.0.0 {DUMMY_SHA_B}\n\n"
|
||||
r = _run_hook(behind_repo, stdin)
|
||||
assert r.returncode == 0, f"tag-only push (with blank lines) was blocked: {r.stdout}{r.stderr}"
|
||||
|
||||
|
||||
def test_negative_control_branch_push_from_behind_is_still_blocked(behind_repo):
|
||||
"""Required by #719: the fix must not weaken H11 for ordinary branch pushes."""
|
||||
stdin = f"refs/heads/feature {DUMMY_SHA_A} refs/heads/feature {DUMMY_SHA_B}\n"
|
||||
r = _run_hook(behind_repo, stdin)
|
||||
assert r.returncode == 1, "a branch push from a behind branch was allowed"
|
||||
assert "H11" in r.stdout
|
||||
|
||||
|
||||
def test_mixed_branch_and_tag_push_is_still_blocked(behind_repo):
|
||||
stdin = (
|
||||
f"refs/heads/feature {DUMMY_SHA_A} refs/heads/feature {DUMMY_SHA_B}\n"
|
||||
f"refs/tags/v1.0.0 {DUMMY_SHA_A} refs/tags/v1.0.0 {DUMMY_SHA_B}\n"
|
||||
)
|
||||
r = _run_hook(behind_repo, stdin)
|
||||
assert r.returncode == 1, "a mixed branch+tag push was allowed through the tag exemption"
|
||||
assert "H11" in r.stdout
|
||||
|
||||
|
||||
def test_zero_ref_lines_does_not_exempt(behind_repo):
|
||||
"""Vacuous-truth guard: 'all refs are tags' is trivially true over zero lines. Empty stdin
|
||||
(hook run manually, or a caller that forgot to forward the ref lines) must fall through to the
|
||||
existing behind-origin/main check, not silently disable H11."""
|
||||
r = _run_hook(behind_repo, "")
|
||||
assert r.returncode == 1, "empty stdin vacuously granted the tag exemption"
|
||||
assert "H11" in r.stdout
|
||||
|
||||
|
||||
def test_zero_ref_lines_of_only_blank_lines_does_not_exempt(behind_repo):
|
||||
r = _run_hook(behind_repo, "\n\n\n")
|
||||
assert r.returncode == 1, "stdin of only blank lines vacuously granted the tag exemption"
|
||||
assert "H11" in r.stdout
|
||||
|
||||
|
||||
# --- final line with NO trailing newline -------------------------------------------------------
|
||||
# `read` returns non-zero on an unterminated final line, so a bare `while read` silently DROPS it.
|
||||
# Both directions matter and they fail differently, which is why each is pinned:
|
||||
# - tag-only, unterminated -> the line is dropped, no refs are seen, and H11 blocks the release
|
||||
# tag push again, i.e. #719 quietly returns.
|
||||
# - mixed, unterminated -> the BRANCH line is dropped, leaving only tag refs, and the
|
||||
# exemption is granted for a push that includes a branch. That is the dangerous direction.
|
||||
# Git always newline-terminates its ref lines and `.husky/pre-push` re-adds one via `printf '%s\n'`,
|
||||
# so this is reachable only on a hand-piped run — but the guard is cheap and the failure is silent.
|
||||
|
||||
|
||||
def test_unterminated_final_line_tag_only_is_still_exempt(behind_repo):
|
||||
stdin = f"refs/tags/v1.0.0 {DUMMY_SHA_A} refs/tags/v1.0.0 {DUMMY_SHA_B}" # no trailing \n
|
||||
r = _run_hook(behind_repo, stdin)
|
||||
assert r.returncode == 0, f"unterminated tag-only line was dropped, reinstating #719: {r.stdout}"
|
||||
|
||||
|
||||
def test_tty_stdin_does_not_hang_and_does_not_exempt(behind_repo):
|
||||
"""The hook gained a stdin reader in #719; before that it read nothing, and its own docs call
|
||||
'run by hand' a supported case. Without the `[ -t 0 ] ||` guard an interactive run blocks
|
||||
forever waiting on the terminal. A pty gives it a real TTY on fd 0; the `timeout` turns a
|
||||
regression into a clean failure instead of a hung CI job."""
|
||||
primary, secondary = os.openpty()
|
||||
try:
|
||||
r = subprocess.run(
|
||||
["bash", str(HOOK)],
|
||||
cwd=str(behind_repo),
|
||||
stdin=secondary,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=30,
|
||||
)
|
||||
except subprocess.TimeoutExpired:
|
||||
pytest.fail("hook hung on TTY stdin — the `[ -t 0 ] ||` guard is missing or ineffective")
|
||||
finally:
|
||||
os.close(primary)
|
||||
os.close(secondary)
|
||||
# A TTY yields no ref lines, so this is the zero-line fall-through: H11 still applies.
|
||||
assert r.returncode == 1, "TTY stdin vacuously granted the tag exemption"
|
||||
assert "H11" in r.stdout
|
||||
|
||||
|
||||
def test_unterminated_final_branch_line_is_not_swallowed_into_the_exemption(behind_repo):
|
||||
"""The dangerous direction: if the unterminated BRANCH line is dropped, only tag refs remain
|
||||
and a branch push wins the tag exemption."""
|
||||
stdin = (
|
||||
f"refs/tags/v1.0.0 {DUMMY_SHA_A} refs/tags/v1.0.0 {DUMMY_SHA_B}\n"
|
||||
f"refs/heads/feature {DUMMY_SHA_A} refs/heads/feature {DUMMY_SHA_B}" # no trailing \n
|
||||
)
|
||||
r = _run_hook(behind_repo, stdin)
|
||||
assert r.returncode == 1, "an unterminated branch ref was swallowed into the tag exemption"
|
||||
assert "H11" in r.stdout
|
||||
@@ -0,0 +1,330 @@
|
||||
"""`.husky/pre-push` line 11 — `unset GIT_DIR GIT_WORK_TREE GIT_INDEX_FILE` — is load-bearing.
|
||||
|
||||
That one line is the fix for a real, silent defect, and ersatztv#785 ranks it third because nothing
|
||||
pinned it: reordering it after the nested git calls, or dropping it in a tidy-up, reintroduces the
|
||||
bug with no symptom at all. The failure is a check reporting SUCCESS, which is the family
|
||||
`testing.guard-ships-with-mutation-proof` exists for.
|
||||
|
||||
**The mechanism, measured rather than asserted.** Measured on both platforms this suite runs on,
|
||||
because the guard it replaces was fail-open for months on exactly the platform nobody measured:
|
||||
macOS/git 2.55 (development) and Linux/git 2.47.3 (the `script-tests` runner host). Identical on
|
||||
both — `GIT_DIR` exported, `show-toplevel` answering `<wt>/web`, the nested diff reporting exit 0.
|
||||
The whole file was run green there, and the deletion mutation was witnessed red there:
|
||||
|
||||
* Git exports `GIT_DIR` to `pre-push` **when the push comes from a worktree** — e.g.
|
||||
`GIT_DIR=/repo/.git/worktrees/wt`. From the main tree it exports nothing, which is why this
|
||||
never bites in a plain checkout and why it bites here constantly: `process.shared-tree-readonly`
|
||||
makes working in a worktree the mandated path, so the exported-`GIT_DIR` case is the NORMAL one.
|
||||
* With `GIT_DIR` set and `GIT_WORK_TREE` unset, git stops discovering the repo and takes the
|
||||
**current directory** as the work tree. `git rev-parse --show-toplevel` from `web/` answers
|
||||
`/repo/wt/web`.
|
||||
* So `pre-push`'s last line — `cd web && npm run check:api`, whose `check:api` ends in
|
||||
`git diff --exit-code` — compares against index paths that do not exist under that root. It
|
||||
reports **no diff and exits 0**. Generated-API drift ships, and the gate that exists to catch it
|
||||
prints success.
|
||||
|
||||
The test drives the REAL `.husky/pre-push` file, unedited, in a real worktree, with the environment
|
||||
git really exports. What is substituted is only what surrounds it: the three `.claude/hooks` calls
|
||||
are stubs (they are separately guarded and are not the subject), and `npm` is a stub on PATH whose
|
||||
`run check:api` performs the nested `git diff --exit-code` that the real one ends in. The subject —
|
||||
the ordering of the `unset` against the nested git call — is untouched.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import shutil
|
||||
import stat
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[2]
|
||||
PRE_PUSH = REPO_ROOT / ".husky" / "pre-push"
|
||||
|
||||
UNSET_CLAUSE = "unset GIT_DIR GIT_WORK_TREE GIT_INDEX_FILE"
|
||||
|
||||
|
||||
# The hooks `pre-push` calls before the CI-parity block. Stubbed because each is guarded on its own
|
||||
# terms and none of them is what this file is about — but DERIVED from the real file rather than
|
||||
# hand-listed, so a hook added to `pre-push` tomorrow cannot leave this harness silently running a
|
||||
# `pre-push` that dies at a missing script and calling that a red.
|
||||
def _hook_calls(text: str) -> list[str]:
|
||||
"""Every `.claude/hooks/` script `pre-push` actually invokes.
|
||||
|
||||
Two refinements over a plain substring scan, each closing a way the harness would misreport:
|
||||
|
||||
* **comment lines are skipped.** A comment naming a hook that no longer exists would otherwise
|
||||
redden `test_the_stub_hooks_are_derived_from_the_real_file` for a file that is perfectly
|
||||
correct.
|
||||
* **the prefix is not assumed to be `./`.** A hook invoked as `bash .claude/hooks/x.sh` would
|
||||
be missed, left unstubbed, and exit 127 — a NON-ZERO status that two tests here read as
|
||||
"drift was caught". That is a false green in the direction that matters, so the match is on
|
||||
the path segment rather than on `./`.
|
||||
"""
|
||||
names = []
|
||||
for line in text.splitlines():
|
||||
# INLINE comments too, not just whole-line ones. Broadening the marker from `./.claude/hooks/`
|
||||
# to the path segment made a trailing `# ... .claude/hooks/removed-helper.sh` match, which
|
||||
# would redden this file for a `pre-push` that is perfectly correct — a false red the
|
||||
# narrower marker did not have. The fix for one over-match must not introduce another.
|
||||
stripped = line.split("#", 1)[0].strip()
|
||||
marker = ".claude/hooks/"
|
||||
if marker in stripped:
|
||||
rest = stripped.split(marker, 1)[1]
|
||||
names.append(rest.split()[0].rstrip("|&;\"'"))
|
||||
return names
|
||||
|
||||
|
||||
def _git(cwd: Path, *args: str) -> str:
|
||||
return subprocess.run(
|
||||
["git", *args],
|
||||
cwd=str(cwd),
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
env={
|
||||
**os.environ,
|
||||
"GIT_AUTHOR_NAME": "t",
|
||||
"GIT_AUTHOR_EMAIL": "t@e",
|
||||
"GIT_COMMITTER_NAME": "t",
|
||||
"GIT_COMMITTER_EMAIL": "t@e",
|
||||
},
|
||||
).stdout.strip()
|
||||
|
||||
|
||||
def _exe(path: Path, body: str) -> None:
|
||||
path.write_text(body)
|
||||
path.chmod(path.stat().st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)
|
||||
|
||||
|
||||
class Bench:
|
||||
"""A real repo, a real worktree, the real `pre-push`, and the env git really exports."""
|
||||
|
||||
def __init__(self, tmp_path: Path, pre_push_text: str, *, drift: bool):
|
||||
tmp_path.mkdir(parents=True, exist_ok=True)
|
||||
self.root = tmp_path / "main-tree"
|
||||
self.root.mkdir()
|
||||
_git(self.root, "init", "-q", "-b", "main", ".")
|
||||
(self.root / "seed").write_text("seed\n")
|
||||
_git(self.root, "add", "seed")
|
||||
_git(self.root, "commit", "-qm", "init")
|
||||
|
||||
self.wt = tmp_path / "wt"
|
||||
_git(self.root, "worktree", "add", "-q", str(self.wt), "-b", "feature")
|
||||
|
||||
web = self.wt / "web"
|
||||
web.mkdir()
|
||||
(web / "gen.txt").write_text("generated\n")
|
||||
_git(self.wt, "add", "web/gen.txt")
|
||||
_git(self.wt, "commit", "-qm", "add generated file")
|
||||
if drift:
|
||||
# The condition `check:api` exists to catch: the committed generated artifact no longer
|
||||
# matches what regeneration produces.
|
||||
(web / "gen.txt").write_text("generated\nDRIFT\n")
|
||||
|
||||
(self.wt / ".husky").mkdir()
|
||||
self.pre_push = self.wt / ".husky" / "pre-push"
|
||||
_exe(self.pre_push, pre_push_text)
|
||||
|
||||
hooks = self.wt / ".claude" / "hooks"
|
||||
hooks.mkdir(parents=True)
|
||||
for name in _hook_calls(pre_push_text):
|
||||
_exe(hooks / name, "#!/usr/bin/env bash\ncat >/dev/null\nexit 0\n")
|
||||
|
||||
# The npm stand-in. `run check:api` performs the nested `git diff --exit-code` the real
|
||||
# script ends in; everything else is a no-op. It records the nested diff's OWN exit code,
|
||||
# not merely that it ran. That distinction is load-bearing: one of the mutations below moves
|
||||
# the `unset` to the end of the file, which also makes the script's terminal status 0 (a
|
||||
# bare `unset` succeeds) — so asserting on pre-push's exit code alone would be satisfied for
|
||||
# a reason that has nothing to do with the nested git call. The recorded diff verdict IS the
|
||||
# defect; the script's exit code is downstream of it.
|
||||
self.witness = tmp_path / "check-api-ran"
|
||||
bindir = tmp_path / "bin"
|
||||
bindir.mkdir()
|
||||
_exe(
|
||||
bindir / "npm",
|
||||
"#!/usr/bin/env bash\n"
|
||||
'if [ "$1" = "run" ] && [ "$2" = "check:api" ]; then\n'
|
||||
" git diff --exit-code -- gen.txt >/dev/null\n"
|
||||
" _rc=$?\n"
|
||||
f' echo "$PWD $_rc" >> "{self.witness}"\n'
|
||||
" exit $_rc\n"
|
||||
"fi\n"
|
||||
"exit 0\n",
|
||||
)
|
||||
self.bindir = bindir
|
||||
# `GIT_DIR` exactly as git exports it for a push from this worktree, verified against a real
|
||||
# push in the investigation that produced this file.
|
||||
self.git_dir = _git(self.wt, "rev-parse", "--absolute-git-dir")
|
||||
|
||||
def run(self, *, export_git_dir: bool) -> subprocess.CompletedProcess:
|
||||
env = {k: v for k, v in os.environ.items() if k not in ("GIT_DIR", "GIT_WORK_TREE", "GIT_INDEX_FILE")}
|
||||
env["PATH"] = f"{self.bindir}:{env['PATH']}"
|
||||
if export_git_dir:
|
||||
env["GIT_DIR"] = self.git_dir
|
||||
return subprocess.run(
|
||||
["bash", str(self.pre_push), "origin", "file:///dev/null"],
|
||||
input=b"refs/heads/feature abc refs/heads/feature def\n",
|
||||
capture_output=True,
|
||||
cwd=str(self.wt),
|
||||
env=env,
|
||||
timeout=120,
|
||||
)
|
||||
|
||||
def check_api_ran(self) -> bool:
|
||||
return self.witness.exists()
|
||||
|
||||
def nested_diff_rc(self) -> int:
|
||||
"""The exit code the nested `git diff --exit-code` actually reported.
|
||||
|
||||
0 means it saw NO diff. With a drifted tree that answer is the bug.
|
||||
"""
|
||||
assert self.witness.exists(), "check:api never ran, so there is no nested diff verdict"
|
||||
return int(self.witness.read_text().split()[-1])
|
||||
|
||||
|
||||
# ------------------------------------------------------------------------------------------------
|
||||
# ANTI-VACUITY — the clause exists, and the harness reaches the check that depends on it
|
||||
# ------------------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_the_unset_clause_is_still_in_pre_push():
|
||||
text = PRE_PUSH.read_text()
|
||||
assert UNSET_CLAUSE in text, (
|
||||
f"`{UNSET_CLAUSE}` is gone from .husky/pre-push. If it was removed deliberately, this file "
|
||||
"must be removed with it and docs/guard-inventory.md updated — do not delete this assertion "
|
||||
"on its own, it is the only thing pinning the ordering"
|
||||
)
|
||||
lines = text.splitlines()
|
||||
unset_at = next(i for i, ln in enumerate(lines) if UNSET_CLAUSE in ln)
|
||||
nested_at = next(i for i, ln in enumerate(lines) if "npm run check:api" in ln)
|
||||
assert unset_at < nested_at, (
|
||||
"the unset now comes AFTER the nested git call it exists to protect — that ordering is the "
|
||||
"regression, and it is silent"
|
||||
)
|
||||
|
||||
|
||||
def test_the_harness_actually_reaches_check_api(tmp_path):
|
||||
bench = Bench(tmp_path, PRE_PUSH.read_text(), drift=True)
|
||||
bench.run(export_git_dir=False)
|
||||
assert bench.check_api_ran(), (
|
||||
"the CI-parity block never ran, so every exit code below would be reporting on the hooks "
|
||||
"before it rather than on the nested git call this file is about"
|
||||
)
|
||||
|
||||
|
||||
# ------------------------------------------------------------------------------------------------
|
||||
# THE GUARD DECIDES
|
||||
# ------------------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_drift_is_CAUGHT_with_no_git_env_exported(tmp_path):
|
||||
"""Positive control: the harness detects real drift when nothing is in the way."""
|
||||
bench = Bench(tmp_path, PRE_PUSH.read_text(), drift=True)
|
||||
p = bench.run(export_git_dir=False)
|
||||
assert bench.nested_diff_rc() != 0, "the nested diff saw no drift even with a clean env"
|
||||
assert p.returncode != 0, f"the harness did not detect drift even with a clean env: {p.stderr!r}"
|
||||
|
||||
|
||||
def test_drift_is_CAUGHT_when_git_exports_GIT_DIR_from_a_worktree(tmp_path):
|
||||
"""THE REAL CASE. Every push from a worktree — the mandated way to work here — runs this."""
|
||||
bench = Bench(tmp_path, PRE_PUSH.read_text(), drift=True)
|
||||
p = bench.run(export_git_dir=True)
|
||||
assert bench.nested_diff_rc() != 0, (
|
||||
"the nested `git diff --exit-code` reported NO DIFF on a drifted file. That is the silent "
|
||||
"fail-open the `unset` exists to prevent, and it is invisible from the exit code alone"
|
||||
)
|
||||
assert p.returncode != 0, (
|
||||
"pre-push reported success on a drifted generated file while git had exported GIT_DIR: "
|
||||
f"{p.stdout!r} {p.stderr!r}"
|
||||
)
|
||||
|
||||
|
||||
def test_a_CLEAN_tree_is_allowed_through(tmp_path):
|
||||
"""Negative control. A harness that always failed would pass both assertions above."""
|
||||
bench = Bench(tmp_path, PRE_PUSH.read_text(), drift=False)
|
||||
p = bench.run(export_git_dir=True)
|
||||
assert p.returncode == 0, f"a clean tree was blocked: {p.stdout!r} {p.stderr!r}"
|
||||
assert bench.nested_diff_rc() == 0, "the nested diff invented a diff on a clean tree"
|
||||
|
||||
|
||||
# ------------------------------------------------------------------------------------------------
|
||||
# MUTATION PROOFS — the two ways the clause stops protecting anything
|
||||
# ------------------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _positive_control(tmp_path: Path, label: str) -> None:
|
||||
bench = Bench(tmp_path / f"pc-{label}", PRE_PUSH.read_text(), drift=True)
|
||||
p = bench.run(export_git_dir=True)
|
||||
assert bench.nested_diff_rc() != 0 and p.returncode != 0, (
|
||||
"the UNMUTATED pre-push did not catch the drift, so 'the mutant lets it through' proves "
|
||||
f"nothing about the clause: {p.stdout!r} {p.stderr!r}"
|
||||
)
|
||||
|
||||
|
||||
def test_MUTATION_DELETING_the_unset_lets_drift_through_silently(tmp_path):
|
||||
_positive_control(tmp_path, "delete")
|
||||
|
||||
text = PRE_PUSH.read_text()
|
||||
assert UNSET_CLAUSE in text, "retarget this mutation; the clause has moved"
|
||||
mutated = text.replace(UNSET_CLAUSE, "# clause removed by the mutation proof", 1)
|
||||
|
||||
bench = Bench(tmp_path / "mut", mutated, drift=True)
|
||||
p = bench.run(export_git_dir=True)
|
||||
assert bench.check_api_ran(), "the mutant died before check:api, so its exit code says nothing"
|
||||
assert bench.nested_diff_rc() == 0, (
|
||||
"removing the unset did NOT blind the nested diff, so the clause is not what protects it "
|
||||
"and this whole file is pinning the wrong thing"
|
||||
)
|
||||
assert p.returncode == 0, f"the drift was still caught somehow: {p.stdout!r}"
|
||||
|
||||
|
||||
def test_MUTATION_REORDERING_the_unset_after_the_nested_git_call_lets_drift_through(tmp_path):
|
||||
"""The regression ersatztv#785 names by hand: not deletion, relocation.
|
||||
|
||||
Deleting a line is a conspicuous diff. Moving it — during a tidy-up, or when a new check is
|
||||
appended above it — reads as a no-op and is not.
|
||||
|
||||
This asserts on the NESTED DIFF's verdict, not on pre-push's exit code. Relocating the `unset`
|
||||
to the end of the file also makes it the script's last statement, and a bare `unset` succeeds —
|
||||
so `returncode == 0` would hold here even if the nested git call had worked perfectly. That is a
|
||||
test passing for the wrong reason, and it was written that way in this file's first draft.
|
||||
"""
|
||||
_positive_control(tmp_path, "reorder")
|
||||
|
||||
lines = PRE_PUSH.read_text().splitlines()
|
||||
kept = [ln for ln in lines if UNSET_CLAUSE not in ln]
|
||||
assert len(kept) == len(lines) - 1, "expected exactly one unset line to relocate"
|
||||
mutated = "\n".join(kept + [UNSET_CLAUSE, ""])
|
||||
|
||||
bench = Bench(tmp_path / "mut", mutated, drift=True)
|
||||
bench.run(export_git_dir=True)
|
||||
assert bench.check_api_ran(), "the mutant died before check:api, so it reports on nothing"
|
||||
assert bench.nested_diff_rc() == 0, (
|
||||
"moving the unset below the nested git call did NOT blind it, so the ORDERING is not "
|
||||
"load-bearing and the ordering assertion in the anti-vacuity test is decoration"
|
||||
)
|
||||
|
||||
|
||||
def test_the_stub_hooks_are_derived_from_the_real_file():
|
||||
"""If `pre-push` gains a hook call, the harness must stub it rather than die at a missing file.
|
||||
|
||||
A `pre-push` that exits 127 at a missing script produces a non-zero exit — indistinguishable
|
||||
from 'drift was caught' in two of the tests above.
|
||||
"""
|
||||
calls = _hook_calls(PRE_PUSH.read_text())
|
||||
assert calls, "no ./.claude/hooks/ call found in pre-push; the extractor has stopped matching"
|
||||
for name in calls:
|
||||
assert (REPO_ROOT / ".claude" / "hooks" / name).is_file(), (
|
||||
f"pre-push calls {name}, which does not exist in .claude/hooks/"
|
||||
)
|
||||
|
||||
|
||||
def test_shutil_which_npm_is_not_what_the_harness_used(tmp_path):
|
||||
"""Anti-vacuity for the stand-in: a real `npm` on PATH would run the real scripts and pass."""
|
||||
bench = Bench(tmp_path, PRE_PUSH.read_text(), drift=True)
|
||||
resolved = shutil.which("npm", path=f"{bench.bindir}:{os.environ['PATH']}")
|
||||
assert resolved == str(bench.bindir / "npm"), (
|
||||
f"the harness would have used {resolved}, not its stand-in — the check it performs would "
|
||||
"then be whatever the real package.json says, not the nested git call under test"
|
||||
)
|
||||
@@ -0,0 +1,389 @@
|
||||
"""scripts/prove-fix.sh witnesses a fix's test failing before the fix (ersatztv#794).
|
||||
|
||||
THE NEGATIVE CONTROL IS THE POINT OF THIS FILE. A prover that reports PROVEN for
|
||||
everything is worse than no prover: it manufactures exactly the confidence #794 exists
|
||||
to withhold. So `test_unrelated_test_is_UNPROVEN` is the load-bearing case here, and the
|
||||
positive case only tells us the script can distinguish the two.
|
||||
|
||||
Each test builds a throwaway git repo rather than pinning real commits from this
|
||||
repository's history — a test anchored to a real sha rots the moment that sha is rebased
|
||||
or the file moves, and then it passes for the wrong reason (or is deleted for being
|
||||
flaky, which is worse).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import subprocess
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[2]
|
||||
# Overridable so a MUTANT copy can be driven through these very tests — that is what makes
|
||||
# the mutation proof below a witnessed red rather than an argument. Gated behind a sentinel
|
||||
# so a stray CI value cannot silently point the whole suite at another script.
|
||||
if os.environ.get("PROVE_FIX_PATH") and os.environ.get("PROVE_FIX_MUTATION_RUN") != "1":
|
||||
raise RuntimeError(
|
||||
"PROVE_FIX_PATH is set without PROVE_FIX_MUTATION_RUN=1. That would silently test a "
|
||||
"different script than the one this suite vouches for."
|
||||
)
|
||||
PROVE_FIX = Path(os.environ.get("PROVE_FIX_PATH") or (REPO_ROOT / "scripts" / "prove-fix.sh"))
|
||||
|
||||
|
||||
def _git(repo: Path, *args: str) -> str:
|
||||
return subprocess.run(
|
||||
["git", "-C", str(repo), *args],
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
).stdout.strip()
|
||||
|
||||
|
||||
def _run(repo: Path, *args: str) -> subprocess.CompletedProcess[str]:
|
||||
return subprocess.run(
|
||||
["bash", str(PROVE_FIX), "--repo", str(repo), *args],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
cwd=str(repo),
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def fixrepo(tmp_path: Path) -> Path:
|
||||
"""A repo whose HEAD is a fix: code change + a test that discriminates.
|
||||
|
||||
It also carries an UNRELATED test, present from the first commit, which passes with
|
||||
or without the fix. That test is the negative control's subject.
|
||||
"""
|
||||
repo = tmp_path / "r"
|
||||
(repo / "scripts" / "tests").mkdir(parents=True)
|
||||
subprocess.run(["git", "init", "-q", str(repo)], check=True)
|
||||
_git(repo, "config", "user.email", "t@example.com")
|
||||
_git(repo, "config", "user.name", "T")
|
||||
|
||||
# --- commit 1: the bug, plus a test that cannot see it
|
||||
(repo / "calc.py").write_text("def add(a, b):\n return a - b # bug\n")
|
||||
(repo / "scripts" / "tests" / "test_unrelated.py").write_text("def test_unrelated():\n assert 1 + 1 == 2\n")
|
||||
_git(repo, "add", "-A")
|
||||
_git(repo, "commit", "-q", "-m", "initial: buggy add, unrelated test")
|
||||
|
||||
# --- commit 2: the fix + a test that discriminates against its absence
|
||||
(repo / "calc.py").write_text("def add(a, b):\n return a + b\n")
|
||||
(repo / "scripts" / "tests" / "test_add.py").write_text(
|
||||
"from calc import add\n\n\ndef test_add():\n assert add(2, 3) == 5\n"
|
||||
)
|
||||
_git(repo, "add", "-A")
|
||||
_git(repo, "commit", "-q", "-m", "fix: add() returned a difference\n\nProves: scripts/tests/test_add.py")
|
||||
return repo
|
||||
|
||||
|
||||
def test_reverting_the_fix_reddens_its_test_PROVEN(fixrepo: Path) -> None:
|
||||
r = _run(fixrepo, "HEAD", "scripts/tests/test_add.py")
|
||||
assert r.returncode == 0, f"expected PROVEN (0), got {r.returncode}\n{r.stdout}\n{r.stderr}"
|
||||
assert "PROVEN" in r.stdout
|
||||
assert "UNPROVEN" not in r.stdout
|
||||
|
||||
|
||||
def test_unrelated_test_is_UNPROVEN(fixrepo: Path) -> None:
|
||||
"""THE NEGATIVE CONTROL. A test that passes without the fix must be refused.
|
||||
|
||||
Without this, every other assertion in this file is compatible with a script that
|
||||
prints PROVEN unconditionally.
|
||||
"""
|
||||
r = _run(fixrepo, "HEAD", "scripts/tests/test_unrelated.py")
|
||||
assert r.returncode == 1, f"expected UNPROVEN (1), got {r.returncode}\n{r.stdout}\n{r.stderr}"
|
||||
assert "UNPROVEN" in r.stdout
|
||||
assert "do not discriminate" in r.stdout
|
||||
|
||||
|
||||
def test_selector_comes_from_the_Proves_trailer(fixrepo: Path) -> None:
|
||||
"""No selector argument: it must read `Proves:` rather than guess."""
|
||||
r = _run(fixrepo, "HEAD")
|
||||
assert r.returncode == 0, f"expected PROVEN via trailer, got {r.returncode}\n{r.stdout}\n{r.stderr}"
|
||||
assert "scripts/tests/test_add.py" in r.stdout
|
||||
|
||||
|
||||
def test_no_selector_and_no_trailer_REFUSES(fixrepo: Path) -> None:
|
||||
"""Refusing beats guessing: a heuristic silently proves nothing when a fix edits an
|
||||
existing test, which is the failure mode being mechanised against."""
|
||||
(fixrepo / "calc.py").write_text("def add(a, b):\n return a + b # touched\n")
|
||||
_git(fixrepo, "add", "-A")
|
||||
_git(fixrepo, "commit", "-q", "-m", "fix: no trailer here")
|
||||
r = _run(fixrepo, "HEAD")
|
||||
assert r.returncode == 3, f"expected 3 (no selector), got {r.returncode}\n{r.stderr}"
|
||||
assert "Proves:" in r.stderr
|
||||
|
||||
|
||||
def test_test_only_commit_REFUSES(fixrepo: Path) -> None:
|
||||
"""A commit with no code side cannot be proven this way — it must say so, not pass."""
|
||||
(fixrepo / "scripts" / "tests" / "test_extra.py").write_text("def test_x():\n assert True\n")
|
||||
_git(fixrepo, "add", "-A")
|
||||
_git(fixrepo, "commit", "-q", "-m", "test: add a test only\n\nProves: scripts/tests/test_extra.py")
|
||||
r = _run(fixrepo, "HEAD")
|
||||
assert r.returncode == 4, f"expected 4 (nothing to revert), got {r.returncode}\n{r.stderr}"
|
||||
assert "nothing to revert" in r.stderr
|
||||
|
||||
|
||||
def test_selector_matching_no_tests_REFUSES(fixrepo: Path) -> None:
|
||||
"""`pytest` exits 5 when it collects nothing. Treating that as red would prove every
|
||||
fix — a check that examined nothing reporting success."""
|
||||
r = _run(fixrepo, "HEAD", "scripts/tests/test_does_not_exist.py")
|
||||
assert r.returncode == 5, f"expected 5 (no tests collected), got {r.returncode}\n{r.stderr}"
|
||||
assert "NO tests" in r.stderr
|
||||
|
||||
|
||||
def test_added_code_file_is_removed_not_checked_out(tmp_path: Path) -> None:
|
||||
"""A file the fix ADDED does not exist in the parent. `git checkout parent -- <new>`
|
||||
fails there, and if that failure were swallowed the fix would stay in place and every
|
||||
run would report a false PROVEN."""
|
||||
repo = tmp_path / "r2"
|
||||
(repo / "scripts" / "tests").mkdir(parents=True)
|
||||
subprocess.run(["git", "init", "-q", str(repo)], check=True)
|
||||
_git(repo, "config", "user.email", "t@example.com")
|
||||
_git(repo, "config", "user.name", "T")
|
||||
(repo / "scripts" / "tests" / "keep.py").write_text("# placeholder\n")
|
||||
_git(repo, "add", "-A")
|
||||
_git(repo, "commit", "-q", "-m", "initial")
|
||||
|
||||
# the fix ADDS helper.py (it has no parent version) and a test that needs it
|
||||
(repo / "helper.py").write_text("def shout(s):\n return s.upper()\n")
|
||||
(repo / "scripts" / "tests" / "test_helper.py").write_text(
|
||||
"from helper import shout\n\n\ndef test_shout():\n assert shout('a') == 'A'\n"
|
||||
)
|
||||
_git(repo, "add", "-A")
|
||||
_git(repo, "commit", "-q", "-m", "feat: add shout\n\nProves: scripts/tests/test_helper.py")
|
||||
|
||||
r = _run(repo, "HEAD")
|
||||
assert r.returncode == 0, f"expected PROVEN, got {r.returncode}\n{r.stdout}\n{r.stderr}"
|
||||
assert "PROVEN" in r.stdout
|
||||
|
||||
|
||||
def test_root_commit_REFUSES(tmp_path: Path) -> None:
|
||||
repo = tmp_path / "r3"
|
||||
repo.mkdir()
|
||||
subprocess.run(["git", "init", "-q", str(repo)], check=True)
|
||||
_git(repo, "config", "user.email", "t@example.com")
|
||||
_git(repo, "config", "user.name", "T")
|
||||
(repo / "a.py").write_text("x = 1\n")
|
||||
_git(repo, "add", "-A")
|
||||
_git(repo, "commit", "-q", "-m", "root")
|
||||
r = _run(repo, "HEAD", "scripts/tests")
|
||||
assert r.returncode == 5
|
||||
assert "root commit" in r.stderr
|
||||
|
||||
|
||||
def test_runs_under_the_system_bash(fixrepo: Path) -> None:
|
||||
"""macOS ships /bin/bash 3.2, where `mapfile` is absent AND yields an empty array
|
||||
instead of erroring. Assert by EXECUTING under that bash, not by grepping the source
|
||||
for "mapfile" — the first version of this test did the latter and matched the comment
|
||||
explaining why mapfile is avoided, which is a string predicate failing exactly as
|
||||
docs/defect-shapes-773.md §3.7 says they do."""
|
||||
system_bash = Path("/bin/bash")
|
||||
if not system_bash.exists():
|
||||
pytest.skip("/bin/bash not present")
|
||||
# Named for the system bash, NOT for 3.2: on the Linux runner /bin/bash is 5.x, so a
|
||||
# name promising bash-3.2 coverage would read as coverage that exists only on a
|
||||
# developer Mac. The 3.2 hazard (mapfile yielding an empty array) is what motivated it.
|
||||
ver = subprocess.run([str(system_bash), "--version"], capture_output=True, text=True).stdout
|
||||
r = subprocess.run(
|
||||
[str(system_bash), str(PROVE_FIX), "--repo", str(fixrepo), "HEAD", "scripts/tests/test_add.py"],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
cwd=str(fixrepo),
|
||||
)
|
||||
assert r.returncode == 0, (
|
||||
f"prove-fix.sh must work under the system bash ({ver.splitlines()[0] if ver else '?'}); "
|
||||
f"got {r.returncode}\n{r.stdout}\n{r.stderr}"
|
||||
)
|
||||
assert "PROVEN" in r.stdout
|
||||
|
||||
|
||||
def test_control_failure_REFUSES(fixrepo: Path) -> None:
|
||||
"""A test that is ALREADY red with the fix in place proves nothing by being red after
|
||||
a revert. Without this control, half a discrimination claim reads as the whole one."""
|
||||
(fixrepo / "scripts" / "tests" / "test_broken.py").write_text(
|
||||
"from calc import add\n\n\ndef test_broken():\n assert add(2, 3) == 99\n"
|
||||
)
|
||||
(fixrepo / "calc.py").write_text("def add(a, b):\n return a + b # unchanged\n")
|
||||
_git(fixrepo, "add", "-A")
|
||||
_git(fixrepo, "commit", "-q", "-m", "fix: with an already-failing test\n\nProves: scripts/tests/test_broken.py")
|
||||
r = _run(fixrepo, "HEAD")
|
||||
assert r.returncode == 6, f"expected 6 (control failed), got {r.returncode}\n{r.stdout}\n{r.stderr}"
|
||||
assert "control FAILED" in r.stderr
|
||||
|
||||
|
||||
def test_control_green_then_red_is_reported_as_both(fixrepo: Path) -> None:
|
||||
"""The PROVEN line must state both halves — green with, red without — because that
|
||||
pair is the claim. Reporting only the red half is the overclaim being mechanised out."""
|
||||
r = _run(fixrepo, "HEAD", "scripts/tests/test_add.py")
|
||||
assert r.returncode == 0
|
||||
assert "control" in r.stdout.lower()
|
||||
assert "Green with the fix (control), red without it" in r.stdout
|
||||
|
||||
|
||||
def test_MUTATION_disarming_the_UNPROVEN_clause_reddens_the_refusal_test(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
"""Clause-level mutation, EXECUTED, with the unchanged test WITNESSED RED against it.
|
||||
|
||||
`docs/guard-inventory.md` is explicit that MUTATION means a mutation was executed and
|
||||
the named test was *witnessed red* — feeding the real script a rejecting input is
|
||||
BEHAVIOUR-ONLY, and the column "is not a grading curve". An earlier version of this
|
||||
test deleted the clause and then asserted the MUTANT misbehaved, which left this test
|
||||
green and proved nothing about whether the clause is load-bearing. Cold review caught
|
||||
that, and it was right.
|
||||
|
||||
So: delete the `RC -eq 0 -> UNPROVEN` clause (the CLAUSE, not the file — #510), then
|
||||
re-run the UNCHANGED `test_unrelated_test_is_UNPROVEN` against the mutant in a nested
|
||||
pytest run. That test must go RED. Its red is the proof.
|
||||
"""
|
||||
real = REPO_ROOT / "scripts" / "prove-fix.sh"
|
||||
src = real.read_text()
|
||||
marker = 'if [ "$RC" -eq 0 ]; then'
|
||||
assert marker in src, "the clause under mutation is gone — regrade the inventory row"
|
||||
|
||||
end = src.index(" exit 1\nfi\n", src.index(marker)) + len(" exit 1\nfi\n")
|
||||
mutant = tmp_path / "prove-fix-mutant.sh"
|
||||
mutant.write_text(src[: src.index(marker)] + src[end:])
|
||||
assert marker not in mutant.read_text(), "mutation did not remove the clause"
|
||||
|
||||
env = {**os.environ, "PROVE_FIX_PATH": str(mutant), "PROVE_FIX_MUTATION_RUN": "1"}
|
||||
nested = subprocess.run(
|
||||
[
|
||||
"python3",
|
||||
"-m",
|
||||
"pytest",
|
||||
f"{Path(__file__).name}::test_unrelated_test_is_UNPROVEN",
|
||||
"-q",
|
||||
"-p",
|
||||
"no:cacheprovider",
|
||||
],
|
||||
cwd=str(Path(__file__).parent),
|
||||
env=env,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
out = nested.stdout + nested.stderr
|
||||
assert nested.returncode == 1, (
|
||||
"the unchanged refusal test must go RED against the mutant (pytest exit 1). "
|
||||
f"got {nested.returncode} — exit 2/3/4/5 would mean the nested run broke rather "
|
||||
f"than the test failing, which proves nothing.\n{out[-2000:]}"
|
||||
)
|
||||
# Require a real reported FAILURE of that specific test. `returncode != 0` alone would
|
||||
# be satisfied by a collection error — the vacuous shape this whole file is against.
|
||||
assert "FAILED" in out and "test_unrelated_test_is_UNPROVEN" in out, (
|
||||
"expected a reported failure of test_unrelated_test_is_UNPROVEN; the nested run "
|
||||
f"failed for some other reason.\n{out[-2000:]}"
|
||||
)
|
||||
assert "1 failed" in out, f"expected exactly one failing test.\n{out[-1200:]}"
|
||||
|
||||
|
||||
def test_SIGTERM_mid_run_never_reports_PROVEN(tmp_path: Path) -> None:
|
||||
"""A killed run must not look like evidence.
|
||||
|
||||
An early DRAFT printed PROVEN and exited 0 after a SIGTERM: the trap cleaned up but did
|
||||
not exit, so the previous status stood. On the first COMMITTED version (587edbecc) the
|
||||
run reaches rc 5 by a different route entirely, so this test earns its keep only via the
|
||||
assertions below — see the comment there.
|
||||
"""
|
||||
repo = tmp_path / "slow"
|
||||
(repo / "scripts" / "tests").mkdir(parents=True)
|
||||
subprocess.run(["git", "init", "-q", str(repo)], check=True)
|
||||
_git(repo, "config", "user.email", "t@example.com")
|
||||
_git(repo, "config", "user.name", "T")
|
||||
(repo / "mod.py").write_text("VALUE = 1\n")
|
||||
(repo / "scripts" / "tests" / "test_slow.py").write_text(
|
||||
"import time\nfrom mod import VALUE\n\n\ndef test_slow():\n time.sleep(20)\n assert VALUE == 2\n"
|
||||
)
|
||||
_git(repo, "add", "-A")
|
||||
_git(repo, "commit", "-q", "-m", "initial")
|
||||
(repo / "mod.py").write_text("VALUE = 2\n")
|
||||
_git(repo, "add", "-A")
|
||||
_git(repo, "commit", "-q", "-m", "fix: bump\n\nProves: scripts/tests/test_slow.py")
|
||||
|
||||
proc = subprocess.Popen(
|
||||
["bash", str(PROVE_FIX), "--repo", str(repo), "HEAD"],
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
text=True,
|
||||
cwd=str(repo),
|
||||
)
|
||||
time.sleep(4) # inside the control run, which sleeps 20s
|
||||
assert proc.poll() is None, "the run finished before it could be signalled; test is void"
|
||||
proc.terminate()
|
||||
out, err = proc.communicate(timeout=60)
|
||||
|
||||
# Assert the observable THIS fix introduced, not merely "non-zero and no PROVEN":
|
||||
# the pre-fix script also satisfied those two, by accident — its `trap cleanup EXIT INT
|
||||
# TERM` fired, deleted $TMP, execution continued, and a later step died 5. Two different
|
||||
# bugs landing on the same observable is not a witnessed fix. Cold review measured that
|
||||
# pair failing to separate old from new; rc==5 AND the handler's own message do separate
|
||||
# them.
|
||||
assert proc.returncode == 5, f"a signalled run must exit 5 from on_signal, got {proc.returncode}\n{out}\n{err}"
|
||||
assert "interrupted by signal" in err, (
|
||||
f"expected the signal handler's own message, so this test cannot be satisfied by an "
|
||||
f"unrelated later failure:\n{err}"
|
||||
)
|
||||
assert "PROVEN" not in out, f"a signalled run must not print a verdict:\n{out}"
|
||||
|
||||
|
||||
def test_a_harness_failure_is_NOT_reported_as_PROVEN(tmp_path: Path) -> None:
|
||||
"""The marker-absence branch, reached the way the old false green was reached.
|
||||
|
||||
An earlier version stubbed `python3` to exit 127, which does NOT reach the marker logic:
|
||||
the marker IS written (127) and the script exits via the control-failure branch. Cold
|
||||
review measured that, and it is why the round-2 commit came back UNPROVEN from this
|
||||
tool's own gate — the fix was executed by no test.
|
||||
|
||||
The real shape is `( cd X && pytest ); rc=$?` returning 1 because `cd` FAILED and pytest
|
||||
never ran; pre-fix that was accepted as red and produced PROVEN. Reproduced by shimming
|
||||
`git` so the SECOND `worktree add` (the reverted phase) exits 0 without creating the
|
||||
directory. The fixture's fix ADDS its code file, so the revert step is `rm -f` — which
|
||||
succeeds on a missing directory and lets execution reach the phase's `cd`.
|
||||
"""
|
||||
repo = tmp_path / "addrepo"
|
||||
(repo / "scripts" / "tests").mkdir(parents=True)
|
||||
subprocess.run(["git", "init", "-q", str(repo)], check=True)
|
||||
_git(repo, "config", "user.email", "t@example.com")
|
||||
_git(repo, "config", "user.name", "T")
|
||||
(repo / "scripts" / "tests" / "keep.py").write_text("# placeholder\n")
|
||||
_git(repo, "add", "-A")
|
||||
_git(repo, "commit", "-q", "-m", "initial")
|
||||
(repo / "added.py").write_text("def val():\n return 7\n")
|
||||
(repo / "scripts" / "tests" / "test_added.py").write_text(
|
||||
"from added import val\n\n\ndef test_val():\n assert val() == 7\n"
|
||||
)
|
||||
_git(repo, "add", "-A")
|
||||
_git(repo, "commit", "-q", "-m", "feat: add val\n\nProves: scripts/tests/test_added.py")
|
||||
|
||||
shim = tmp_path / "bin"
|
||||
shim.mkdir()
|
||||
counter = tmp_path / "count"
|
||||
(shim / "git").write_text(
|
||||
"#!/bin/sh\n"
|
||||
'if [ "$3" = "worktree" ] && [ "$4" = "add" ]; then\n'
|
||||
f' n=$(cat "{counter}" 2>/dev/null || echo 0); n=$((n+1)); echo "$n" > "{counter}"\n'
|
||||
' if [ "$n" -ge 2 ]; then exit 0; fi\n'
|
||||
"fi\n"
|
||||
'exec /usr/bin/git "$@"\n'
|
||||
)
|
||||
(shim / "git").chmod(0o755)
|
||||
|
||||
env = {**os.environ, "PATH": f"{shim}:{os.environ['PATH']}"}
|
||||
r = subprocess.run(
|
||||
["bash", str(PROVE_FIX), "--repo", str(repo), "HEAD"],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
cwd=str(repo),
|
||||
env=env,
|
||||
)
|
||||
assert "PROVEN" not in r.stdout, (
|
||||
"a phase whose worktree does not exist cannot witness anything; pre-fix this "
|
||||
f"produced PROVEN from the subshell's status:\n{r.stdout}\n{r.stderr}"
|
||||
)
|
||||
assert r.returncode == 5, f"expected refusal (5), got {r.returncode}\n{r.stdout}\n{r.stderr}"
|
||||
assert "did not complete" in r.stderr, (
|
||||
f"expected the marker-absence diagnostic, not some other refusal:\n{r.stderr}"
|
||||
)
|
||||
@@ -0,0 +1,328 @@
|
||||
"""`docs/remote-state-inventory.md` covers exactly the in-scope files, each classified for whether
|
||||
it reads live remote state and acts on that read (ersatztv#778).
|
||||
|
||||
WHAT THIS CANNOT DO, said first because #778's own issue body says it: there is no lint for "this
|
||||
code should have pinned a sha." `docs/defect-shapes-773.md` §4 grades detector D as a *fix pattern*
|
||||
whose detector is detector A applied to an enumerated inventory. So this file does not try to grade
|
||||
pinning. It guarantees that every in-scope file has been CLASSIFIED BY SOMEONE, and that no file
|
||||
joined the scope without acquiring a row — which converts "remember to think about this" into "the
|
||||
suite is red until you have".
|
||||
|
||||
The split is deliberate and mirrors `test_guard_inventory.py`:
|
||||
|
||||
* the POPULATION is derived from `git ls-files` and compared for SET EQUALITY, both directions;
|
||||
* the CLASSIFICATION vocabulary is closed, so a typo cannot invent a state;
|
||||
* whether a `PINNED` row is TELLING THE TRUTH is not checked here and cannot be. That stays with
|
||||
review, and the inventory's prose is what review reads.
|
||||
|
||||
SCOPE vs POPULATION, per `testing.guard-derives-population-from-source`: the SCOPE — four
|
||||
directories, their per-directory file patterns, and one excluded subdirectory — is a hand-written
|
||||
policy choice and is reviewable as one. The POPULATION inside that scope is derived on every run
|
||||
from the git index, with no content predicate at all: a file that reads no remote state earns an
|
||||
explicit `N/A` row rather than staying out, which is why most rows are `N/A`.
|
||||
|
||||
No count is written here on purpose. An exact "N of M" has now gone stale FOUR times in this change,
|
||||
most recently in the same commit that demoted two rows — a hand-maintained number is a second copy of
|
||||
the table, and `docs/guard-inventory.md` earns its counts by having a test assert them. Nothing
|
||||
asserts one here, so nothing states one.
|
||||
|
||||
That is the fourth version, and the history is the point — see the `SCOPE` comment below for the
|
||||
three that failed and what each one hid.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import fnmatch
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
from scripts.tests import tracked_files
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[2]
|
||||
INVENTORY = REPO_ROOT / "docs" / "remote-state-inventory.md"
|
||||
|
||||
CLASSES = {"PINNED", "CAS", "UNSAFE-KNOWN", "N/A"}
|
||||
|
||||
# NO CONTENT FILTER. The population is every file in the scoped directories, and a file that reads
|
||||
# no remote state earns an `N/A` row rather than silently staying out.
|
||||
#
|
||||
# The first version filtered on a token list (`curl`, `wget`, `urllib`, ...) and called that a
|
||||
# SCOPE choice rather than a population filter. Cold review rejected the distinction and was right:
|
||||
# the list omitted `git fetch`, which is this repo's most common remote read, so
|
||||
# `.claude/hooks/prepush-rebase-check.sh` — which fetches `origin/main` and derives a PUSH DECISION
|
||||
# from it — was structurally invisible to a guard whose stated claim is "every executable that
|
||||
# reads live remote state". Three more (`prepush-clean-worktree-check.sh`, `ci-detect-docs-only.sh`,
|
||||
# `refresh-shared-checkout.sh`) were missing for the same reason.
|
||||
#
|
||||
# That is precisely the defect `testing.guard-derives-population-from-source` describes: a filter
|
||||
# cannot see the member that is missing, because the absent member is not a row the predicate
|
||||
# rejected, it is a row that was never produced. The defence offered for it — "over-inclusion is the
|
||||
# safe direction" — was answered by the filter ALSO under-including. Enumerating the directories
|
||||
# costs more rows and has no blind spot; that is the trade the rule already made.
|
||||
SCOPE = (
|
||||
("scripts", ("*.sh", "*.py")),
|
||||
(".claude/hooks", ("*.sh",)),
|
||||
(".husky", ("*",)),
|
||||
(".gitea/workflows", ("*.yml", "*.yaml")),
|
||||
)
|
||||
|
||||
# THE POPULATION IS DERIVED FROM `git ls-files`, NOT FROM THE FILESYSTEM.
|
||||
#
|
||||
# This is the third time this population has been found incomplete or wrong, each time by a
|
||||
# different mechanism, and the third fix is the one that stops patching the traversal:
|
||||
#
|
||||
# 1. a content filter on an outbound-network token list, which omitted `git fetch` — this repo's
|
||||
# commonest remote read — so a hook that fetches `origin/main` and derives a PUSH DECISION was
|
||||
# structurally invisible;
|
||||
# 2. a non-recursive `Path.glob`, which missed four nested files including one that calls a live
|
||||
# ErsatzTV API and acts on the reply;
|
||||
# 3. `Path.rglob`, which is recursive and therefore ALSO enumerated `.husky/_/` — 17 husky shims
|
||||
# generated by `npm ci` via web/package.json's `prepare` script, gitignored (`.husky/_/.gitignore`
|
||||
# is `*`) and untracked. That made this guard RED on every developer checkout while staying green
|
||||
# in CI, whose `script-tests` job checks out and pip-installs but never runs `npm ci`. A guard
|
||||
# that fails everywhere except where it runs is worse than no guard: it trains its readers to
|
||||
# ignore it, and it would have done so on the artifact whose entire thesis is population
|
||||
# correctness.
|
||||
#
|
||||
# The lesson each time was the same one this repo already wrote down — derive the population from an
|
||||
# AUTHORITATIVE source — and the filesystem is not one. It reports build output, editor droppings and
|
||||
# anything else that happens to be on disk, and it varies per machine. The repo's index is
|
||||
# authoritative: it holds the same set of files every checkout receives from a clone, and it
|
||||
# excludes untracked generated files by construction rather than by an exclusion list that must be
|
||||
# maintained.
|
||||
#
|
||||
# `scripts/tests/` is still excluded explicitly, because those files ARE tracked. That exclusion is a
|
||||
# scope decision, reviewable in one line: they run only under pytest and authorize nothing. Their
|
||||
# network activity is CONFINED rather than absent — `test_hook_fire_log.py` starts a real
|
||||
# `http.server` on 127.0.0.1 and drives it with real `curl`, and several suites create real local git
|
||||
# remotes — but all of it is fixture state the test creates and tears down, so there is no live remote
|
||||
# to race.
|
||||
EXCLUDED_DIRS = ("scripts/tests",)
|
||||
|
||||
# `| ` + backticked path, optionally followed by ` — <site description>`, then the class cell.
|
||||
_ROW = re.compile(r"^\|\s*`([^`]+?)`[^|]*\|\s*`([^`]+)`\s*\|", re.M)
|
||||
|
||||
|
||||
def _tracked_files() -> list[str]:
|
||||
"""Every file git tracks, as repo-relative posix paths.
|
||||
|
||||
A thin wrapper over the SHARED derivation rather than a second copy of it (ersatztv#806): one
|
||||
definition of the rule, not two, which is detector C — dedup by construction — applied to the
|
||||
file that first stated the rule. The wrapper survives because this module patches
|
||||
`_tracked_files` by name in its own proofs, and because its scope is RECURSIVE over `scripts/`
|
||||
where `tracked_children` is deliberately flat.
|
||||
|
||||
It therefore carries BOTH its own proofs and a row in that file's `DERIVATIONS`, which is not
|
||||
the duplication that masks: they cut at different seams — the in-file pair patches
|
||||
`_tracked_files` (this wrapper), the shared pair patches `_git_ls_files` (the subprocess) — and
|
||||
each was witnessed red independently, so neither can hide the other's total failure.
|
||||
|
||||
Fails LOUDLY rather than returning nothing, now including git's own stderr: an empty population
|
||||
would make every completeness assertion below pass vacuously, which is the exact failure this
|
||||
guard exists to prevent.
|
||||
"""
|
||||
return tracked_files._git_ls_files()
|
||||
|
||||
|
||||
def derived_population() -> set[str]:
|
||||
"""Every in-scope tracked file, as a repo-relative posix path.
|
||||
|
||||
No content predicate of any kind: a file that reads no remote state earns an explicit `N/A` row
|
||||
rather than silently staying out.
|
||||
"""
|
||||
found: set[str] = set()
|
||||
for path in _tracked_files():
|
||||
if any(path == d or path.startswith(d + "/") for d in EXCLUDED_DIRS):
|
||||
continue
|
||||
for directory, globs in SCOPE:
|
||||
if not (path == directory or path.startswith(directory + "/")):
|
||||
continue
|
||||
name = path.rsplit("/", 1)[-1]
|
||||
if any(fnmatch.fnmatch(name, pattern) for pattern in globs):
|
||||
found.add(path)
|
||||
# First matching scope entry wins. Safe only while no scope directory nests inside
|
||||
# another; if one ever does, the inner entry's patterns would be silently skipped.
|
||||
break
|
||||
return found
|
||||
|
||||
|
||||
def _inventory_section(text: str) -> str:
|
||||
"""Only the classification tables, never the surrounding prose.
|
||||
|
||||
This is the second time this parser has read the document's own explanation as data: the
|
||||
UNSAFE-KNOWN justification check once parsed the "Columns" paragraph that DEFINES
|
||||
`UNSAFE-KNOWN`, and adding a scope TABLE to the heading made three more prose rows parse as
|
||||
sites. A guard that treats its own documentation as input is the failure this whole change is
|
||||
about, so the boundary is explicit rather than left to a cleverer regex.
|
||||
"""
|
||||
for heading in ("## The inventory", "## Limits"):
|
||||
if heading not in text:
|
||||
raise AssertionError(
|
||||
f"{INVENTORY.name} has no {heading!r} heading. Row parsing is bounded by "
|
||||
"'## The inventory' and '## Limits'; renaming or reordering either one would "
|
||||
"silently change which rows are checked, so it fails here instead."
|
||||
)
|
||||
start = text.index("## The inventory")
|
||||
try:
|
||||
end = text.index("## Limits", start)
|
||||
except ValueError:
|
||||
# Reachable when '## Limits' exists but PRECEDES '## The inventory' — the presence check
|
||||
# above passes and the bounded search does not. An earlier version put an `end <= start`
|
||||
# guard here instead, which `str.index(…, start)` makes unreachable by construction: it
|
||||
# either returns an index >= start or raises. A guard that cannot execute proves nothing.
|
||||
raise AssertionError(
|
||||
f"{INVENTORY.name}: '## Limits' precedes '## The inventory', so the parsed window "
|
||||
"would be empty and every completeness assertion would pass vacuously."
|
||||
) from None
|
||||
return text[start:end]
|
||||
|
||||
|
||||
def inventory_rows(text: str | None = None) -> list[tuple[str, str]]:
|
||||
if text is None:
|
||||
text = INVENTORY.read_text(encoding="utf-8")
|
||||
return _ROW.findall(_inventory_section(text))
|
||||
|
||||
|
||||
def inventory_sites(text: str | None = None) -> set[str]:
|
||||
return {site for site, _ in inventory_rows(text)}
|
||||
|
||||
|
||||
def test_the_inventory_file_exists_and_is_not_empty():
|
||||
assert INVENTORY.is_file(), f"{INVENTORY} is missing"
|
||||
assert INVENTORY.stat().st_size > 0
|
||||
|
||||
|
||||
def test_anti_vacuity_the_derivation_and_the_table_both_found_something():
|
||||
"""The characteristic failure of a completeness check is reporting success over an empty
|
||||
population. Both sides get a floor, because either one collapsing to zero would make the set
|
||||
comparison below pass trivially."""
|
||||
population = derived_population()
|
||||
sites = inventory_sites()
|
||||
assert len(population) >= 40, (
|
||||
f"derived only {len(population)} in-scope files — the globs are broken, not the repo "
|
||||
"(the scope held 59 files on 2026-08-16, and it only grows)"
|
||||
)
|
||||
assert len(sites) >= 40, (
|
||||
f"parsed only {len(sites)} rows out of the inventory — the row regex has drifted from the table format"
|
||||
)
|
||||
|
||||
|
||||
def test_every_in_scope_file_has_a_row_and_every_row_names_a_real_file():
|
||||
"""Set equality in BOTH directions, because the two failures are different defects and a single
|
||||
'sets differ' message invites fixing one and re-running.
|
||||
|
||||
MISSING: a script that talks to a remote service and was never classified — the defect #778
|
||||
exists to prevent. PHANTOM: a row for a file that was renamed or deleted, which leaves the table
|
||||
claiming coverage it has lost.
|
||||
"""
|
||||
population = derived_population()
|
||||
sites = inventory_sites()
|
||||
missing = sorted(population - sites)
|
||||
phantom = sorted(sites - population)
|
||||
assert not missing, (
|
||||
"in scope but absent from docs/remote-state-inventory.md (classify each as "
|
||||
f"PINNED / CAS / UNSAFE-KNOWN / N/A): {missing}"
|
||||
)
|
||||
assert not phantom, f"listed in docs/remote-state-inventory.md but no such in-scope file exists: {phantom}"
|
||||
|
||||
|
||||
def test_MUTATION_PROOF_a_dropped_row_and_a_phantom_row_are_both_detected():
|
||||
"""The proof that the set comparison above is load-bearing (`testing.guard-ships-with-mutation-
|
||||
proof`). This guard IS a test, so disarming it makes it absent rather than red; the admissible
|
||||
proof is therefore the contrapositive — introduce the defect into an isolated copy of the
|
||||
GUARDED ARTIFACT and show the comparison reports it.
|
||||
|
||||
Both directions are mutated, because they are different defects: a dropped row is an
|
||||
unclassified script, a phantom row is a table claiming coverage it has lost. This ran for real
|
||||
on the day it was written — `dependency-scan.yml` was genuinely absent from the first draft of
|
||||
the inventory and this comparison is what found it.
|
||||
"""
|
||||
text = INVENTORY.read_text(encoding="utf-8")
|
||||
population = derived_population()
|
||||
|
||||
victim = sorted(population)[0]
|
||||
dropped = "\n".join(line for line in text.splitlines() if not line.startswith(f"| `{victim}`"))
|
||||
assert victim not in inventory_sites(dropped), (
|
||||
f"the mutation did not actually remove {victim}; the proof below would be vacuous"
|
||||
)
|
||||
assert population - inventory_sites(dropped), (
|
||||
"a row was removed from the inventory and the comparison still reported complete coverage"
|
||||
)
|
||||
|
||||
# Inserted INSIDE the inventory section, not appended to the file: rows are parsed only between
|
||||
# "## The inventory" and "## Limits", so appending at the end would test nothing.
|
||||
phantom = text.replace("## Limits", "| `scripts/does-not-exist.sh` — invented | `PINNED` | n/a |\n\n## Limits", 1)
|
||||
assert inventory_sites(phantom) - population == {"scripts/does-not-exist.sh"}, (
|
||||
"a row naming a file that does not exist was not reported as phantom"
|
||||
)
|
||||
|
||||
|
||||
def test_every_class_cell_comes_from_the_closed_vocabulary():
|
||||
bad = sorted({cls for _, cls in inventory_rows() if cls not in CLASSES})
|
||||
assert not bad, (
|
||||
f"unknown classification(s) {bad}; allowed: {sorted(CLASSES)}. A typo here would silently "
|
||||
"create a state nobody reviews."
|
||||
)
|
||||
|
||||
|
||||
def test_every_unsafe_row_states_why_the_residual_is_accepted():
|
||||
"""`UNSAFE-KNOWN` means 'accepted with a reason', not 'noticed'. A row that records the window
|
||||
without the argument for tolerating it is how a deferral becomes permanent by default."""
|
||||
text = _inventory_section(INVENTORY.read_text(encoding="utf-8"))
|
||||
thin = []
|
||||
for line in text.splitlines():
|
||||
# TABLE ROWS ONLY. The first version matched any line containing the token, so the prose in
|
||||
# "Columns" that DEFINES `UNSAFE-KNOWN` was parsed as a row and the split blew up. A guard
|
||||
# that reads its own documentation as data is the failure this whole change is about.
|
||||
if not line.startswith("|") or "`UNSAFE-KNOWN`" not in line:
|
||||
continue
|
||||
cells = [c.strip() for c in line.strip().strip("|").split("|")]
|
||||
if len(cells) < 3:
|
||||
continue
|
||||
site, note = cells[0], cells[-1]
|
||||
if len(note) < 120:
|
||||
thin.append(site[:60])
|
||||
assert not thin, f"UNSAFE-KNOWN row(s) with no stated justification: {thin}"
|
||||
|
||||
|
||||
def test_the_population_never_includes_a_file_git_does_not_track(monkeypatch):
|
||||
"""The regression for the third population defect, and the reason the source is the index.
|
||||
|
||||
`Path.rglob` enumerated `.husky/_/` — 17 husky shims generated by `npm ci`, gitignored and
|
||||
untracked — so this guard was RED on every developer checkout and GREEN in CI, which never runs
|
||||
`npm ci`. A guard that fails everywhere except where it runs trains its readers to ignore it.
|
||||
|
||||
Asserted through the mechanism rather than against the current disk, so it holds on a machine
|
||||
that has never installed husky: the tracked list is narrowed, and anything outside it must
|
||||
disappear from the population even though it is still sitting on disk and still matches the
|
||||
scope globs.
|
||||
"""
|
||||
real = derived_population()
|
||||
assert real, "empty population — the derivation is broken, not the repo"
|
||||
|
||||
victim = sorted(real)[0]
|
||||
tracked = [p for p in _tracked_files() if p != victim]
|
||||
monkeypatch.setattr(
|
||||
"scripts.tests.test_remote_state_inventory._tracked_files",
|
||||
lambda: tracked,
|
||||
raising=False,
|
||||
)
|
||||
# Patch the module object this test is running inside, whatever name it was imported under.
|
||||
import sys
|
||||
|
||||
mod = sys.modules[__name__]
|
||||
monkeypatch.setattr(mod, "_tracked_files", lambda: tracked)
|
||||
|
||||
assert (REPO_ROOT / victim).is_file(), f"{victim} must still exist on disk for this proof to mean anything"
|
||||
assert victim not in derived_population(), (
|
||||
f"{victim} is on disk and matches the scope, but git no longer tracks it — it must not enter "
|
||||
"the population, or untracked build output can redden this guard again"
|
||||
)
|
||||
|
||||
|
||||
def test_every_derived_member_is_tracked():
|
||||
"""The same property stated as an invariant over the real tree, so a future refactor that goes
|
||||
back to walking the filesystem fails here rather than on someone's laptop."""
|
||||
tracked = set(_tracked_files())
|
||||
stray = sorted(p for p in derived_population() if p not in tracked)
|
||||
assert not stray, f"population contains untracked path(s): {stray}"
|
||||
@@ -0,0 +1,490 @@
|
||||
"""The worktree-ownership mechanism is TWO files, and this drives both halves as one thing.
|
||||
|
||||
`pretooluse-worktree-guard.sh` denies a `git commit`/`git merge` inside a worktree another session
|
||||
created. It can only do that because `posttooluse-worktree-marker.sh` wrote the
|
||||
`.claude-worktree-owner` marker at `git worktree add` time. Neither file had a test, and — the part
|
||||
that makes this rank second in ersatztv#785 — **the halves had never been exercised together**, so a
|
||||
regression in either one is invisible: the marker hook silently writing nothing and the guard hook
|
||||
silently reading nothing produce the identical outcome, which is *the commit is allowed*, which is
|
||||
also what a correct fail-open looks like.
|
||||
|
||||
Both hooks are deliberately fail-open (`docs/decisions` — the main tree is never marked, and
|
||||
pre-convention worktrees have no marker), and that is exactly why an absent mechanism is
|
||||
indistinguishable from a working one from the outside. It is the shape
|
||||
`testing.guard-ships-with-mutation-proof` was written for: "in every case a human had read the guard
|
||||
and believed it worked. The guard was not subtly wrong, it was *absent*."
|
||||
|
||||
So this file:
|
||||
|
||||
* drives the REAL pair end to end over a REAL `git worktree add`, in the real payload shape —
|
||||
marker hook first as the harness would fire it, then the guard hook;
|
||||
* carries a negative control (a non-mutating git command) and a fail-open control (an unmarked
|
||||
worktree), because a guard that denied everything would satisfy the deny assertions;
|
||||
* and performs FOUR clause-level mutations: the guard's marker read, the guard's ownership
|
||||
comparison, the command-detection alternation (`commit|merge`), and the *other file's* marker
|
||||
write. The last is the one that could not exist while the halves were tested apart.
|
||||
|
||||
These four are the clauses whose disarm this file detects. They are not every line in either hook —
|
||||
the `git -C` / `cd` redirection extraction and the marker hook's argument parsing are exercised
|
||||
behaviourally but not mutated, and the grade in `docs/guard-inventory.md` covers the clause its
|
||||
cited case mutates, not the whole file.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[2]
|
||||
GUARD = REPO_ROOT / ".claude" / "hooks" / "pretooluse-worktree-guard.sh"
|
||||
MARKER_HOOK = REPO_ROOT / ".claude" / "hooks" / "posttooluse-worktree-marker.sh"
|
||||
MARKER_NAME = ".claude-worktree-owner"
|
||||
|
||||
SESSION_A = "session-aaaa-1111"
|
||||
SESSION_B = "session-bbbb-2222"
|
||||
|
||||
|
||||
def _env() -> dict:
|
||||
"""The subprocess environment, built PER CALL — never snapshotted at import.
|
||||
|
||||
Two things it must get right.
|
||||
|
||||
`CLAUDE_PROJECT_DIR` is pinned because the hooks resolve `scripts/hook-fire-log.sh` from it,
|
||||
falling back to a path relative to their own location; a MUTATED copy lives in tmp_path, where
|
||||
that fallback finds nothing. Without the pin the mutant differs from the subject in a second way
|
||||
and the comparison stops being about the mutated clause.
|
||||
|
||||
And it is a FUNCTION rather than a module-level dict because `conftest.py`'s autouse
|
||||
`isolate_hook_fire_log` fixture monkeypatches `ETV_HOOK_FIRE_LOG_DIR` into `os.environ` at test
|
||||
setup — which happens AFTER this module is imported. A `{**os.environ}` snapshot taken at import
|
||||
time captures the environment as it was before the fixture ran, so every hook subprocess writes
|
||||
to the REAL `$HOME/.cache/ersatztv/hook-fire/` log instead of the fixture's tmp dir. That is not
|
||||
untidiness: it is #776's defect reintroduced in the file that is meant to prove #776's hooks,
|
||||
and it corrupts the `hook-fire-log.sh report` surface this repo cites as the observability claim
|
||||
for every guard still graded NONE. The reproduction, rather than a figure whose evidence has
|
||||
since been deleted: reintroduce the snapshot and run this file, then count records for the two
|
||||
synthetic session ids below — 58 per run, on macOS and Linux alike.
|
||||
"""
|
||||
return {**os.environ, "CLAUDE_PROJECT_DIR": str(REPO_ROOT)}
|
||||
|
||||
|
||||
def _git(cwd: Path, *args: str) -> str:
|
||||
p = subprocess.run(
|
||||
["git", *args],
|
||||
cwd=str(cwd),
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
env={
|
||||
**os.environ,
|
||||
"GIT_AUTHOR_NAME": "t",
|
||||
"GIT_AUTHOR_EMAIL": "t@e",
|
||||
"GIT_COMMITTER_NAME": "t",
|
||||
"GIT_COMMITTER_EMAIL": "t@e",
|
||||
},
|
||||
)
|
||||
return p.stdout
|
||||
|
||||
|
||||
def _scratch_repo(tmp_path: Path) -> Path:
|
||||
repo = tmp_path / "main-tree"
|
||||
repo.mkdir()
|
||||
_git(repo, "init", "-q", "-b", "main", ".")
|
||||
(repo / "f.txt").write_text("one\n")
|
||||
_git(repo, "add", "f.txt")
|
||||
_git(repo, "commit", "-qm", "init")
|
||||
return repo
|
||||
|
||||
|
||||
def _run(hook: Path, payload: dict, cwd: Path) -> tuple[int, bytes]:
|
||||
p = subprocess.run(
|
||||
["bash", str(hook)],
|
||||
input=json.dumps(payload).encode(),
|
||||
capture_output=True,
|
||||
cwd=str(cwd),
|
||||
env=_env(),
|
||||
timeout=60,
|
||||
)
|
||||
return p.returncode, p.stdout
|
||||
|
||||
|
||||
def _add_worktree(repo: Path, name: str, marker_hook: Path | None, session: str) -> Path:
|
||||
"""`git worktree add` exactly as a session does it, then fire the PostToolUse marker hook.
|
||||
|
||||
The marker hook is driven with the payload the harness would hand it AFTER the command
|
||||
succeeded, which is when PostToolUse fires — not a hand-planted marker file. A hand-planted
|
||||
marker would make every deny below a test of the guard alone, and the untested seam is the
|
||||
handoff between the two files.
|
||||
"""
|
||||
wt = repo.parent / name
|
||||
_git(repo, "worktree", "add", "-q", str(wt))
|
||||
if marker_hook is not None:
|
||||
rc, _ = _run(
|
||||
marker_hook,
|
||||
{
|
||||
"session_id": session,
|
||||
"hook_event_name": "PostToolUse",
|
||||
"tool_name": "Bash",
|
||||
"cwd": str(repo),
|
||||
"tool_input": {"command": f"git worktree add {wt}"},
|
||||
},
|
||||
repo,
|
||||
)
|
||||
assert rc == 0, "the marker hook must always exit 0"
|
||||
return wt
|
||||
|
||||
|
||||
def _commit_payload(session: str, cwd: Path, command: str = "git commit -m x") -> dict:
|
||||
return {
|
||||
"session_id": session,
|
||||
"hook_event_name": "PreToolUse",
|
||||
"tool_name": "Bash",
|
||||
"cwd": str(cwd),
|
||||
"tool_input": {"command": command},
|
||||
}
|
||||
|
||||
|
||||
def _denied(out: bytes) -> bool:
|
||||
return b'"permissionDecision": "deny"' in out or b'"permissionDecision":"deny"' in out
|
||||
|
||||
|
||||
# ------------------------------------------------------------------------------------------------
|
||||
# ANTI-VACUITY — if the fixture never produces a marked worktree, every deny below is meaningless
|
||||
# ------------------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_the_subprocess_env_CARRIES_the_isolated_hook_fire_log_dir():
|
||||
"""The hooks these tests drive must log to the fixture's dir, never the production one.
|
||||
|
||||
`conftest.py`'s autouse `isolate_hook_fire_log` monkeypatches `ETV_HOOK_FIRE_LOG_DIR` into
|
||||
`os.environ` at test setup. Anything that snapshots `os.environ` at IMPORT time captures the
|
||||
value from before the fixture ran and silently defeats it — the hooks then append to
|
||||
`$HOME/.cache/ersatztv/hook-fire/`, which is #776's defect reintroduced inside the file that
|
||||
proves #776's hooks, corrupting the one surface this repo cites as the observability claim for
|
||||
every guard still graded NONE.
|
||||
|
||||
It is invisible from the outside: the tests pass either way, because the fire-log library is
|
||||
fail-open by design. So it needs its own assertion.
|
||||
"""
|
||||
env = _env()
|
||||
isolated = os.environ.get("ETV_HOOK_FIRE_LOG_DIR")
|
||||
assert isolated, "the autouse isolation fixture did not run; conftest.py is not being loaded"
|
||||
assert env.get("ETV_HOOK_FIRE_LOG_DIR") == isolated, (
|
||||
"the subprocess environment does not carry the isolated log dir, so every hook driven by "
|
||||
"this file is writing into the production hook-fire log. Build the env per call; do not "
|
||||
f"snapshot os.environ at import time. env has {env.get('ETV_HOOK_FIRE_LOG_DIR')!r}"
|
||||
)
|
||||
# The path `hook-fire-log.sh` falls back to when ETV_HOOK_FIRE_LOG_DIR is unset, derived the
|
||||
# same way it derives it rather than restated as a literal.
|
||||
production = Path(os.environ.get("HOME", "/tmp")) / ".cache" / "ersatztv" / "hook-fire" # noqa: S108 — mirrors hook-fire-log.sh's own ${HOME:-/tmp}
|
||||
assert Path(isolated).resolve() != production.resolve(), (
|
||||
f"the 'isolated' log dir IS the production one ({production}), so the fixture is isolating "
|
||||
"nothing and this test would pass while the leak continued"
|
||||
)
|
||||
|
||||
|
||||
def test_driving_a_hook_LANDS_its_records_in_the_ISOLATED_dir(tmp_path):
|
||||
"""The invariant, asserted at the EFFECT rather than at the helper that is supposed to produce it.
|
||||
|
||||
`test_the_subprocess_env_CARRIES_...` above checks `_env()`'s return value, and that is not the
|
||||
same claim: `_env()` can be perfectly correct while a call site passes something else. Cold
|
||||
review demonstrated exactly that — restore the module-level snapshot and change one `env=_env()`
|
||||
back to `env=_ENV`, and all thirteen tests pass while 54 records leak into the real log. The
|
||||
guard was pinned to the shape of the fix instead of to the property, which is
|
||||
`verify-against-the-REAL-predecessor`: a hand-written revert is not the code a future tidy-up
|
||||
produces.
|
||||
|
||||
So this drives a real hook through the real `_run()` and asserts the records landed where the
|
||||
fixture put them.
|
||||
|
||||
ITS SCOPE, stated because the first version of this docstring claimed more than it delivers: it
|
||||
guards THE LAUNCH PATH IT DRIVES, not the file. Cold review demonstrated the gap — add a second
|
||||
launcher alongside `_run()` that passes a stale snapshot and point the mutation tests at it, and
|
||||
this test stays green while 18 records leak, because the hooks IT drives still log correctly.
|
||||
Every hook in this file goes through `_run()` today, which is what makes the guard sufficient
|
||||
HERE and not a general property. The general form is a suite-level check, tracked in #809; the
|
||||
reason it is hard is that the obvious version — diff the production log around each test — races
|
||||
against a real session's hooks firing concurrently.
|
||||
"""
|
||||
isolated = Path(os.environ["ETV_HOOK_FIRE_LOG_DIR"])
|
||||
before = {p.name for p in isolated.glob("*.jsonl")} if isolated.exists() else set()
|
||||
|
||||
repo = _scratch_repo(tmp_path)
|
||||
wt = _add_worktree(repo, "wt-a", MARKER_HOOK, SESSION_A)
|
||||
rc, out = _run(GUARD, _commit_payload(SESSION_B, wt), wt)
|
||||
|
||||
# Anti-vacuity: if the hook decided nothing, it may simply have had nothing to log.
|
||||
assert rc == 0 and _denied(out), f"the hook reached no decision, so 'records landed' would prove nothing: {out!r}"
|
||||
|
||||
after = {p.name for p in isolated.glob("*.jsonl")} if isolated.exists() else set()
|
||||
assert after > before, (
|
||||
f"driving two hooks added no record to the isolated log dir {isolated}. Either the "
|
||||
"instrumentation stopped firing, or these hooks are logging somewhere else — and the only "
|
||||
"somewhere else is the production log this file must never touch"
|
||||
)
|
||||
|
||||
|
||||
def test_the_marker_hook_really_marks_the_worktree_it_was_told_about(tmp_path):
|
||||
repo = _scratch_repo(tmp_path)
|
||||
wt = _add_worktree(repo, "wt-a", MARKER_HOOK, SESSION_A)
|
||||
|
||||
marker = wt / MARKER_NAME
|
||||
assert marker.is_file(), (
|
||||
f"no {MARKER_NAME} in {wt}. Without it the guard has nothing to read and every 'denied' "
|
||||
"assertion in this file would be testing a mechanism that is not there"
|
||||
)
|
||||
assert marker.read_text().strip() == SESSION_A, (
|
||||
f"the marker names {marker.read_text().strip()!r}, not the session that created the "
|
||||
"worktree — ownership would be attributed to the wrong session"
|
||||
)
|
||||
|
||||
|
||||
def test_the_marker_hook_ignores_a_command_that_is_not_a_worktree_add(tmp_path):
|
||||
"""The write side's own negative control: a hook that marked on any command would pass above."""
|
||||
repo = _scratch_repo(tmp_path)
|
||||
wt = repo.parent / "wt-unrelated"
|
||||
_git(repo, "worktree", "add", "-q", str(wt))
|
||||
rc, _ = _run(
|
||||
MARKER_HOOK,
|
||||
{
|
||||
"session_id": SESSION_A,
|
||||
"hook_event_name": "PostToolUse",
|
||||
"tool_name": "Bash",
|
||||
"cwd": str(repo),
|
||||
"tool_input": {"command": f"ls {wt}"},
|
||||
},
|
||||
repo,
|
||||
)
|
||||
assert rc == 0
|
||||
assert not (wt / MARKER_NAME).exists(), "the marker hook stamped a worktree it never created"
|
||||
|
||||
|
||||
# ------------------------------------------------------------------------------------------------
|
||||
# THE PAIR DECIDES — both halves, in sequence, as the harness fires them
|
||||
# ------------------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_a_commit_in_ANOTHER_sessions_worktree_is_DENIED(tmp_path):
|
||||
"""The #289 case the mechanism exists for, end to end across both files."""
|
||||
repo = _scratch_repo(tmp_path)
|
||||
wt = _add_worktree(repo, "wt-a", MARKER_HOOK, SESSION_A)
|
||||
|
||||
rc, out = _run(GUARD, _commit_payload(SESSION_B, wt), wt)
|
||||
assert rc == 0, "the hook communicates by printing, and must always exit 0"
|
||||
assert _denied(out), f"a commit into session A's worktree was not denied from session B: {out!r}"
|
||||
assert SESSION_A.encode() in out, (
|
||||
"the deny reason must name the owning session — without it the operator cannot tell whether "
|
||||
f"the marker is stale or the worktree is genuinely foreign: {out!r}"
|
||||
)
|
||||
|
||||
|
||||
def test_a_commit_in_MY_OWN_worktree_is_ALLOWED(tmp_path):
|
||||
"""Negative control. A guard that denied every marked worktree would pass the test above."""
|
||||
repo = _scratch_repo(tmp_path)
|
||||
wt = _add_worktree(repo, "wt-a", MARKER_HOOK, SESSION_A)
|
||||
|
||||
rc, out = _run(GUARD, _commit_payload(SESSION_A, wt), wt)
|
||||
assert rc == 0
|
||||
assert out == b"", f"the owning session was blocked from committing in its own worktree: {out!r}"
|
||||
|
||||
|
||||
def test_an_UNMARKED_worktree_is_ALLOWED(tmp_path):
|
||||
"""The deliberate fail-open: pre-convention worktrees and the main tree carry no marker."""
|
||||
repo = _scratch_repo(tmp_path)
|
||||
wt = _add_worktree(repo, "wt-none", None, SESSION_A)
|
||||
|
||||
rc, out = _run(GUARD, _commit_payload(SESSION_B, wt), wt)
|
||||
assert rc == 0
|
||||
assert out == b"", f"an unmarked worktree was blocked, which breaks the main tree too: {out!r}"
|
||||
|
||||
|
||||
def test_a_NON_MUTATING_git_command_in_a_foreign_worktree_is_ALLOWED(tmp_path):
|
||||
"""Only `commit`/`merge` are guarded; `git status` in a sibling worktree is normal work."""
|
||||
repo = _scratch_repo(tmp_path)
|
||||
wt = _add_worktree(repo, "wt-a", MARKER_HOOK, SESSION_A)
|
||||
|
||||
rc, out = _run(GUARD, _commit_payload(SESSION_B, wt, "git status"), wt)
|
||||
assert rc == 0
|
||||
assert out == b"", f"a read-only git command was denied: {out!r}"
|
||||
|
||||
|
||||
def test_a_MERGE_in_a_foreign_worktree_is_DENIED(tmp_path):
|
||||
"""The other half of the guarded alternation.
|
||||
|
||||
Every other deny case here uses `git commit`, so `merge` could be dropped from the detection
|
||||
regex and this file would stay green — the mechanism guards the plumbing-merge path
|
||||
(`process.foreign-worktree-plumbing-merge`) specifically, which makes that the more damaging
|
||||
half to lose.
|
||||
"""
|
||||
repo = _scratch_repo(tmp_path)
|
||||
wt = _add_worktree(repo, "wt-a", MARKER_HOOK, SESSION_A)
|
||||
|
||||
rc, out = _run(GUARD, _commit_payload(SESSION_B, wt, "git merge --no-ff topic"), wt)
|
||||
assert rc == 0
|
||||
assert _denied(out), f"a merge into session A's worktree was not denied: {out!r}"
|
||||
|
||||
|
||||
def test_a_git_C_into_a_foreign_worktree_is_DENIED_from_the_main_tree(tmp_path):
|
||||
"""The redirection that makes the guard non-trivial.
|
||||
|
||||
The session's cwd is its OWN tree — where committing is fine — and only the `-C` argument moves
|
||||
the operation into the foreign worktree. A guard that looked at `cwd` alone would allow this,
|
||||
and `git -C` is how the sibling-worktree commit actually gets typed.
|
||||
"""
|
||||
repo = _scratch_repo(tmp_path)
|
||||
wt = _add_worktree(repo, "wt-a", MARKER_HOOK, SESSION_A)
|
||||
|
||||
rc, out = _run(GUARD, _commit_payload(SESSION_B, repo, f"git -C {wt} commit -m x"), repo)
|
||||
assert rc == 0
|
||||
assert _denied(out), f"`git -C <foreign worktree> commit` was not denied: {out!r}"
|
||||
|
||||
|
||||
# ------------------------------------------------------------------------------------------------
|
||||
# MUTATION PROOFS — four clauses, one per thing the mechanism hangs on
|
||||
#
|
||||
# Each mutant is an isolated copy with ONE clause disarmed, and each test asserts the UNMUTATED pair
|
||||
# reaches the opposite decision on the same fixture FIRST. Without that positive control a mutation
|
||||
# proof passes when the mechanism detects nothing at all, which is how the BOM guard sat fail-open
|
||||
# for months while reading as covered.
|
||||
# ------------------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _mutate(src: Path, tmp_path: Path, old: str, new: str, why: str) -> Path:
|
||||
assert old in src.read_text(), (
|
||||
f"the clause {old!r} has moved in {src.name}; RETARGET this mutation rather than loosening "
|
||||
f"it — a mutation that silently stops mutating is the failure this file is about ({why})"
|
||||
)
|
||||
dst = tmp_path / f"mutated-{src.name}"
|
||||
dst.write_text(src.read_text().replace(old, new, 1))
|
||||
return dst
|
||||
|
||||
|
||||
def test_MUTATION_disarming_the_guards_MARKER_READ_stops_the_deny(tmp_path):
|
||||
repo = _scratch_repo(tmp_path)
|
||||
wt = _add_worktree(repo, "wt-a", MARKER_HOOK, SESSION_A)
|
||||
payload = _commit_payload(SESSION_B, wt)
|
||||
|
||||
rc_live, out_live = _run(GUARD, payload, wt)
|
||||
assert rc_live == 0 and _denied(out_live), (
|
||||
"the UNMUTATED guard did not deny, so 'the mutant is silent' would prove nothing about the "
|
||||
f"marker read: {out_live!r}"
|
||||
)
|
||||
|
||||
mutant = _mutate(
|
||||
GUARD,
|
||||
tmp_path,
|
||||
'marker="$root/.claude-worktree-owner"',
|
||||
'marker="$root/.claude-worktree-owner-NOTHING-WRITES-THIS"',
|
||||
"the guard's marker read",
|
||||
)
|
||||
rc, out = _run(mutant, payload, wt)
|
||||
assert rc == 0
|
||||
assert out == b"", (
|
||||
"the guard still denied with its marker read pointed at a file nothing writes, so the deny "
|
||||
f"is not coming from ownership at all: {out!r}"
|
||||
)
|
||||
|
||||
|
||||
def test_MUTATION_inverting_the_OWNERSHIP_COMPARISON_blocks_the_owner(tmp_path):
|
||||
"""The allow direction, which the deny mutation above cannot reach.
|
||||
|
||||
Disarming the comparison the other way would only make the guard deny more, and every deny
|
||||
assertion in this file would stay green. Inverting it is what shows the comparison — rather than
|
||||
the mere presence of a marker — is what decides.
|
||||
"""
|
||||
repo = _scratch_repo(tmp_path)
|
||||
wt = _add_worktree(repo, "wt-a", MARKER_HOOK, SESSION_A)
|
||||
own_payload = _commit_payload(SESSION_A, wt)
|
||||
|
||||
rc_live, out_live = _run(GUARD, own_payload, wt)
|
||||
assert rc_live == 0 and out_live == b"", (
|
||||
f"the UNMUTATED guard already blocked the owner, so the inversion below proves nothing: {out_live!r}"
|
||||
)
|
||||
|
||||
mutant = _mutate(
|
||||
GUARD,
|
||||
tmp_path,
|
||||
'[ "$owner" = "$me" ] && exit 0',
|
||||
'[ "$owner" != "$me" ] && exit 0',
|
||||
"the guard's ownership comparison",
|
||||
)
|
||||
rc, out = _run(mutant, own_payload, wt)
|
||||
assert rc == 0
|
||||
assert _denied(out), (
|
||||
"inverting the ownership comparison did not change the decision for the OWNING session, so "
|
||||
f"the comparison is not what allows it through: {out!r}"
|
||||
)
|
||||
|
||||
|
||||
def test_MUTATION_a_marker_hook_that_stops_WRITING_makes_the_guard_go_quiet(tmp_path):
|
||||
"""THE CROSS-FILE PROOF — the one that could not exist while the halves were tested apart.
|
||||
|
||||
The clause disarmed here is in `posttooluse-worktree-marker.sh`; the assertion is about
|
||||
`pretooluse-worktree-guard.sh`. A regression in the write half is otherwise completely silent:
|
||||
the marker hook exits 0 either way, and the guard's fail-open turns a missing marker into an
|
||||
allowed commit that looks exactly like a correctly allowed one.
|
||||
"""
|
||||
repo = _scratch_repo(tmp_path)
|
||||
|
||||
wt_live = _add_worktree(repo, "wt-live", MARKER_HOOK, SESSION_A)
|
||||
rc_live, out_live = _run(GUARD, _commit_payload(SESSION_B, wt_live), wt_live)
|
||||
assert rc_live == 0 and _denied(out_live), (
|
||||
f"the UNMUTATED pair did not deny, so a silent mutant proves nothing about the marker write: {out_live!r}"
|
||||
)
|
||||
|
||||
mutant_marker = _mutate(
|
||||
MARKER_HOOK,
|
||||
tmp_path,
|
||||
'printf \'%s\\n\' "$me" > "$abs/.claude-worktree-owner" 2>/dev/null || true',
|
||||
"true",
|
||||
"the marker hook's write",
|
||||
)
|
||||
wt_dead = _add_worktree(repo, "wt-dead", mutant_marker, SESSION_A)
|
||||
assert not (wt_dead / MARKER_NAME).exists(), (
|
||||
"the mutated marker hook wrote a marker anyway — the mutation did not disarm the write, so "
|
||||
"the assertion below would be about nothing"
|
||||
)
|
||||
|
||||
rc, out = _run(GUARD, _commit_payload(SESSION_B, wt_dead), wt_dead)
|
||||
assert rc == 0
|
||||
assert out == b"", (
|
||||
"the guard denied a commit in a worktree that carries NO marker, which means the deny in "
|
||||
f"the live case above is not evidence that the two halves are connected: {out!r}"
|
||||
)
|
||||
|
||||
|
||||
def test_MUTATION_dropping_MERGE_from_the_detection_clause_stops_denying_a_merge(tmp_path):
|
||||
"""The alternation is two guarded operations, and losing one of them is silent.
|
||||
|
||||
This mutation is deliberately narrow: it must stop the guard denying a `merge` while leaving it
|
||||
denying a `commit`. Asserting both is what distinguishes "the alternation is load-bearing" from
|
||||
"the mutant broke the regex", which would redden everything and prove nothing about `merge`.
|
||||
"""
|
||||
repo = _scratch_repo(tmp_path)
|
||||
wt = _add_worktree(repo, "wt-a", MARKER_HOOK, SESSION_A)
|
||||
merge_payload = _commit_payload(SESSION_B, wt, "git merge --no-ff topic")
|
||||
commit_payload = _commit_payload(SESSION_B, wt)
|
||||
|
||||
rc_live, out_live = _run(GUARD, merge_payload, wt)
|
||||
assert rc_live == 0 and _denied(out_live), (
|
||||
f"the UNMUTATED guard did not deny a merge, so a silent mutant proves nothing: {out_live!r}"
|
||||
)
|
||||
|
||||
mutant = _mutate(
|
||||
GUARD,
|
||||
tmp_path,
|
||||
"(commit|merge)\\b",
|
||||
"(commit)\\b",
|
||||
"the command-detection alternation",
|
||||
)
|
||||
rc, out = _run(mutant, merge_payload, wt)
|
||||
assert rc == 0
|
||||
assert out == b"", f"dropping `merge` from the alternation did not stop the merge being denied: {out!r}"
|
||||
|
||||
rc_c, out_c = _run(mutant, commit_payload, wt)
|
||||
assert rc_c == 0 and _denied(out_c), (
|
||||
"the mutant stopped denying COMMITS too, so it broke detection wholesale rather than "
|
||||
f"removing the merge alternative — this proves nothing about `merge`: {out_c!r}"
|
||||
)
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user