"""Every hook reports that it fired, and the reporting changes nothing else (ersatztv#776). Two separable claims, and conflating them is how instrumentation ships as a regression: 1. COVERAGE — every hook script records its own execution. The population is DERIVED from the tracked `.claude/hooks/*.sh`, never listed, per `testing.guard-derives-population-from-source`, so a hook staged tomorrow is uninstrumented-and-red rather than silently unobserved. From the git index rather than a filesystem walk since ersatztv#806 — an untracked file on one machine is not part of the repo, and a guard whose population differs per checkout is one nobody trusts. 2. TRANSPARENCY — the wrapper is invisible to the harness. It slurps stdin and replays it, and it diverts stdout and replays it, which puts it directly in the path of the most load-bearing guards in this repo. If it drops a byte of a `deny` payload or swallows an exit code, it disables a guard while reporting that the guard fired. That is strictly worse than the blindness #776 set out to fix, so the differential test below drives EVERY hook, with and without the instrumentation, over a payload matrix and demands byte-equal stdout and equal exit status. The mutation proof for claim 1 is the contrapositive form allowed by `testing.guard-ships-with-mutation-proof` for a checker-guard: the defect the guard exists to catch (an uninstrumented hook) is introduced into an isolated copy of the guarded artifact, and the named check must go red. See `test_a_hook_that_LOSES_its_instrumentation_is_DETECTED`. Claim 2's proof is not a mutation of an assertion but a real A/B over the real scripts, which is the only thing that can establish it — a unit test of the wrapper in isolation would pass on a wrapper that is fine alone and lethal in context. """ from __future__ import annotations import os import pty import re import shlex import shutil import subprocess import sys from pathlib import Path import pytest from scripts.tests.hook_fire_isolation import ( ENV_VAR, SHARED_LOG_DIRS, isolation_violation, resolved_log_dir, ) from scripts.tests.tracked_files import tracked_paths REPO_ROOT = Path(__file__).resolve().parents[2] # Directory + patterns resolved against the GIT INDEX, not `Path.glob` (ersatztv#806) — see # `scripts/tests/tracked_files.py`. `HOOKS_DIR` survives only for error messages. HOOKS = (".claude/hooks", ("*.sh",)) HUSKY = (".husky", ("*",)) HOOKS_DIR = REPO_ROOT / ".claude" / "hooks" SINK = REPO_ROOT / "scripts" / "hook-fire-log.sh" # CAPTURED AT IMPORT TIME ON PURPOSE — this is a specimen, not an oversight. # # `{**os.environ}` at module level is the shape #785 shipped: a suite that snapshots the environment # once, at collection, and hands that stale mapping to every subprocess it launches. Because the # autouse isolation fixture runs at test SETUP, such a snapshot used to predate it and every hook # driven from it wrote to `$HOME/.cache/ersatztv/hook-fire/` — 58 records per run, with every # assertion green, since the fire-log library is fail-open by design. # # `test_the_suite_does_not_write_to_the_PRODUCTION_log` asserts this specimen is now SAFE, which is # the cross-suite claim that test has always made in its docstring and never checked in its body. _IMPORT_TIME_ENV = {**os.environ} # The three lines a hook must carry. Matched structurally rather than as one frozen blob so that a # comment reflow does not redden the suite, but the two things that matter — sourcing the single # shared sink, and calling begin with this hook's own name — are both pinned. _SOURCES_SINK = re.compile(r'^\[ -r "\$ETV_HOOK_FIRE_LIB" \] && \. "\$ETV_HOOK_FIRE_LIB" \|\| true$', re.M) _BEGINS = re.compile(r"^etv_hook_fire_begin (\S+) .*\|\| true$", re.M) def hook_scripts() -> list[Path]: """THE POPULATION, from the GIT INDEX. Never a list, and never the filesystem (ersatztv#806). The filesystem is not an authoritative source: an untracked `.sh` dropped in `.claude/hooks/` — a scratch copy, a half-written hook — used to enter this population and be demanded to carry instrumentation, reddening the suite on that checkout while CI, which never sees the file, stayed green. That is #778's third shape, and a guard that fails everywhere except where it runs trains its readers to ignore it. """ return tracked_paths(*HOOKS) def expected_mode(name: str) -> str: """DERIVED from the wiring, not listed. A hook registered in `.claude/settings.json` is a Claude hook: it always exits 0 and decides by PRINTING JSON, so its stdout must be captured or the decision is unobservable. A hook registered in `.husky/*` is a git hook: it decides by exit status and its stdout is progress text a human watches live, so capturing it would hold the output back until the end. Reading this from the settings files rather than a hardcoded set means a hook that changes wiring gets the right expectation automatically — and, unlike a list, it cannot be quietly edited to match a mistake. """ if name in (REPO_ROOT / ".claude" / "settings.json").read_text(): return "capture" for husky in tracked_paths(*HUSKY): if name in husky.read_text(): return "stream" return "capture" def instrumentation_faults(text: str, name: str) -> list[str]: """The check itself, factored out so a mutation can be fed to the same code the suite runs.""" faults = [] if not _SOURCES_SINK.search(text): faults.append(f"{name}: does not source scripts/hook-fire-log.sh") # The sink PATH, not just the sourcing line. Repointing `ETV_HOOK_FIRE_LIB` at /dev/null leaves # the `[ -r ] && .` line untouched and disables all reporting silently — a mutation the first # version of this checker passed clean. lib = re.search(r"^ETV_HOOK_FIRE_LIB=(.*)$", text, re.M) if not lib: faults.append(f"{name}: no ETV_HOOK_FIRE_LIB assignment") elif "/scripts/hook-fire-log.sh" not in lib.group(1): faults.append(f"{name}: ETV_HOOK_FIRE_LIB does not point at the shared sink: {lib.group(1)}") # A hook that disables its own reporting reads as instrumented and reports nothing. if re.search(r"^\s*(export\s+)?ETV_HOOK_FIRE_DISABLE=1", text, re.M): faults.append(f"{name}: sets ETV_HOOK_FIRE_DISABLE=1, so it never reports") m = _BEGINS.search(text) if not m: faults.append(f"{name}: never calls etv_hook_fire_begin") elif m.group(1) != name: faults.append(f"{name}: calls etv_hook_fire_begin for {m.group(1)!r}, not for itself") else: # MODE. Flipping a Claude hook from `capture` to `stream` stops its stdout being read, so # every decision it makes is recorded as `pass`/`blocked` from the exit status it never # uses — the log stays full and becomes wrong. Nothing else in the suite notices. mode = re.search(rf"^etv_hook_fire_begin {re.escape(name)} \S+ (\S+)", text, re.M) want = expected_mode(name) if not mode: faults.append(f"{name}: etv_hook_fire_begin names no stdout mode") elif mode.group(1) != want: faults.append(f"{name}: begins in {mode.group(1)!r} mode but its wiring implies {want!r}") # Order is the whole point: 8 of the hooks slurp stdin with `input=$(cat)`, and a begin # placed after that read would find the pipe already drained — recording a fire with no # tool name, and replaying nothing into a hook that has already consumed its payload. stdin_read = text.find("$(cat)") if 0 <= stdin_read < m.start(): faults.append(f"{name}: reads stdin before etv_hook_fire_begin") return faults def strip_instrumentation(text: str) -> str: """Reconstruct the pre-#776 script. Used BOTH as the mutation and as the A/B control. The blank line that separates the preamble from the code above it is dropped too. Leaving it made the reconstruction differ from the original by one blank line in all 13 hooks — harmless behaviourally, but the A/B control is only trustworthy insofar as it IS the original, and "differs only in ways I judged harmless" is a claim, not a property. `test_the_stripper_reproduces_the_ORIGINAL_byte_for_byte` turns it into a property. """ out, skipping = [], False for line in text.splitlines(keepends=True): if line.startswith("# ersatztv#776 —"): skipping = True if out and out[-1].strip() == "": out.pop() continue if skipping: if line.startswith( ("#", "ETV_HOOK_FIRE_LIB=", "[ -r ", "type etv_hook_fire_begin", "etv_hook_fire_begin ") ): continue skipping = False out.append(line) return "".join(out) # ------------------------------------------------------------------------------------------------ # ANTI-VACUITY FIRST — a glob that stopped matching would make every assertion below iterate over # nothing and report full coverage. That is the characteristic failure of a completeness check, and # this repo has shipped it (#631, #751), so it is asserted before anything depends on it. # ------------------------------------------------------------------------------------------------ def test_the_population_is_not_empty(): hooks = hook_scripts() assert len(hooks) >= 10, ( f"only found {len(hooks)} hook scripts tracked under {HOOKS_DIR} — the derivation has " "stopped matching, " "so every coverage assertion in this file is vacuous." ) assert SINK.exists(), "the shared sink is missing; the instrumentation cannot work" assert os.access(SINK, os.X_OK), "the sink is not executable, so its report side cannot be run" def test_the_stripper_removes_EXACTLY_the_preamble_and_nothing_else(): """The A/B control must be the hook minus the instrumentation — no more, no less. This compared `strip_instrumentation(hook)` against the hook at the merge base, which asserted two unrelated things at once: that the stripper is exact, AND that no hook was edited in this PR for any other reason. The second is not a property worth pinning — it went red the moment `pretooluse-bom-guard.sh` had a real defect fixed (`xxd`, absent on the CI runner, made it fail open) — and coupling them means a legitimate change reads as a broken control. The property that actually protects the differential is structural: every line the stripper removes is a preamble line, and it adds nothing. Checked without reference to history, so it holds on any branch and on any platform. """ import difflib allowed = ( "# ersatztv#776", "# git hook:", "# Claude hook:", "ETV_HOOK_FIRE_LIB=", '[ -r "$ETV_HOOK_FIRE_LIB"', "type etv_hook_fire_begin", "etv_hook_fire_begin ", ) for hook in hook_scripts(): cur = hook.read_text().splitlines(keepends=True) stripped = strip_instrumentation("".join(cur)).splitlines(keepends=True) for line in difflib.Differ().compare(cur, stripped): if line.startswith("+ "): raise AssertionError(f"{hook.name}: the stripper ADDED a line: {line[2:]!r}") if line.startswith("- "): body = line[2:] assert body.strip() == "" or body.startswith(allowed), ( f"{hook.name}: the stripper removed a NON-preamble line, so the A/B control is " f"not this hook: {body!r}" ) def test_the_stripper_actually_strips(): """The A/B control and the mutation are the same function. If it were a no-op, the differential test would compare each hook against itself and pass on a wrapper that breaks everything.""" for hook in hook_scripts(): text = hook.read_text() stripped = strip_instrumentation(text) assert stripped != text, f"{hook.name}: strip_instrumentation() removed nothing" assert "etv_hook_fire_begin" not in stripped, f"{hook.name}: begin survived the strip" assert not instrumentation_faults(text, hook.stem), f"{hook.name} is not instrumented" # ------------------------------------------------------------------------------------------------ # CLAIM 1 — COVERAGE # ------------------------------------------------------------------------------------------------ def test_every_hook_reports_that_it_fired(): faults = [] for hook in hook_scripts(): faults += instrumentation_faults(hook.read_text(), hook.stem) assert not faults, ( "these hooks do not report their own execution:\n " + "\n ".join(faults) + "\n\nAdd the three-line preamble after the `set -` line and BEFORE any stdin read. A hook " "that does not report is one whose firing we can only infer, which is ersatztv#776." ) def test_a_hook_that_LOSES_its_instrumentation_is_DETECTED(): """THE MUTATION PROOF (contrapositive form, for a checker-guard). Introduce the defect this guard exists to catch — an uninstrumented hook — into an isolated copy of each hook in turn, and the check must report it. Every hook, not a sample: the interesting loss is whichever file someone actually edits. """ for hook in hook_scripts(): mutated = strip_instrumentation(hook.read_text()) faults = instrumentation_faults(mutated, hook.stem) assert faults, ( f"stripping the instrumentation from {hook.name} left the check GREEN. The check is " "not load-bearing — it would not notice a hook silently losing its reporting." ) def test_begin_placed_AFTER_the_stdin_read_is_DETECTED(): """The subtler mutation: present but too late. Ordering is the property that makes it work.""" late = 'set -euo pipefail\ninput=$(cat)\nETV_HOOK_FIRE_LIB="x"\n[ -r "$ETV_HOOK_FIRE_LIB" ] && . "$ETV_HOOK_FIRE_LIB" || true\netv_hook_fire_begin demo "" capture || true\n' # noqa: E501 - kept on one line so the fixture is greppable against the payload the hook receives faults = instrumentation_faults(late, "demo") assert any("reads stdin before" in f for f in faults), ( "a begin call placed after `input=$(cat)` was accepted. It would record a fire with no " f"payload and replay nothing. faults={faults}" ) # ------------------------------------------------------------------------------------------------ # CLAIM 2 — TRANSPARENCY. The wrapper must be invisible to the harness. # ------------------------------------------------------------------------------------------------ # A BROAD, CHEAP matrix. It reaches the deciding branch of only a few hooks; the ones it cannot # reach with a bare payload get a constructed positive case in `positive_cases()` below, and # `test_the_AB_is_not_VACUOUS_for_any_hook` refuses to let any hook rely on this matrix alone. # # Stated because the first version of this file claimed the opposite in three places: this matrix # alone produced empty-vs-empty comparisons for 8 of 13 hooks and a zero exit status for all 13, so # deleting the entire stdout replay left the differential test green for four hooks — two of which # issue `deny`. An A/B over silent allows proves transparency on the one path where there is nothing # to be transparent about. PAYLOADS = { "bash-deny": '{"session_id":"s","hook_event_name":"PreToolUse","tool_name":"Bash","cwd":"%(cwd)s","tool_input":{"command":"ETV_UPDATE_GOLDENS=1 dotnet test"}}', # noqa: E501 - kept on one line so the fixture is greppable against the payload the hook receives "bash-allow": '{"session_id":"s","hook_event_name":"PreToolUse","tool_name":"Bash","cwd":"%(cwd)s","tool_input":{"command":"ls -la"}}', # noqa: E501 - kept on one line so the fixture is greppable against the payload the hook receives "git-commit": '{"session_id":"s","hook_event_name":"PreToolUse","tool_name":"Bash","cwd":"%(cwd)s","tool_input":{"command":"git commit -m x"}}', # noqa: E501 - kept on one line so the fixture is greppable against the payload the hook receives "nav-deny": '{"session_id":"s","hook_event_name":"PreToolUse","tool_name":"mcp__x__navigate","cwd":"%(cwd)s","tool_input":{"url":"http://h/iptv/channels.m3u"}}', "agent-no-model": '{"session_id":"s","hook_event_name":"PreToolUse","tool_name":"Agent","cwd":"%(cwd)s","tool_input":{"description":"d","prompt":"p"}}', # noqa: E501 - kept on one line so the fixture is greppable against the payload the hook receives "agent-model": '{"session_id":"s","hook_event_name":"PreToolUse","tool_name":"Agent","cwd":"%(cwd)s","tool_input":{"description":"d","prompt":"p","model":"sonnet"}}', # noqa: E501 - kept on one line so the fixture is greppable against the payload the hook receives "merge": '{"session_id":"s","hook_event_name":"PreToolUse","tool_name":"mcp__gitea__pull_request_write","cwd":"%(cwd)s","tool_input":{"method":"merge","owner":"timothy","repo":"ersatztv","pull_number":1}}', # noqa: E501 - kept on one line so the fixture is greppable against the payload the hook receives "worktree-add": '{"session_id":"s","hook_event_name":"PostToolUse","tool_name":"Bash","cwd":"%(cwd)s","tool_input":{"command":"git worktree add /tmp/nope-%(nonce)s HEAD"}}', # noqa: E501 - kept on one line so the fixture is greppable against the payload the hook receives "ui-edit": '{"session_id":"s","hook_event_name":"PreToolUse","tool_name":"Edit","cwd":"%(cwd)s","tool_input":{"file_path":"web/src/screens/Channels.tsx"}}', # noqa: E501 - kept on one line so the fixture is greppable against the payload the hook receives "empty": "", "garbage": "not json at all", } @pytest.fixture(scope="module") def sandbox(tmp_path_factory): """A controlled environment so the A/B pair cannot differ for reasons unrelated to the wrapper. `memory_pressure` is stubbed because `pretooluse-agent-ram.sh` reads live free RAM: a real reading that crossed the 10% or 20% threshold between the two runs would make the A and B halves disagree for a reason that has nothing to do with instrumentation, and a differential test that flakes is one that gets rerun until green. """ root = tmp_path_factory.mktemp("hookfire") bindir = root / "bin" bindir.mkdir() stub = bindir / "memory_pressure" stub.write_text("#!/bin/sh\necho 'System-wide memory free percentage: 42%'\n") stub.chmod(0o755) env = dict(os.environ) env["PATH"] = f"{bindir}:{env['PATH']}" env["ETV_HOOK_FIRE_LOG_DIR"] = str(root / "log") env["CLAUDE_PROJECT_DIR"] = str(REPO_ROOT) # No creds: `pretooluse-merge-consent.sh` must take its documented "cannot derive state" path # rather than making live Gitea calls from a unit test. for k in ("ETV_GITEA_TOKEN", "ETV_GITEA_BASICAUTH"): env.pop(k, None) return root, env def _run(script: Path, payload: str, env: dict, cwd: Path, arg: str | None = None): cmd = ["bash", str(script)] + ([arg] if arg else []) p = subprocess.run(cmd, input=payload.encode(), capture_output=True, cwd=str(cwd), env=env, timeout=60) return p.returncode, p.stdout def _git(cwd: Path, *args: str) -> None: subprocess.run( ["git", *args], cwd=str(cwd), check=True, capture_output=True, env={ **os.environ, "GIT_AUTHOR_NAME": "t", "GIT_AUTHOR_EMAIL": "t@e", "GIT_COMMITTER_NAME": "t", "GIT_COMMITTER_EMAIL": "t@e", }, ) @pytest.fixture(scope="module") def positives(sandbox, tmp_path_factory): """CONSTRUCTED cases that drive hooks the bare payload matrix cannot reach to a DECIDING branch. Each entry is (case-name, payload, arg, env, cwd). Building real git repositories is the point: the hooks these cover — the BOM guard, the worktree-ownership guard, the three pre-push guards — all decide by inspecting a repository, so a payload alone can only ever reach their early exit. """ _root, base_env = sandbox w = tmp_path_factory.mktemp("positives") cases: dict[str, list[tuple]] = {} def pay(**kw) -> str: import json as _json return _json.dumps({"session_id": "me", "hook_event_name": "PreToolUse", "tool_name": "Bash", **kw}) # --- worktree-guard: a marker naming ANOTHER session, on a `git commit` ------------------- wt = w / "sibling" wt.mkdir() _git(wt, "init", "-q", ".") (wt / ".claude-worktree-owner").write_text("SOME-OTHER-SESSION\n") cases["pretooluse-worktree-guard"] = [ ( "foreign-marker-deny", pay(cwd=str(wt), tool_input={"command": "git commit -m x"}), None, base_env, wt, ) ] # --- bom-guard: repo path must match `*ersatztv*`, with a staged BOM-carrying .cs ---------- br = w / "ersatztv-scratch" br.mkdir() _git(br, "init", "-q", ".") (br / "Bad.cs").write_bytes(b"\xef\xbb\xbfclass A {}\n") _git(br, "add", "Bad.cs") cases["pretooluse-bom-guard"] = [ ( "staged-bom-deny", pay(cwd=str(br), tool_input={"command": "git commit -m x"}), None, base_env, br, ) ] # --- agent-ram: both thresholds, via a stubbed `memory_pressure` --------------------------- ram_cases = [] for pct, label in ((5, "deny"), (15, "ask")): bindir = w / f"bin{pct}" bindir.mkdir() stub = bindir / "memory_pressure" stub.write_text(f"#!/bin/sh\necho 'System-wide memory free percentage: {pct}%'\n") stub.chmod(0o755) env = dict(base_env) env["PATH"] = f"{bindir}:{os.environ['PATH']}" ram_cases.append( ( f"free-{pct}pct-{label}", pay(tool_name="Agent", cwd=str(w), tool_input={"prompt": "p", "model": "sonnet"}), None, env, w, ) ) cases["pretooluse-agent-ram"] = ram_cases # --- decisions-guard: a validator that prints and blocks, and one that prints and passes --- dg = w / "dg" (dg / "scripts").mkdir(parents=True) _git(dg, "init", "-q", ".") dg_cases = [] for rc, label in ((1, "block"), (0, "pass")): d = w / f"dg{rc}" (d / "scripts").mkdir(parents=True) _git(d, "init", "-q", ".") (d / "scripts" / "decisions_validate.py").write_text( f"import sys\nprint('VALIDATOR SAID SOMETHING')\nsys.exit({rc})\n" ) dg_cases.append((f"validator-{label}", "", None, base_env, d)) cases["decisions-guard"] = dg_cases # --- pre-push guards: a real remote, so `git fetch origin main` resolves ------------------- bare = w / "remote.git" _git(w, "init", "-q", "--bare", "--initial-branch=main", str(bare)) def clone(name: str) -> Path: p = w / name subprocess.run(["git", "clone", "-q", str(bare), str(p)], check=True, capture_output=True) _git(p, "config", "user.email", "t@e") _git(p, "config", "user.name", "t") return p seed = clone("seed") (seed / "file.txt").write_text("one\n") _git(seed, "add", "-A") _git(seed, "commit", "-qm", "seed") _git(seed, "push", "-q", "origin", "main") # rebase-check: local HEAD is an ANCESTOR of a since-advanced origin/main -> blocks. behind = clone("behind") _git(behind, "checkout", "-qb", "feat") ahead = clone("ahead") (ahead / "file.txt").write_text("two\n") _git(ahead, "add", "-A") _git(ahead, "commit", "-qm", "advance main") _git(ahead, "push", "-q", "origin", "main") head = subprocess.run(["git", "rev-parse", "HEAD"], cwd=str(behind), capture_output=True, text=True).stdout.strip() cases["prepush-rebase-check"] = [ ( "behind-origin-main-blocks", f"refs/heads/feat {head} refs/heads/feat {'0' * 40}\n", None, base_env, behind, ) ] # clean-worktree-check: a file both MODIFIED in the tree and present in the pushed set. dirty = clone("dirty") (dirty / "file.txt").write_text("committed change\n") _git(dirty, "add", "-A") _git(dirty, "commit", "-qm", "change file") (dirty / "file.txt").write_text("uncommitted change\n") dhead = subprocess.run(["git", "rev-parse", "HEAD"], cwd=str(dirty), capture_output=True, text=True).stdout.strip() cases["prepush-clean-worktree-check"] = [ ( "dirty-file-in-pushed-set-blocks", f"refs/heads/main {dhead} refs/heads/main {'0' * 40}\n", None, base_env, dirty, ) ] # --- prepush-donewhen: a push to main closing an issue with an unticked Done-when box ------- # # This one needs a Gitea, so it gets a stub HTTP server rather than an exemption. Without it the # hook's only reachable path in a test is its no-creds fail-open, and the guard that blocks # pushes to `main` would be the one hook whose transparency nothing checks — precisely the # "covered everywhere except where it matters" shape this repo keeps paying for. import http.server import json as _json import threading body = _json.dumps({"body": "## Done-when\n\n- [ ] not finished\n- [x] finished\n"}).encode() class _Stub(http.server.BaseHTTPRequestHandler): def do_GET(self): self.send_response(200) self.send_header("Content-Type", "application/json") self.send_header("Content-Length", str(len(body))) self.end_headers() self.wfile.write(body) def log_message(self, *a): # keep pytest output clean pass srv = http.server.HTTPServer(("127.0.0.1", 0), _Stub) threading.Thread(target=srv.serve_forever, daemon=True).start() dw = clone("donewhen") (dw / "src.cs").write_text("class A {}\n") # NOT docs-only, so the exemption does not apply _git(dw, "add", "-A") _git(dw, "commit", "-qm", "feat: thing\n\nfixes #999") dw_head = subprocess.run(["git", "rev-parse", "HEAD"], cwd=str(dw), capture_output=True, text=True).stdout.strip() dw_prev = subprocess.run(["git", "rev-parse", "HEAD~1"], cwd=str(dw), capture_output=True, text=True).stdout.strip() dw_env = dict(base_env) dw_env["ETV_GITEA_URL"] = f"http://127.0.0.1:{srv.server_address[1]}" dw_env["ETV_GITEA_BASICAUTH"] = "stub:stub" cases["prepush-donewhen"] = [ ( "unticked-donewhen-blocks-push-to-main", f"refs/heads/main {dw_head} refs/heads/main {dw_prev}\n", None, dw_env, dw, ) ] return cases def _ab_cases(hook: Path, sandbox, positives) -> list[tuple]: """(label, payload, arg, env, cwd) for the generic matrix PLUS this hook's constructed cases.""" root, env = sandbox cases: list[tuple] = [] args = [None, "start", "finish"] if hook.stem == "design-sync-reminder" else [None] for name, template in PAYLOADS.items(): for arg in args: cases.append( ( f"matrix:{name}:{arg}", template % {"cwd": str(root), "nonce": name}, arg, env, root, ) ) for label, payload, arg, penv, cwd in positives.get(hook.stem, []): cases.append((f"positive:{label}", payload, arg, penv, cwd)) return cases def _ab_run(hook: Path, sandbox, case, tag_root: str): """Run the control and the instrumented hook for one case. Returns both (rc, stdout, stderr).""" root = sandbox[0] control = root / f"control-{hook.name}" if not control.exists(): control.write_text(strip_instrumentation(hook.read_text())) control.chmod(0o755) _label, payload, arg, env, cwd = case # A SEPARATE TMPDIR PER SIDE, not per pair. `design-sync-reminder.sh` throttles itself with a # marker file under $TMPDIR, so a shared one let the control fire and then silently suppress the # instrumented run — the A/B reported a swallowed decision that was really the hook's own # one-shot behaviour working correctly. Sharing state between the two halves of a differential # test makes it measure the state, not the difference. def side(t: str) -> dict: e = dict(env) e["TMPDIR"] = str(root / f"tmp-{tag_root}-{t}") Path(e["TMPDIR"]).mkdir(parents=True, exist_ok=True) return e def one(script: Path, t: str): cmd = ["bash", str(script)] + ([arg] if arg else []) p = subprocess.run(cmd, input=payload.encode(), capture_output=True, cwd=str(cwd), env=side(t), timeout=90) return p.returncode, p.stdout, p.stderr return one(control, "control"), one(hook, "instrumented") def _normalise_stderr(err: bytes) -> bytes: """Erase the two things that MUST differ between the control and the instrumented hook. bash prefixes its own diagnostics with `$0` and a line number. The control is a copy at a different path, and the instrumentation adds four lines to the top, so *every* bash-emitted message differs in exactly those two fields for reasons that have nothing to do with transparency. Only the script path and `line N:` are normalised — narrowly, so that a genuine difference in the message body still fails. """ # The two fields are normalised TOGETHER, as one anchored prefix, not independently. A global # `line \d+:` substitution also rewrites application text that happens to contain that phrase, # and cold review demonstrated it collapsing two genuinely different diagnostics ("highest # private fd line 0" vs "line 4") into one — hiding exactly the kind of fd-state difference this # comparison exists to catch. return re.sub(rb"(?m)^[^\s:]*/[^\s:]*\.sh: line \d+:", b"