timothyandClaude Fable 5.1
a956cc6b8d
The claim the shipped GREEN entry binds — what removing the `+++ /dev/null` arm does — was written twice: in `check-doc-narrative.py`, where the entry binds it, and again in `test_a_DELETED_doc_is_not_reported_as_added_content`'s docstring, where nothing does. That is the copy-of-an-outcome shape this rule forbids, in a site class the rule names, found while reading the proof for the residual below. The docstring now points at the manifest entry and keeps its rationale (a deletion yields no `+` lines either way), which is the half the carve-out protects. The residual paragraph is also made exact rather than general. The reach mutation proves the proof depends on the clause through the `b/` stripping every scanned header goes through, not through the `/dev/null` arm itself, so in general such a green cannot separate "no test feeds that input" from "the arm changes nothing". For this entry it can, by reading the proof: the deleted-doc test deletes a tracked file, and a deletion diff under the flags `run_diff` pins carries a `+++ /dev/null` header — probed rather than reasoned. Refs #881 Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_015QqCpYFsKgnAnx6jVwrKiV
645 lines
34 KiB
Python
645 lines
34 KiB
Python
"""Proofs for `scripts/check-doc-narrative.py` (ersatztv#784).
|
|
|
|
Every case below is a defect DEMONSTRATED in the first, shell implementation. They are
|
|
here because the never-fails invariant and the reported line numbers are both asserted in prose in
|
|
four places (the script header, the workflow comment, `docs/guard-inventory.md` and
|
|
`docs/remote-state-inventory.md`), and an invariant asserted only in prose is the shape this repo
|
|
keeps getting wrong.
|
|
|
|
The line-number cases all compare against the TRUTH computed from the file on disk, never against a
|
|
number written into the test — a hand-written expectation is a second copy of the parser.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import importlib.util
|
|
import re
|
|
import subprocess
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
REPO = Path(__file__).resolve().parents[2]
|
|
SCRIPT = REPO / "scripts" / "check-doc-narrative.py"
|
|
|
|
NARRATIVE = "I initially thought otherwise"
|
|
HIT = re.compile(r"^::warning file=(?P<path>[^:]+)::(?P=path):(?P<line>\d+) ", re.MULTILINE)
|
|
|
|
|
|
def run(cwd: Path, *args: str) -> subprocess.CompletedProcess:
|
|
return subprocess.run([sys.executable, str(SCRIPT), *args], cwd=cwd, capture_output=True, text=True)
|
|
|
|
|
|
def hits(out: str) -> set[tuple[str, int]]:
|
|
return {(m.group("path"), int(m.group("line"))) for m in HIT.finditer(out)}
|
|
|
|
|
|
def truth(root: Path, rel: str) -> set[tuple[str, int]]:
|
|
"""Where the narrative marker ACTUALLY is, read back off disk."""
|
|
text = (root / rel).read_text(encoding="utf-8")
|
|
return {
|
|
(rel, i)
|
|
for i, line in enumerate(text.splitlines(), start=1)
|
|
if "initially thought" in line or "an earlier draft" in line.lower()
|
|
}
|
|
|
|
|
|
def git(repo: Path, *args: str) -> None:
|
|
subprocess.run(["git", *args], cwd=repo, check=True, capture_output=True)
|
|
|
|
|
|
@pytest.fixture()
|
|
def repo(tmp_path: Path) -> Path:
|
|
r = tmp_path / "r"
|
|
(r / "docs").mkdir(parents=True)
|
|
git(r.parent, "init", "-q", "r")
|
|
git(r, "config", "user.email", "t@example.com")
|
|
git(r, "config", "user.name", "t")
|
|
(r / "docs" / "seed.md").write_text("seed\n", encoding="utf-8")
|
|
git(r, "add", "-A")
|
|
git(r, "commit", "-qm", "base")
|
|
return r
|
|
|
|
|
|
def commit(r: Path, msg: str = "c") -> None:
|
|
git(r, "add", "-A")
|
|
git(r, "commit", "-qm", msg)
|
|
|
|
|
|
# --- the never-fails invariant -------------------------------------------------------------------
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("args", "expected"),
|
|
[
|
|
((), "scanned "),
|
|
(("--all",), "scanned "),
|
|
(("--diff",), "--diff needs a base ref"), # no base ref at all
|
|
(("--diff", ""), "--diff needs a base ref"), # `origin/` with base_ref unset
|
|
(("--diff", "no-such-ref-xyz"), "SCANNED NOTHING"), # an unresolvable ref
|
|
(("--diff", "origin/"), "SCANNED NOTHING"),
|
|
(("--nonsense",), "unknown mode"),
|
|
],
|
|
)
|
|
def test_every_argument_shape_exits_zero_HAVING_HANDLED_IT(repo: Path, args, expected: str) -> None:
|
|
"""The header, the workflow comment and both inventories state exit 0 absolutely.
|
|
|
|
The expected message is asserted alongside the exit code on purpose: the script ends with a bare
|
|
`except` that returns 0, so an exit-code-only assertion is satisfied by an unhandled crash and
|
|
would pass against a script that handles none of these shapes.
|
|
"""
|
|
p = run(repo, *args)
|
|
assert p.returncode == 0, f"args={args} exited {p.returncode}: {p.stderr}"
|
|
assert expected in p.stdout, f"args={args} exited 0 but did not HANDLE it: {p.stdout!r}"
|
|
assert "internal error" not in p.stdout, f"args={args} reached the last-resort handler: {p.stdout!r}"
|
|
|
|
|
|
def test_an_unknown_mode_prints_usage_and_scans_nothing(repo: Path) -> None:
|
|
"""Stated as the observable behaviour it actually pins. It is NOT a proof that the else-branch
|
|
cannot reach `run_all`: the branch returns before the warnings are printed, so a mutant calling
|
|
`run_all` there is silent and no black-box test can see it."""
|
|
(repo / "docs" / "u.md").write_text(f"x\n{NARRATIVE}\n", encoding="utf-8")
|
|
commit(repo)
|
|
p = run(repo, "--nonsense")
|
|
assert hits(p.stdout) == set()
|
|
assert "scanned" not in p.stdout
|
|
|
|
|
|
def test_a_DELETED_doc_is_not_reported_as_added_content(repo: Path) -> None:
|
|
"""A deletion contributes no added lines.
|
|
|
|
Stated exactly: this pins the BEHAVIOUR, and it is NOT a proof of the `+++ /dev/null` arm, which
|
|
is defensive — a deletion yields no `+` lines either way. What removing that arm does to this
|
|
file is deliberately NOT restated here: it is a declared claim in
|
|
`scripts/tests/mutation_manifest.py`, re-executed on every run (ersatztv#881), and a second copy
|
|
of an outcome is free to drift from the entry that executes it. A docstring claiming a proof it
|
|
does not have is worse than no docstring: it is the thing a later reader trusts instead of
|
|
re-checking."""
|
|
(repo / "docs" / "del.md").write_text(f"x\n{NARRATIVE}\n", encoding="utf-8")
|
|
commit(repo)
|
|
(repo / "docs" / "del.md").unlink()
|
|
commit(repo)
|
|
assert hits(run(repo, "--diff", "HEAD~1").stdout) == set()
|
|
|
|
|
|
# Every knob DEMONSTRATED to break the parse — not every knob that reshapes diff output, which is a
|
|
# universal nobody can check and which is the enumeration that failed three rounds running.
|
|
# (`diff.mnemonicPrefix` is a live example of one that reshapes the header and has no row: it emits
|
|
# `+++ w/f.md`. It is inert HERE for a reason worth stating exactly, because the obvious explanation
|
|
# is the wrong one — not because `--dst-prefix=b/` beats it, though it does, but because git only
|
|
# uses mnemonic prefixes when a diff side is the worktree or the index, and `run_diff` issues the
|
|
# three-dot `base...HEAD` form, where git emits plain `a/`…`b/` either way. Measured, not reasoned.)
|
|
# Three of these were each demonstrated turning
|
|
# a real hit into `scanned 0 file(s)`, one at a time, in three separate rounds — which is why the fix
|
|
# stopped pinning variants and removed the surface. The table is here so the next knob someone finds
|
|
# gets a row instead of a round.
|
|
FORMAT_KNOBS = [
|
|
("core.quotePath", "true"), # quotes non-ASCII paths out of the population
|
|
("diff.dstPrefix", "dst/"), # rewrites the header the path is read from
|
|
("diff.srcPrefix", "src/"),
|
|
("diff.noprefix", "true"),
|
|
("color.diff", "always"), # injects ANSI escapes into every line
|
|
("color.ui", "always"),
|
|
("diff.renames", "false"), # turns a `git mv` into a whole-file add
|
|
("diff.context", "9"), # a configured context must not beat the -U0 on the CLI
|
|
("diff.external", "/bin/echo"), # replaces the output wholesale
|
|
]
|
|
|
|
|
|
@pytest.mark.parametrize(("key", "value"), FORMAT_KNOBS)
|
|
def test_no_git_FORMAT_CONFIG_can_produce_a_false_clean(repo: Path, key: str, value: str) -> None:
|
|
"""A false clean is the worst outcome available to an advisory check: it is indistinguishable
|
|
from a real one and nobody looks twice."""
|
|
(repo / "docs" / "pfx.md").write_text("x\n", encoding="utf-8")
|
|
commit(repo)
|
|
(repo / "docs" / "pfx.md").write_text(f"x\n{NARRATIVE}\n", encoding="utf-8")
|
|
commit(repo)
|
|
git(repo, "config", key, value)
|
|
assert hits(run(repo, "--diff", "HEAD~1").stdout) == truth(repo, "docs/pfx.md"), f"{key}={value}"
|
|
|
|
|
|
# Only PYTHONIOENCODING is listed. `LC_ALL=C` was here too and was VACUOUS — PEP 540 UTF-8 mode
|
|
# means the pre-fix script passed it as well, so it read as a second witness where there was one.
|
|
@pytest.mark.parametrize("env_name,env_value", [("PYTHONIOENCODING", "ascii")])
|
|
def test_a_NON_UTF8_stdio_does_not_break_the_never_fails_invariant(repo: Path, env_name, env_value) -> None:
|
|
"""Both the summary line and the last-resort handler carry U+2014, so under ascii stdio the code
|
|
guaranteeing exit 0 was itself what raised."""
|
|
import os
|
|
|
|
env = dict(os.environ, **{env_name: env_value})
|
|
env.pop("PYTHONUTF8", None)
|
|
p = subprocess.run([sys.executable, str(SCRIPT), "--all"], cwd=repo, capture_output=True, text=True, env=env)
|
|
assert p.returncode == 0, f"{env_name}={env_value} exited {p.returncode}: {p.stderr}"
|
|
|
|
|
|
def test_a_tracked_but_DELETED_doc_does_not_kill_the_run(repo: Path) -> None:
|
|
"""`git ls-files` lists index entries; deleting a doc before committing is ordinary."""
|
|
(repo / "docs" / "gone.md").write_text("x\n", encoding="utf-8")
|
|
commit(repo)
|
|
(repo / "docs" / "gone.md").unlink()
|
|
p = run(repo, "--all")
|
|
assert p.returncode == 0
|
|
assert "skipped docs/gone.md" in p.stdout
|
|
|
|
|
|
def test_an_unresolvable_base_REPORTS_that_it_scanned_nothing(repo: Path) -> None:
|
|
"""A silent zero-file scan is indistinguishable from a clean one — the whole point of #751."""
|
|
p = run(repo, "--diff", "no-such-ref-xyz")
|
|
assert p.returncode == 0
|
|
assert "SCANNED NOTHING" in p.stdout
|
|
# The guard names the line this branch must NOT also print. It previously named "nothing to
|
|
# flag", a string that occurs nowhere in this script (it belongs to the sibling parity step), so
|
|
# it could never fail — a negative assertion over a literal that does not exist is not a guard.
|
|
assert "scanned 0 file(s)" not in p.stdout, "printed a clean-looking summary after scanning nothing"
|
|
|
|
|
|
def test_a_genuine_clean_scan_REPORTS_its_population(repo: Path) -> None:
|
|
p = run(repo, "--all")
|
|
assert "scanned 1 file(s); 0 advisory warning(s)" in p.stdout
|
|
|
|
|
|
# --- line numbers, against truth read off disk ---------------------------------------------------
|
|
|
|
|
|
def test_a_file_with_NO_trailing_newline_does_not_shift_later_lines(repo: Path) -> None:
|
|
"""`\\ No newline at end of file` is a marker, not a line of the new file."""
|
|
(repo / "docs" / "n.md").write_text("a\nb\nc", encoding="utf-8") # no trailing newline
|
|
commit(repo)
|
|
(repo / "docs" / "n.md").write_text(f"a\nb\nZ\n{NARRATIVE}\n", encoding="utf-8")
|
|
commit(repo)
|
|
assert hits(run(repo, "--diff", "HEAD~1").stdout) == truth(repo, "docs/n.md")
|
|
|
|
|
|
def test_an_added_line_whose_TEXT_starts_with_plus_plus_is_content_not_a_header(repo: Path) -> None:
|
|
"""Docs here embed diff output in fenced blocks, so `++ ` at column 0 is real."""
|
|
(repo / "docs" / "p.md").write_text("p\n", encoding="utf-8")
|
|
commit(repo)
|
|
(repo / "docs" / "p.md").write_text(f"p\n++ a fenced diff line\n{NARRATIVE}\n", encoding="utf-8")
|
|
commit(repo)
|
|
assert hits(run(repo, "--diff", "HEAD~1").stdout) == truth(repo, "docs/p.md")
|
|
|
|
|
|
def test_a_multi_hunk_file_numbers_every_hunk_from_its_own_header(repo: Path) -> None:
|
|
body = [f"l{i}" for i in range(1, 31)]
|
|
(repo / "docs" / "m.md").write_text("\n".join(body) + "\n", encoding="utf-8")
|
|
commit(repo)
|
|
body[4] = NARRATIVE # replace, early hunk
|
|
body.insert(15, NARRATIVE) # pure insert, middle hunk
|
|
body[-1] = NARRATIVE # replace, last hunk
|
|
(repo / "docs" / "m.md").write_text("\n".join(body) + "\n", encoding="utf-8")
|
|
commit(repo)
|
|
assert hits(run(repo, "--diff", "HEAD~1").stdout) == truth(repo, "docs/m.md")
|
|
|
|
|
|
def test_a_NON_ASCII_path_is_scanned_rather_than_silently_skipped(repo: Path) -> None:
|
|
"""`core.quotePath` quotes the path, and a quoted path matches no scope rule — it vanishes."""
|
|
(repo / "docs" / "café.md").write_text(f"x\n{NARRATIVE}\n", encoding="utf-8")
|
|
commit(repo)
|
|
assert hits(run(repo, "--all").stdout) == truth(repo, "docs/café.md")
|
|
assert hits(run(repo, "--diff", "HEAD~1").stdout) == truth(repo, "docs/café.md")
|
|
|
|
|
|
def test_a_final_UNTERMINATED_line_is_still_scanned_in_all_mode(repo: Path) -> None:
|
|
(repo / "docs" / "t.md").write_text(f"ok\n{NARRATIVE}", encoding="utf-8") # no trailing newline
|
|
commit(repo)
|
|
assert hits(run(repo, "--all").stdout) == truth(repo, "docs/t.md")
|
|
|
|
|
|
def test_a_RENAME_does_not_re_flag_the_whole_pre_existing_file(repo: Path) -> None:
|
|
"""Without rename detection a `git mv` reports every line of the file as newly added."""
|
|
(repo / "docs" / "r1.md").write_text("a\n" + f"{NARRATIVE}\n" + "b\n", encoding="utf-8")
|
|
commit(repo)
|
|
git(repo, "mv", "docs/r1.md", "docs/r2.md")
|
|
commit(repo)
|
|
assert hits(run(repo, "--diff", "HEAD~1").stdout) == set()
|
|
|
|
# `diff.renames` defaults to true, so the assertion above passes with or without the explicit
|
|
# `--find-renames`. Turning the config off is what makes that flag load-bearing and this test a
|
|
# real proof of it rather than a restatement of a git default.
|
|
git(repo, "config", "diff.renames", "false")
|
|
assert hits(run(repo, "--diff", "HEAD~1").stdout) == set(), "the --find-renames pin is not doing its job"
|
|
|
|
|
|
# --- the population ------------------------------------------------------------------------------
|
|
|
|
|
|
# Hand-written, one entry per exempt prefix. NOT derived from the script — a population read out of
|
|
# the code under test proves only that the code agrees with itself. The completeness guard below is
|
|
# what stops a new prefix from shipping without a case here.
|
|
EXEMPT_SAMPLES = {
|
|
"docs/decisions/": ("docs/decisions/records/x/y.md", "docs/decisions/archive/x/y.md"),
|
|
# test DATA under a process prefix (#876): a decision-record copy and a shell fixture
|
|
"scripts/tests/fixtures/": ("scripts/tests/fixtures/premigration/decisions.md", "scripts/tests/fixtures/x/y.sh"),
|
|
}
|
|
|
|
|
|
@pytest.mark.parametrize("mode", ["--all", "--diff"])
|
|
def test_every_exempt_genre_is_exempt_in_BOTH_modes(repo: Path, mode: str) -> None:
|
|
"""`docs/decisions/**` is exempt WHOLESALE (#784). Both modes, because an exemption that held
|
|
only for `--all` would still flag the file on the PR that adds it.
|
|
|
|
The CONTROLS are what make this a measurement. `hits(out) == set()` alone also passes when the
|
|
scan examined nothing at all — a scanned-zero run reads identical to a working exemption — so a
|
|
non-exempt file carrying the same marker must come back HIT in the same invocation.
|
|
|
|
Each exempt sample is also paired with a MINIMAL TWIN — the same path with the exempt directory
|
|
renamed — which raises the cost of a structural bypass but does NOT close it. Stated as a limit
|
|
rather than a guarantee, because three successive attempts to close it were each defeated by the
|
|
next: a lone depth-1 control fell to a depth-1 population rule; a depth-3 control fell to a
|
|
`count("/") <= 3` cap; the twins themselves fell to a directory-NAME rule keyed on the renamed
|
|
SECOND segment (`path.count("/") > 1 and "-" not in path.split("/", 2)[1]`), which skipped both
|
|
samples and kept both twins. It reddens here now, but only because the `docs/superpowers/`
|
|
fixture below happens to have a hyphen-free second segment — luck, not design, and not a
|
|
property this test establishes.
|
|
|
|
So this test proves the scan EXAMINED something; it does NOT prove a sample's absence is
|
|
attributable to `EXEMPT_PREFIXES` rather than to its shape. Do not add a fourth finite control —
|
|
each of the three was locally correct and the sequence did not converge, the same shape as the
|
|
withdrawn `test_review_verdict_vocabulary_parity.py` (six rounds, then deleted) that
|
|
`docs.no-session-narrative` cites as the case for stopping. What DOES close it is a closed-form
|
|
proof rather than a fixture:
|
|
`test_the_population_agrees_with_an_INDEPENDENT_RESTATEMENT_over_the_REAL_corpus` below.
|
|
"""
|
|
for rels in EXEMPT_SAMPLES.values():
|
|
for rel in rels:
|
|
f = repo / rel
|
|
f.parent.mkdir(parents=True, exist_ok=True)
|
|
f.write_text(f"x\n{NARRATIVE}\n", encoding="utf-8")
|
|
# The twin of `docs/decisions/records/x/y.md` is `docs/decisions-not/records/x/y.md`: identical
|
|
# in every structural respect, and NOT under the exempt prefix.
|
|
twins = {rel.replace(prefix, prefix[:-1] + "-not/", 1) for prefix, rels in EXEMPT_SAMPLES.items() for rel in rels}
|
|
# `docs/superpowers/` is deliberately NOT exempt (#812), and re-exempting it is the specific
|
|
# regression this change argues against. It is named here rather than left to the closed-form
|
|
# oracle, because that oracle proves a property of `is_scanned_path`, not that the SCAN consults
|
|
# it: a `continue` added in `run_all`/`run_diff` reinstates the exemption with the function
|
|
# untouched. Only a path that reaches the OUTPUT witnesses the wiring.
|
|
twins = twins | {"docs/superpowers/plans/2026-01-01-x.md"}
|
|
for rel in twins | {"docs/control.md"}:
|
|
control = repo / rel
|
|
control.parent.mkdir(parents=True, exist_ok=True)
|
|
control.write_text(f"x\n{NARRATIVE}\n", encoding="utf-8")
|
|
commit(repo)
|
|
out = run(repo, mode, "HEAD~1").stdout if mode == "--diff" else run(repo, mode).stdout
|
|
assert hits(out) == {(rel, 2) for rel in twins | {"docs/control.md"}}
|
|
|
|
|
|
def test_the_population_agrees_with_an_INDEPENDENT_RESTATEMENT_over_the_REAL_corpus() -> None:
|
|
"""CLOSED-FORM. This is what three successive finite controls could not buy, and it is why the
|
|
retraction in the exemption test's docstring stops at "not another fixture" rather than at
|
|
"unprovable".
|
|
|
|
The rule as the record states it — `docs/**/*.md` minus `docs/decisions/**`, plus root-level
|
|
`*.md`, plus every file under the four process directories minus the detector and its test
|
|
(#876) — is restated below over path SEGMENTS rather than string prefixes, so the two are not
|
|
one expression copy-pasted, then compared against `is_scanned_path` on every tracked path in
|
|
this repo. A finite fixture can only witness the shapes someone thought to write down;
|
|
this witnesses every real path, so any population change that alters a verdict on one of them
|
|
reddens — the depth-1 rule and the directory-NAME rule among them.
|
|
|
|
It is NOT sufficient on its own, and the arm below says why with the measurement: a
|
|
`count("/") <= 3` cap changes no real path's verdict (re-measured 2026-09-03 after #876 grew the
|
|
population: 179 files, none deeper than three slashes — and only because `scripts/tests/fixtures/`
|
|
is exempt, whose record copies sit at four to six), so within this test only the synthetic arm
|
|
catches it. Do not trim that arm as belt-and-braces; it is the half that sees a
|
|
rule the corpus has no instance of yet. (The exemption test above happens to redden on that cap
|
|
too — its two minimal twins sit at four slashes — but that is incidental, not the arm carrying the claim.)
|
|
"""
|
|
|
|
def restated(path: str) -> bool:
|
|
parts = path.split("/")
|
|
if parts in (["scripts", "check-doc-narrative.py"], ["scripts", "tests", "test_check_doc_narrative.py"]):
|
|
return False # the detector and its test: exempt by NAME, their fixtures ARE the phrasings
|
|
if parts[:3] == ["scripts", "tests", "fixtures"] and len(parts) > 3:
|
|
return False # test DATA, including decision-record copies (#876)
|
|
if len(parts) > 1 and parts[0] in {".claude", ".gitea", ".husky", "scripts"}:
|
|
return True # the process corpus, any extension (#876)
|
|
if not parts[-1].endswith(".md"):
|
|
return False
|
|
if len(parts) == 1:
|
|
return True # root-level *.md
|
|
if parts[0] != "docs":
|
|
return False # nested markdown outside docs/ and the process corpus is out of scope
|
|
return parts[1] != "decisions" # docs/** minus docs/decisions/**
|
|
|
|
spec = importlib.util.spec_from_file_location("_cdn_pop", SCRIPT)
|
|
assert spec is not None and spec.loader is not None
|
|
mod = importlib.util.module_from_spec(spec)
|
|
spec.loader.exec_module(mod)
|
|
|
|
rc = subprocess.run(["git", "ls-files", "-s", "-z"], cwd=REPO, capture_output=True, text=True, check=True)
|
|
entries = [e.partition("\t") for e in rc.stdout.split("\0") if e]
|
|
paths = sorted({path for _meta, _, path in entries})
|
|
# "regular" as `run_all` defines it: EVERY stage of the path a regular blob, not any one of them.
|
|
stage_modes: dict[str, set[str]] = {}
|
|
for meta, _, path in entries:
|
|
stage_modes.setdefault(path, set()).add(meta.split(" ", 1)[0])
|
|
regular = {path for path, ms in stage_modes.items() if all(m.startswith("100") for m in ms)}
|
|
assert len(paths) > 1000, f"corpus looks wrong: {len(paths)} paths"
|
|
|
|
disagree = [p for p in paths if mod.is_scanned_path(p) != restated(p)]
|
|
assert not disagree, f"population disagrees with the restated rule on {len(disagree)}: {disagree[:8]}"
|
|
|
|
# Anti-vacuity: an oracle that says False everywhere would agree with a scan-nothing mutant.
|
|
# `scanned` is what `--all` opens: the predicate over REGULAR blobs only, because a symlink or
|
|
# gitlink the predicate admits is excluded by mode, and a claim about the scanned corpus's shape
|
|
# must be measured over the corpus that is scanned.
|
|
scanned = [p for p in paths if restated(p) and p in regular]
|
|
exempt = [p for p in paths if p.startswith("docs/decisions/")]
|
|
process = [p for p in scanned if not p.endswith(".md")]
|
|
assert len(scanned) > 50 and len(exempt) > 50, f"{len(scanned)} scanned / {len(exempt)} exempt"
|
|
assert len(process) > 50, f"{len(process)} non-markdown process files — the #876 arm saw nothing"
|
|
# The record states a MEASUREMENT about this corpus — no scanned path deeper than three slashes,
|
|
# so a `count("/") <= 3` cap is a no-op against it — and nothing else couples the claim to the
|
|
# corpus. A red here is the claim going stale, not a defect in the path that broke it. Re-measure,
|
|
# then re-state every site that carries the claim: the record's `mechanics:`
|
|
# (`docs.no-session-narrative`, `mechanics:` AND the #876 paragraph of its body), this function's
|
|
# docstring, the comment on the cross product below, and the fixtures paragraph of the exemption
|
|
# comment in `check-doc-narrative.py`; then move this bound.
|
|
assert max(p.count("/") for p in scanned) <= 3, "the record's depth measurement is stale — re-state it"
|
|
assert not any(mod.is_scanned_path(p) for p in mod.EXEMPT_FILES)
|
|
assert all(p in paths for p in mod.EXEMPT_FILES), "an EXEMPT_FILES entry names nothing tracked"
|
|
|
|
# The real corpus alone cannot see a rule that differs only on a shape it does not currently
|
|
# contain — no scanned path carries more than three slashes (re-measured 2026-09-03, #876), so a
|
|
# `count("/") <= 3` cap is a no-op against it and would ship latent. So the same oracle is also
|
|
# compared over a cross product of FOUR dimensions a population rule has been observed to key on here — depth, first
|
|
# segment, second segment, extension — widened with basename, case and dotted directories after
|
|
# a mutant keyed on each was found. This ENUMERATES; it is not a universal over the
|
|
# space of rules, and a dimension on neither list has simply not been measured.
|
|
heads = ["docs", "Docs", "web", "scripts", "Scripts", ".github", ".gitea", ".claude", ".husky", "scripts-not", ""]
|
|
seconds = [
|
|
"decisions",
|
|
"superpowers",
|
|
"other",
|
|
"decisions-not",
|
|
"Decisions",
|
|
".hidden",
|
|
"tests",
|
|
"hooks",
|
|
"tests/fixtures",
|
|
"tests/fixtures-not",
|
|
"",
|
|
]
|
|
leaves = ["f", "index", "README", "check-doc-narrative", "test_check_doc_narrative"]
|
|
# `.py.bak` is the strict extension of an exempt NAME: it separates `path in EXEMPT_FILES` from
|
|
# a `startswith` over the same tuple, which the real corpus cannot (nothing tracked extends it).
|
|
exts = [".md", ".markdown", ".MD", ".txt", ".py", ".py.bak", ".sh", ".yml", ""]
|
|
synthetic = set()
|
|
for ext in exts:
|
|
for leaf in leaves:
|
|
for head in heads:
|
|
for second in seconds:
|
|
for depth in range(0, 3):
|
|
segs = [p for p in (head, second) if p] + ["x"] * depth
|
|
synthetic.add("/".join(segs + [leaf + ext]))
|
|
bad = [p for p in sorted(synthetic) if mod.is_scanned_path(p) != restated(p)]
|
|
assert not bad, f"population disagrees with the restated rule on {len(bad)} shapes: {bad[:8]}"
|
|
|
|
|
|
def test_every_exempt_prefix_has_a_case_above() -> None:
|
|
"""Completeness: adding a prefix to `EXEMPT_PREFIXES` without a sample above reddens here, so the
|
|
exemption test cannot silently stop covering the population it claims to cover."""
|
|
spec = importlib.util.spec_from_file_location("_cdn", SCRIPT)
|
|
assert spec is not None and spec.loader is not None
|
|
mod = importlib.util.module_from_spec(spec)
|
|
spec.loader.exec_module(mod)
|
|
assert set(EXEMPT_SAMPLES) == set(mod.EXEMPT_PREFIXES)
|
|
for prefix, rels in EXEMPT_SAMPLES.items():
|
|
assert all(r.startswith(prefix) for r in rels), prefix
|
|
|
|
|
|
def test_markdown_outside_the_stated_population_is_not_scanned(repo: Path) -> None:
|
|
"""The population is `docs/**/*.md` minus `EXEMPT_PREFIXES`, plus root-level `*.md`, plus the
|
|
PROCESS corpus (#876) regardless of extension, minus `EXEMPT_FILES`. Nested markdown under
|
|
`web/` is out of scope, and the record's `mechanics:` says so — out of the DETECTOR, still bound
|
|
by the RULE. A skill under `.claude/`, a shell hook and a Python test are IN, and the detector's
|
|
own test file — which carries these phrasings as fixtures — is out BY NAME, so its twin one
|
|
directory over is still scanned."""
|
|
for rel in (
|
|
"web/docs/x.md",
|
|
".claude/skills/s/SKILL.md",
|
|
".claude/hooks/h.sh",
|
|
"scripts/tests/test_x.py",
|
|
"scripts/tests/test_check_doc_narrative.py",
|
|
"scripts/tests/x/test_check_doc_narrative.py",
|
|
"README.md",
|
|
):
|
|
(repo / rel).parent.mkdir(parents=True, exist_ok=True)
|
|
(repo / rel).write_text(f"# {NARRATIVE}\n", encoding="utf-8")
|
|
commit(repo)
|
|
assert hits(run(repo, "--all").stdout) == {
|
|
("README.md", 1),
|
|
(".claude/skills/s/SKILL.md", 1),
|
|
(".claude/hooks/h.sh", 1),
|
|
("scripts/tests/test_x.py", 1),
|
|
("scripts/tests/x/test_check_doc_narrative.py", 1),
|
|
}
|
|
|
|
|
|
def test_a_tracked_SYMLINK_is_not_scanned_and_not_counted(repo: Path) -> None:
|
|
"""The population is tracked REGULAR files. A tracked symlink under a process prefix is a path
|
|
the predicate admits whose worktree content is its TARGET: a directory (this repo's
|
|
`.claude/skills/jellyfin`) would be skipped with a notice, and a file outside the population
|
|
would be scanned under the link's name — the target's narrative attributed to a path the rule
|
|
does not reach. Both are excluded by mode, and the count says so."""
|
|
(repo / "web").mkdir()
|
|
(repo / "web" / "target.md").write_text(f"{NARRATIVE}\n", encoding="utf-8")
|
|
(repo / ".claude" / "skills").mkdir(parents=True)
|
|
(repo / ".claude" / "skills" / "linked.md").symlink_to("../../web/target.md")
|
|
(repo / ".claude" / "skills" / "dir").symlink_to("../../web")
|
|
(repo / ".claude" / "hooks").mkdir()
|
|
(repo / ".claude" / "hooks" / "h.sh").write_text(f"# {NARRATIVE}\n", encoding="utf-8")
|
|
commit(repo)
|
|
out = run(repo, "--all").stdout
|
|
assert hits(out) == {(".claude/hooks/h.sh", 1)}
|
|
assert "skipped" not in out
|
|
# the fixture's `docs/seed.md` plus the hook: neither symlink is counted, and neither is skipped
|
|
assert "scanned 2 file(s)" in out
|
|
|
|
|
|
def test_an_UNMERGED_path_is_scanned_once_and_counted_once(repo: Path) -> None:
|
|
"""`git ls-files -s` emits one row per index STAGE, so a path in conflict appears up to three
|
|
times. Without de-duplication `--all` opens the worktree file once per row: the same warning
|
|
three times over, and a population count that is not a count of files."""
|
|
git(repo, "checkout", "-qb", "side")
|
|
(repo / "docs" / "seed.md").write_text(f"side\n{NARRATIVE}\n", encoding="utf-8")
|
|
commit(repo)
|
|
git(repo, "checkout", "-q", "-")
|
|
(repo / "docs" / "seed.md").write_text(f"main\n{NARRATIVE}\n", encoding="utf-8")
|
|
commit(repo)
|
|
merge = subprocess.run(["git", "merge", "side"], cwd=repo, capture_output=True, text=True)
|
|
assert merge.returncode != 0, "the fixture must be in conflict"
|
|
stages = subprocess.run(["git", "ls-files", "-s", "--", "docs/seed.md"], cwd=repo, capture_output=True, text=True)
|
|
assert stages.stdout.count("\n") == 3, stages.stdout
|
|
out = run(repo, "--all").stdout
|
|
assert out.count("::warning") == 1
|
|
assert "scanned 1 file(s)" in out
|
|
|
|
|
|
def test_a_MIXED_MODE_conflict_is_not_opened(repo: Path) -> None:
|
|
"""Two stages of DIFFERENT mode under one path — a symlink beside a regular blob — with the
|
|
worktree holding one of them. A per-row mode test lets the regular row authorise opening the
|
|
path, which may be the symlink; a path is opened only when EVERY stage is a regular blob.
|
|
|
|
Built with `update-index --index-info` rather than a merge: git resolves a symlink/file add/add
|
|
conflict by RENAMING one side (`h.sh~HEAD`), so a merge never yields this index shape — an
|
|
older git, a tool, or a hand-edited index can, and `ls-files -s` reports it exactly like this."""
|
|
(repo / ".claude" / "hooks").mkdir(parents=True)
|
|
(repo / ".claude" / "hooks" / "h.sh").write_text(f"# {NARRATIVE}\n", encoding="utf-8")
|
|
commit(repo)
|
|
blob = subprocess.run(
|
|
["git", "rev-parse", "HEAD:.claude/hooks/h.sh"], cwd=repo, capture_output=True, text=True
|
|
).stdout.strip()
|
|
link = subprocess.run(
|
|
["git", "hash-object", "-w", "--stdin"], cwd=repo, input="../../web/t.sh", capture_output=True, text=True
|
|
).stdout.strip()
|
|
subprocess.run(
|
|
["git", "update-index", "--index-info"],
|
|
cwd=repo,
|
|
input=(
|
|
"0 0000000000000000000000000000000000000000\t.claude/hooks/h.sh\n"
|
|
f"100644 {blob} 2\t.claude/hooks/h.sh\n"
|
|
f"120000 {link} 3\t.claude/hooks/h.sh\n"
|
|
),
|
|
text=True,
|
|
check=True,
|
|
)
|
|
stages = subprocess.run(
|
|
["git", "ls-files", "-s", "--", ".claude/hooks/h.sh"], cwd=repo, capture_output=True, text=True
|
|
)
|
|
assert {line.split(" ", 1)[0] for line in stages.stdout.splitlines()} == {"100644", "120000"}, stages.stdout
|
|
out = run(repo, "--all").stdout
|
|
assert hits(out) == set()
|
|
assert "skipped" not in out
|
|
assert "scanned 1 file(s)" in out # the fixture's docs/seed.md only
|
|
|
|
|
|
def test_a_path_with_a_SPACE_is_scanned_in_diff_mode(repo: Path) -> None:
|
|
"""Git terminates the `+++` filename field with a TAB when the path contains a space. A parser
|
|
that keeps the tab asks `is_scanned_path("docs/my notes.md\\t")`, which is False, and the file is
|
|
dropped with no notice — the scanned-0 channel one level below the config pins. The positive
|
|
control is the same content at a space-free path in the same diff."""
|
|
(repo / "docs" / "my notes.md").write_text(f"{NARRATIVE}\n", encoding="utf-8")
|
|
(repo / "docs" / "mynotes.md").write_text(f"{NARRATIVE}\n", encoding="utf-8")
|
|
commit(repo)
|
|
out = run(repo, "--diff", "HEAD~1").stdout
|
|
assert hits(out) == {("docs/my notes.md", 1), ("docs/mynotes.md", 1)}
|
|
assert "scanned 2 file(s)" in out
|
|
|
|
|
|
# --- the detector actually detects ---------------------------------------------------------------
|
|
|
|
|
|
def test_a_MULTI_FILE_diff_scans_every_file_not_just_the_first(repo: Path) -> None:
|
|
"""Every real CI run is multi-file. Without the `in_hunk` reset on `diff --git`, the parser stays
|
|
inside the previous file's hunk and silently drops every file after the first."""
|
|
for name in ("a", "b", "c"):
|
|
(repo / "docs" / f"{name}.md").write_text("x\n", encoding="utf-8")
|
|
commit(repo)
|
|
for name in ("a", "b", "c"):
|
|
(repo / "docs" / f"{name}.md").write_text(f"x\n{NARRATIVE}\n", encoding="utf-8")
|
|
commit(repo)
|
|
p = run(repo, "--diff", "HEAD~1")
|
|
assert hits(p.stdout) == {(f"docs/{n}.md", 2) for n in ("a", "b", "c")}
|
|
assert "scanned 3 file(s); 3 advisory warning(s)" in p.stdout
|
|
|
|
|
|
def test_the_reported_POPULATION_COUNT_matches_the_files_actually_scanned(repo: Path) -> None:
|
|
"""The count is the observable that made every false clean in this file's history visible. A
|
|
mutant that never populated the scanned set reported `scanned 0 file(s); 3 warning(s)` — green,
|
|
and self-contradictory."""
|
|
(repo / "docs" / "one.md").write_text("x\n", encoding="utf-8")
|
|
(repo / "docs" / "two.md").write_text("x\n", encoding="utf-8")
|
|
commit(repo)
|
|
(repo / "docs" / "one.md").write_text(f"x\n{NARRATIVE}\n", encoding="utf-8")
|
|
(repo / "docs" / "two.md").write_text("x\nharmless\n", encoding="utf-8")
|
|
commit(repo)
|
|
assert "scanned 2 file(s); 1 advisory warning(s)" in run(repo, "--diff", "HEAD~1").stdout
|
|
|
|
|
|
def test_a_NON_UTF8_LOCALE_does_not_silently_empty_the_scan(repo: Path) -> None:
|
|
"""`subprocess.run(errors="replace")` is the other half of the locale channel: without it a
|
|
UTF-8 doc under an ascii locale raises inside `git()`, the bare `except` catches it, and the
|
|
whole scan degrades to nothing while still exiting 0."""
|
|
import os
|
|
|
|
(repo / "docs" / "u8.md").write_text(f"héllo — em dash\n{NARRATIVE}\n", encoding="utf-8")
|
|
commit(repo)
|
|
env = dict(os.environ, LC_ALL="C", PYTHONUTF8="0", PYTHONIOENCODING="utf-8")
|
|
p = subprocess.run(
|
|
[sys.executable, str(SCRIPT), "--diff", "HEAD~1"],
|
|
cwd=repo,
|
|
capture_output=True,
|
|
text=True,
|
|
env=env,
|
|
)
|
|
assert p.returncode == 0, p.stderr
|
|
assert "internal error" not in p.stdout, p.stdout
|
|
assert hits(p.stdout) == {("docs/u8.md", 2)}
|
|
|
|
|
|
def test_the_pattern_is_case_insensitive(repo: Path) -> None:
|
|
(repo / "docs" / "case.md").write_text("AN EARLIER DRAFT of this said otherwise\n", encoding="utf-8")
|
|
commit(repo)
|
|
assert hits(run(repo, "--all").stdout) == {("docs/case.md", 1)}
|
|
|
|
|
|
def test_MUTATION_neutering_the_pattern_makes_a_known_hit_go_quiet(repo: Path, tmp_path: Path) -> None:
|
|
"""The clause-level disarm: with PATTERNS unable to match, a file that IS flagged stops being
|
|
flagged. Without this, every assertion above is satisfied by a detector that finds nothing."""
|
|
(repo / "docs" / "d.md").write_text(f"x\n{NARRATIVE}\n", encoding="utf-8")
|
|
commit(repo)
|
|
assert hits(run(repo, "--all").stdout) == truth(repo, "docs/d.md")
|
|
|
|
disarmed = tmp_path / "disarmed.py"
|
|
src = SCRIPT.read_text(encoding="utf-8")
|
|
marker = "PATTERNS = re.compile("
|
|
assert src.count(marker) == 1
|
|
disarmed.write_text(
|
|
src.replace(marker, 'PATTERNS = re.compile(r"(?!x)x" # disarmed\n or ', 1), encoding="utf-8"
|
|
)
|
|
p = subprocess.run([sys.executable, str(disarmed), "--all"], cwd=repo, capture_output=True, text=True)
|
|
assert p.returncode == 0, p.stderr
|
|
assert hits(p.stdout) == set(), "the disarmed detector still flagged something — the mutation did not take"
|