"""Proofs for `scripts/check-doc-narrative.py` (ersatztv#784). Every case below is a defect a cold review DEMONSTRATED in the first, shell implementation. They are here because the never-fails invariant and the reported line numbers are both asserted in prose in four places (the script header, the workflow comment, `docs/guard-inventory.md` and `docs/remote-state-inventory.md`), and an invariant asserted only in prose is the shape this repo keeps getting wrong. The line-number cases all compare against the TRUTH computed from the file on disk, never against a number written into the test — a hand-written expectation is a second copy of the parser. """ from __future__ import annotations import re import subprocess import sys from pathlib import Path import pytest REPO = Path(__file__).resolve().parents[2] SCRIPT = REPO / "scripts" / "check-doc-narrative.py" NARRATIVE = "I initially thought otherwise" HIT = re.compile(r"^::warning file=(?P[^:]+)::(?P=path):(?P\d+) ", re.MULTILINE) def run(cwd: Path, *args: str) -> subprocess.CompletedProcess: return subprocess.run([sys.executable, str(SCRIPT), *args], cwd=cwd, capture_output=True, text=True) def hits(out: str) -> set[tuple[str, int]]: return {(m.group("path"), int(m.group("line"))) for m in HIT.finditer(out)} def truth(root: Path, rel: str) -> set[tuple[str, int]]: """Where the narrative marker ACTUALLY is, read back off disk.""" text = (root / rel).read_text(encoding="utf-8") return { (rel, i) for i, line in enumerate(text.splitlines(), start=1) if "initially thought" in line or "an earlier draft" in line.lower() } def git(repo: Path, *args: str) -> None: subprocess.run(["git", *args], cwd=repo, check=True, capture_output=True) @pytest.fixture() def repo(tmp_path: Path) -> Path: r = tmp_path / "r" (r / "docs").mkdir(parents=True) git(r.parent, "init", "-q", "r") git(r, "config", "user.email", "t@example.com") git(r, "config", "user.name", "t") (r / "docs" / "seed.md").write_text("seed\n", encoding="utf-8") git(r, "add", "-A") git(r, "commit", "-qm", "base") return r def commit(r: Path, msg: str = "c") -> None: git(r, "add", "-A") git(r, "commit", "-qm", msg) # --- the never-fails invariant ------------------------------------------------------------------- @pytest.mark.parametrize( ("args", "expected"), [ ((), "scanned "), (("--all",), "scanned "), (("--diff",), "--diff needs a base ref"), # no base ref at all (("--diff", ""), "--diff needs a base ref"), # `origin/` with base_ref unset (("--diff", "no-such-ref-xyz"), "SCANNED NOTHING"), # an unresolvable ref (("--diff", "origin/"), "SCANNED NOTHING"), (("--nonsense",), "unknown mode"), ], ) def test_every_argument_shape_exits_zero_HAVING_HANDLED_IT(repo: Path, args, expected: str) -> None: """The header, the workflow comment and both inventories state exit 0 absolutely. The expected message is asserted alongside the exit code on purpose: the script ends with a bare `except` that returns 0, so an exit-code-only assertion is satisfied by an unhandled crash and would pass against a script that handles none of these shapes. """ p = run(repo, *args) assert p.returncode == 0, f"args={args} exited {p.returncode}: {p.stderr}" assert expected in p.stdout, f"args={args} exited 0 but did not HANDLE it: {p.stdout!r}" assert "internal error" not in p.stdout, f"args={args} reached the last-resort handler: {p.stdout!r}" def test_an_unknown_mode_prints_usage_and_scans_nothing(repo: Path) -> None: """Stated as the observable behaviour it actually pins. It is NOT a proof that the else-branch cannot reach `run_all`: the branch returns before the warnings are printed, so a mutant calling `run_all` there is silent and no black-box test can see it.""" (repo / "docs" / "u.md").write_text(f"x\n{NARRATIVE}\n", encoding="utf-8") commit(repo) p = run(repo, "--nonsense") assert hits(p.stdout) == set() assert "scanned" not in p.stdout def test_a_DELETED_doc_is_not_reported_as_added_content(repo: Path) -> None: """A deletion contributes no added lines. Stated exactly: this pins the BEHAVIOUR, and it is NOT a proof of the `+++ /dev/null` arm, which is defensive — removing that arm reddens nothing, because a deletion yields no `+` lines either way. The script comment says the same. A docstring claiming a proof it does not have is worse than no docstring: it is the thing a later reader trusts instead of re-checking.""" (repo / "docs" / "del.md").write_text(f"x\n{NARRATIVE}\n", encoding="utf-8") commit(repo) (repo / "docs" / "del.md").unlink() commit(repo) assert hits(run(repo, "--diff", "HEAD~1").stdout) == set() # Every knob DEMONSTRATED to break the parse — not every knob that reshapes diff output, which is a # universal nobody can check and which is the enumeration that failed three rounds running. # (`diff.mnemonicPrefix` is a live example of one that reshapes the header and has no row: it emits # `+++ w/f.md`. It is inert HERE for a reason worth stating exactly, because the obvious explanation # is the wrong one — not because `--dst-prefix=b/` beats it, though it does, but because git only # uses mnemonic prefixes when a diff side is the worktree or the index, and `run_diff` issues the # three-dot `base...HEAD` form, where git emits plain `a/`…`b/` either way. Measured, not reasoned.) # Three of these were each demonstrated turning # a real hit into `scanned 0 file(s)`, one at a time, in three separate rounds — which is why the fix # stopped pinning variants and removed the surface. The table is here so the next knob someone finds # gets a row instead of a round. FORMAT_KNOBS = [ ("core.quotePath", "true"), # quotes non-ASCII paths out of the population ("diff.dstPrefix", "dst/"), # rewrites the header the path is read from ("diff.srcPrefix", "src/"), ("diff.noprefix", "true"), ("color.diff", "always"), # injects ANSI escapes into every line ("color.ui", "always"), ("diff.renames", "false"), # turns a `git mv` into a whole-file add ("diff.context", "9"), # a configured context must not beat the -U0 on the CLI ("diff.external", "/bin/echo"), # replaces the output wholesale ] @pytest.mark.parametrize(("key", "value"), FORMAT_KNOBS) def test_no_git_FORMAT_CONFIG_can_produce_a_false_clean(repo: Path, key: str, value: str) -> None: """A false clean is the worst outcome available to an advisory check: it is indistinguishable from a real one and nobody looks twice.""" (repo / "docs" / "pfx.md").write_text("x\n", encoding="utf-8") commit(repo) (repo / "docs" / "pfx.md").write_text(f"x\n{NARRATIVE}\n", encoding="utf-8") commit(repo) git(repo, "config", key, value) assert hits(run(repo, "--diff", "HEAD~1").stdout) == truth(repo, "docs/pfx.md"), f"{key}={value}" # Only PYTHONIOENCODING is listed. `LC_ALL=C` was here too and was VACUOUS — PEP 540 UTF-8 mode # means the pre-fix script passed it as well, so it read as a second witness where there was one. @pytest.mark.parametrize("env_name,env_value", [("PYTHONIOENCODING", "ascii")]) def test_a_NON_UTF8_stdio_does_not_break_the_never_fails_invariant(repo: Path, env_name, env_value) -> None: """Both the summary line and the last-resort handler carry U+2014, so under ascii stdio the code guaranteeing exit 0 was itself what raised.""" import os env = dict(os.environ, **{env_name: env_value}) env.pop("PYTHONUTF8", None) p = subprocess.run([sys.executable, str(SCRIPT), "--all"], cwd=repo, capture_output=True, text=True, env=env) assert p.returncode == 0, f"{env_name}={env_value} exited {p.returncode}: {p.stderr}" def test_a_tracked_but_DELETED_doc_does_not_kill_the_run(repo: Path) -> None: """`git ls-files` lists index entries; deleting a doc before committing is ordinary.""" (repo / "docs" / "gone.md").write_text("x\n", encoding="utf-8") commit(repo) (repo / "docs" / "gone.md").unlink() p = run(repo, "--all") assert p.returncode == 0 assert "skipped docs/gone.md" in p.stdout def test_an_unresolvable_base_REPORTS_that_it_scanned_nothing(repo: Path) -> None: """A silent zero-file scan is indistinguishable from a clean one — the whole point of #751.""" p = run(repo, "--diff", "no-such-ref-xyz") assert p.returncode == 0 assert "SCANNED NOTHING" in p.stdout # The guard names the line this branch must NOT also print. It previously named "nothing to # flag", a string that occurs nowhere in this script (it belongs to the sibling parity step), so # it could never fail — a negative assertion over a literal that does not exist is not a guard. assert "scanned 0 file(s)" not in p.stdout, "printed a clean-looking summary after scanning nothing" def test_a_genuine_clean_scan_REPORTS_its_population(repo: Path) -> None: p = run(repo, "--all") assert "scanned 1 file(s); 0 advisory warning(s)" in p.stdout # --- line numbers, against truth read off disk --------------------------------------------------- def test_a_file_with_NO_trailing_newline_does_not_shift_later_lines(repo: Path) -> None: """`\\ No newline at end of file` is a marker, not a line of the new file.""" (repo / "docs" / "n.md").write_text("a\nb\nc", encoding="utf-8") # no trailing newline commit(repo) (repo / "docs" / "n.md").write_text(f"a\nb\nZ\n{NARRATIVE}\n", encoding="utf-8") commit(repo) assert hits(run(repo, "--diff", "HEAD~1").stdout) == truth(repo, "docs/n.md") def test_an_added_line_whose_TEXT_starts_with_plus_plus_is_content_not_a_header(repo: Path) -> None: """Docs here embed diff output in fenced blocks, so `++ ` at column 0 is real.""" (repo / "docs" / "p.md").write_text("p\n", encoding="utf-8") commit(repo) (repo / "docs" / "p.md").write_text(f"p\n++ a fenced diff line\n{NARRATIVE}\n", encoding="utf-8") commit(repo) assert hits(run(repo, "--diff", "HEAD~1").stdout) == truth(repo, "docs/p.md") def test_a_multi_hunk_file_numbers_every_hunk_from_its_own_header(repo: Path) -> None: body = [f"l{i}" for i in range(1, 31)] (repo / "docs" / "m.md").write_text("\n".join(body) + "\n", encoding="utf-8") commit(repo) body[4] = NARRATIVE # replace, early hunk body.insert(15, NARRATIVE) # pure insert, middle hunk body[-1] = NARRATIVE # replace, last hunk (repo / "docs" / "m.md").write_text("\n".join(body) + "\n", encoding="utf-8") commit(repo) assert hits(run(repo, "--diff", "HEAD~1").stdout) == truth(repo, "docs/m.md") def test_a_NON_ASCII_path_is_scanned_rather_than_silently_skipped(repo: Path) -> None: """`core.quotePath` quotes the path, and a quoted path matches no scope rule — it vanishes.""" (repo / "docs" / "café.md").write_text(f"x\n{NARRATIVE}\n", encoding="utf-8") commit(repo) assert hits(run(repo, "--all").stdout) == truth(repo, "docs/café.md") assert hits(run(repo, "--diff", "HEAD~1").stdout) == truth(repo, "docs/café.md") def test_a_final_UNTERMINATED_line_is_still_scanned_in_all_mode(repo: Path) -> None: (repo / "docs" / "t.md").write_text(f"ok\n{NARRATIVE}", encoding="utf-8") # no trailing newline commit(repo) assert hits(run(repo, "--all").stdout) == truth(repo, "docs/t.md") def test_a_RENAME_does_not_re_flag_the_whole_pre_existing_file(repo: Path) -> None: """Without rename detection a `git mv` reports every line of the file as newly added.""" (repo / "docs" / "r1.md").write_text("a\n" + f"{NARRATIVE}\n" + "b\n", encoding="utf-8") commit(repo) git(repo, "mv", "docs/r1.md", "docs/r2.md") commit(repo) assert hits(run(repo, "--diff", "HEAD~1").stdout) == set() # `diff.renames` defaults to true, so the assertion above passes with or without the explicit # `--find-renames`. Turning the config off is what makes that flag load-bearing and this test a # real proof of it rather than a restatement of a git default. git(repo, "config", "diff.renames", "false") assert hits(run(repo, "--diff", "HEAD~1").stdout) == set(), "the --find-renames pin is not doing its job" # --- the population ------------------------------------------------------------------------------ @pytest.mark.parametrize("mode", ["--all", "--diff"]) def test_decision_records_are_exempt_in_BOTH_modes(repo: Path, mode: str) -> None: for rel in ("docs/decisions/records/x/y.md", "docs/decisions/archive/x/y.md"): p = repo / rel p.parent.mkdir(parents=True, exist_ok=True) p.write_text(f"x\n{NARRATIVE}\n", encoding="utf-8") commit(repo) out = run(repo, mode, "HEAD~1").stdout if mode == "--diff" else run(repo, mode).stdout assert hits(out) == set() def test_markdown_outside_the_stated_population_is_not_scanned(repo: Path) -> None: """The population is `docs/**/*.md` minus `docs/decisions/**`, plus root-level `*.md`. A skill under `.claude/` is out of scope, and the record's `mechanics:` says so.""" (repo / ".claude" / "skills" / "s").mkdir(parents=True) (repo / ".claude" / "skills" / "s" / "SKILL.md").write_text(f"{NARRATIVE}\n", encoding="utf-8") (repo / "README.md").write_text(f"{NARRATIVE}\n", encoding="utf-8") commit(repo) assert hits(run(repo, "--all").stdout) == {("README.md", 1)} # --- the detector actually detects --------------------------------------------------------------- def test_a_MULTI_FILE_diff_scans_every_file_not_just_the_first(repo: Path) -> None: """Every real CI run is multi-file. Without the `in_hunk` reset on `diff --git`, the parser stays inside the previous file's hunk and silently drops every file after the first.""" for name in ("a", "b", "c"): (repo / "docs" / f"{name}.md").write_text("x\n", encoding="utf-8") commit(repo) for name in ("a", "b", "c"): (repo / "docs" / f"{name}.md").write_text(f"x\n{NARRATIVE}\n", encoding="utf-8") commit(repo) p = run(repo, "--diff", "HEAD~1") assert hits(p.stdout) == {(f"docs/{n}.md", 2) for n in ("a", "b", "c")} assert "scanned 3 file(s); 3 advisory warning(s)" in p.stdout def test_the_reported_POPULATION_COUNT_matches_the_files_actually_scanned(repo: Path) -> None: """The count is the observable that made every false clean in this file's history visible. A mutant that never populated the scanned set reported `scanned 0 file(s); 3 warning(s)` — green, and self-contradictory.""" (repo / "docs" / "one.md").write_text("x\n", encoding="utf-8") (repo / "docs" / "two.md").write_text("x\n", encoding="utf-8") commit(repo) (repo / "docs" / "one.md").write_text(f"x\n{NARRATIVE}\n", encoding="utf-8") (repo / "docs" / "two.md").write_text("x\nharmless\n", encoding="utf-8") commit(repo) assert "scanned 2 file(s); 1 advisory warning(s)" in run(repo, "--diff", "HEAD~1").stdout def test_a_NON_UTF8_LOCALE_does_not_silently_empty_the_scan(repo: Path) -> None: """`subprocess.run(errors="replace")` is the other half of the locale channel: without it a UTF-8 doc under an ascii locale raises inside `git()`, the bare `except` catches it, and the whole scan degrades to nothing while still exiting 0.""" import os (repo / "docs" / "u8.md").write_text(f"héllo — em dash\n{NARRATIVE}\n", encoding="utf-8") commit(repo) env = dict(os.environ, LC_ALL="C", PYTHONUTF8="0", PYTHONIOENCODING="utf-8") p = subprocess.run( [sys.executable, str(SCRIPT), "--diff", "HEAD~1"], cwd=repo, capture_output=True, text=True, env=env, ) assert p.returncode == 0, p.stderr assert "internal error" not in p.stdout, p.stdout assert hits(p.stdout) == {("docs/u8.md", 2)} def test_the_pattern_is_case_insensitive(repo: Path) -> None: (repo / "docs" / "case.md").write_text("AN EARLIER DRAFT of this said otherwise\n", encoding="utf-8") commit(repo) assert hits(run(repo, "--all").stdout) == {("docs/case.md", 1)} def test_MUTATION_neutering_the_pattern_makes_a_known_hit_go_quiet(repo: Path, tmp_path: Path) -> None: """The clause-level disarm: with PATTERNS unable to match, a file that IS flagged stops being flagged. Without this, every assertion above is satisfied by a detector that finds nothing.""" (repo / "docs" / "d.md").write_text(f"x\n{NARRATIVE}\n", encoding="utf-8") commit(repo) assert hits(run(repo, "--all").stdout) == truth(repo, "docs/d.md") disarmed = tmp_path / "disarmed.py" src = SCRIPT.read_text(encoding="utf-8") marker = "PATTERNS = re.compile(" assert src.count(marker) == 1 disarmed.write_text( src.replace(marker, 'PATTERNS = re.compile(r"(?!x)x" # disarmed\n or ', 1), encoding="utf-8" ) p = subprocess.run([sys.executable, str(disarmed), "--all"], cwd=repo, capture_output=True, text=True) assert p.returncode == 0, p.stderr assert hits(p.stdout) == set(), "the disarmed detector still flagged something — the mutation did not take"