timothyandtimothy
736649b3b7
Build ErsatzTV Image / CI toolchain image resolves (push) Successful in 7s
Build ErsatzTV Image / Delimiter ban (release path) (push) Successful in 22s
Build ErsatzTV Image / Build & test (.NET) (push) Successful in 9m18s
Build ErsatzTV Image / EF migration integrity (SQLite + MySql) (push) Successful in 6m33s
Build ErsatzTV Image / Functional E2E (curl + UI contracts) (push) Skipped
Build ErsatzTV Image / API docs in sync (OpenAPI + endpoint index) (push) Skipped
Build ErsatzTV Image / Formatting (changed .cs conform to .editorconfig) (push) Skipped
Build ErsatzTV Image / Build & push image (amd64) (push) Successful in 4m52s
Co-authored-by: Timothy <timothy@noreply.gitea.tblindustries.be>
488 lines
25 KiB
Python
488 lines
25 KiB
Python
"""Proofs for `scripts/check-doc-narrative.py` (ersatztv#784).
|
|
|
|
Every case below is a defect a cold review DEMONSTRATED in the first, shell implementation. They are
|
|
here because the never-fails invariant and the reported line numbers are both asserted in prose in
|
|
four places (the script header, the workflow comment, `docs/guard-inventory.md` and
|
|
`docs/remote-state-inventory.md`), and an invariant asserted only in prose is the shape this repo
|
|
keeps getting wrong.
|
|
|
|
The line-number cases all compare against the TRUTH computed from the file on disk, never against a
|
|
number written into the test — a hand-written expectation is a second copy of the parser.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import importlib.util
|
|
import re
|
|
import subprocess
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
REPO = Path(__file__).resolve().parents[2]
|
|
SCRIPT = REPO / "scripts" / "check-doc-narrative.py"
|
|
|
|
NARRATIVE = "I initially thought otherwise"
|
|
HIT = re.compile(r"^::warning file=(?P<path>[^:]+)::(?P=path):(?P<line>\d+) ", re.MULTILINE)
|
|
|
|
|
|
def run(cwd: Path, *args: str) -> subprocess.CompletedProcess:
|
|
return subprocess.run([sys.executable, str(SCRIPT), *args], cwd=cwd, capture_output=True, text=True)
|
|
|
|
|
|
def hits(out: str) -> set[tuple[str, int]]:
|
|
return {(m.group("path"), int(m.group("line"))) for m in HIT.finditer(out)}
|
|
|
|
|
|
def truth(root: Path, rel: str) -> set[tuple[str, int]]:
|
|
"""Where the narrative marker ACTUALLY is, read back off disk."""
|
|
text = (root / rel).read_text(encoding="utf-8")
|
|
return {
|
|
(rel, i)
|
|
for i, line in enumerate(text.splitlines(), start=1)
|
|
if "initially thought" in line or "an earlier draft" in line.lower()
|
|
}
|
|
|
|
|
|
def git(repo: Path, *args: str) -> None:
|
|
subprocess.run(["git", *args], cwd=repo, check=True, capture_output=True)
|
|
|
|
|
|
@pytest.fixture()
|
|
def repo(tmp_path: Path) -> Path:
|
|
r = tmp_path / "r"
|
|
(r / "docs").mkdir(parents=True)
|
|
git(r.parent, "init", "-q", "r")
|
|
git(r, "config", "user.email", "t@example.com")
|
|
git(r, "config", "user.name", "t")
|
|
(r / "docs" / "seed.md").write_text("seed\n", encoding="utf-8")
|
|
git(r, "add", "-A")
|
|
git(r, "commit", "-qm", "base")
|
|
return r
|
|
|
|
|
|
def commit(r: Path, msg: str = "c") -> None:
|
|
git(r, "add", "-A")
|
|
git(r, "commit", "-qm", msg)
|
|
|
|
|
|
# --- the never-fails invariant -------------------------------------------------------------------
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("args", "expected"),
|
|
[
|
|
((), "scanned "),
|
|
(("--all",), "scanned "),
|
|
(("--diff",), "--diff needs a base ref"), # no base ref at all
|
|
(("--diff", ""), "--diff needs a base ref"), # `origin/` with base_ref unset
|
|
(("--diff", "no-such-ref-xyz"), "SCANNED NOTHING"), # an unresolvable ref
|
|
(("--diff", "origin/"), "SCANNED NOTHING"),
|
|
(("--nonsense",), "unknown mode"),
|
|
],
|
|
)
|
|
def test_every_argument_shape_exits_zero_HAVING_HANDLED_IT(repo: Path, args, expected: str) -> None:
|
|
"""The header, the workflow comment and both inventories state exit 0 absolutely.
|
|
|
|
The expected message is asserted alongside the exit code on purpose: the script ends with a bare
|
|
`except` that returns 0, so an exit-code-only assertion is satisfied by an unhandled crash and
|
|
would pass against a script that handles none of these shapes.
|
|
"""
|
|
p = run(repo, *args)
|
|
assert p.returncode == 0, f"args={args} exited {p.returncode}: {p.stderr}"
|
|
assert expected in p.stdout, f"args={args} exited 0 but did not HANDLE it: {p.stdout!r}"
|
|
assert "internal error" not in p.stdout, f"args={args} reached the last-resort handler: {p.stdout!r}"
|
|
|
|
|
|
def test_an_unknown_mode_prints_usage_and_scans_nothing(repo: Path) -> None:
|
|
"""Stated as the observable behaviour it actually pins. It is NOT a proof that the else-branch
|
|
cannot reach `run_all`: the branch returns before the warnings are printed, so a mutant calling
|
|
`run_all` there is silent and no black-box test can see it."""
|
|
(repo / "docs" / "u.md").write_text(f"x\n{NARRATIVE}\n", encoding="utf-8")
|
|
commit(repo)
|
|
p = run(repo, "--nonsense")
|
|
assert hits(p.stdout) == set()
|
|
assert "scanned" not in p.stdout
|
|
|
|
|
|
def test_a_DELETED_doc_is_not_reported_as_added_content(repo: Path) -> None:
|
|
"""A deletion contributes no added lines.
|
|
|
|
Stated exactly: this pins the BEHAVIOUR, and it is NOT a proof of the `+++ /dev/null` arm, which
|
|
is defensive — removing that arm reddens nothing, because a deletion yields no `+` lines either
|
|
way. The script comment says the same. A docstring claiming a proof it does not have is worse
|
|
than no docstring: it is the thing a later reader trusts instead of re-checking."""
|
|
(repo / "docs" / "del.md").write_text(f"x\n{NARRATIVE}\n", encoding="utf-8")
|
|
commit(repo)
|
|
(repo / "docs" / "del.md").unlink()
|
|
commit(repo)
|
|
assert hits(run(repo, "--diff", "HEAD~1").stdout) == set()
|
|
|
|
|
|
# Every knob DEMONSTRATED to break the parse — not every knob that reshapes diff output, which is a
|
|
# universal nobody can check and which is the enumeration that failed three rounds running.
|
|
# (`diff.mnemonicPrefix` is a live example of one that reshapes the header and has no row: it emits
|
|
# `+++ w/f.md`. It is inert HERE for a reason worth stating exactly, because the obvious explanation
|
|
# is the wrong one — not because `--dst-prefix=b/` beats it, though it does, but because git only
|
|
# uses mnemonic prefixes when a diff side is the worktree or the index, and `run_diff` issues the
|
|
# three-dot `base...HEAD` form, where git emits plain `a/`…`b/` either way. Measured, not reasoned.)
|
|
# Three of these were each demonstrated turning
|
|
# a real hit into `scanned 0 file(s)`, one at a time, in three separate rounds — which is why the fix
|
|
# stopped pinning variants and removed the surface. The table is here so the next knob someone finds
|
|
# gets a row instead of a round.
|
|
FORMAT_KNOBS = [
|
|
("core.quotePath", "true"), # quotes non-ASCII paths out of the population
|
|
("diff.dstPrefix", "dst/"), # rewrites the header the path is read from
|
|
("diff.srcPrefix", "src/"),
|
|
("diff.noprefix", "true"),
|
|
("color.diff", "always"), # injects ANSI escapes into every line
|
|
("color.ui", "always"),
|
|
("diff.renames", "false"), # turns a `git mv` into a whole-file add
|
|
("diff.context", "9"), # a configured context must not beat the -U0 on the CLI
|
|
("diff.external", "/bin/echo"), # replaces the output wholesale
|
|
]
|
|
|
|
|
|
@pytest.mark.parametrize(("key", "value"), FORMAT_KNOBS)
|
|
def test_no_git_FORMAT_CONFIG_can_produce_a_false_clean(repo: Path, key: str, value: str) -> None:
|
|
"""A false clean is the worst outcome available to an advisory check: it is indistinguishable
|
|
from a real one and nobody looks twice."""
|
|
(repo / "docs" / "pfx.md").write_text("x\n", encoding="utf-8")
|
|
commit(repo)
|
|
(repo / "docs" / "pfx.md").write_text(f"x\n{NARRATIVE}\n", encoding="utf-8")
|
|
commit(repo)
|
|
git(repo, "config", key, value)
|
|
assert hits(run(repo, "--diff", "HEAD~1").stdout) == truth(repo, "docs/pfx.md"), f"{key}={value}"
|
|
|
|
|
|
# Only PYTHONIOENCODING is listed. `LC_ALL=C` was here too and was VACUOUS — PEP 540 UTF-8 mode
|
|
# means the pre-fix script passed it as well, so it read as a second witness where there was one.
|
|
@pytest.mark.parametrize("env_name,env_value", [("PYTHONIOENCODING", "ascii")])
|
|
def test_a_NON_UTF8_stdio_does_not_break_the_never_fails_invariant(repo: Path, env_name, env_value) -> None:
|
|
"""Both the summary line and the last-resort handler carry U+2014, so under ascii stdio the code
|
|
guaranteeing exit 0 was itself what raised."""
|
|
import os
|
|
|
|
env = dict(os.environ, **{env_name: env_value})
|
|
env.pop("PYTHONUTF8", None)
|
|
p = subprocess.run([sys.executable, str(SCRIPT), "--all"], cwd=repo, capture_output=True, text=True, env=env)
|
|
assert p.returncode == 0, f"{env_name}={env_value} exited {p.returncode}: {p.stderr}"
|
|
|
|
|
|
def test_a_tracked_but_DELETED_doc_does_not_kill_the_run(repo: Path) -> None:
|
|
"""`git ls-files` lists index entries; deleting a doc before committing is ordinary."""
|
|
(repo / "docs" / "gone.md").write_text("x\n", encoding="utf-8")
|
|
commit(repo)
|
|
(repo / "docs" / "gone.md").unlink()
|
|
p = run(repo, "--all")
|
|
assert p.returncode == 0
|
|
assert "skipped docs/gone.md" in p.stdout
|
|
|
|
|
|
def test_an_unresolvable_base_REPORTS_that_it_scanned_nothing(repo: Path) -> None:
|
|
"""A silent zero-file scan is indistinguishable from a clean one — the whole point of #751."""
|
|
p = run(repo, "--diff", "no-such-ref-xyz")
|
|
assert p.returncode == 0
|
|
assert "SCANNED NOTHING" in p.stdout
|
|
# The guard names the line this branch must NOT also print. It previously named "nothing to
|
|
# flag", a string that occurs nowhere in this script (it belongs to the sibling parity step), so
|
|
# it could never fail — a negative assertion over a literal that does not exist is not a guard.
|
|
assert "scanned 0 file(s)" not in p.stdout, "printed a clean-looking summary after scanning nothing"
|
|
|
|
|
|
def test_a_genuine_clean_scan_REPORTS_its_population(repo: Path) -> None:
|
|
p = run(repo, "--all")
|
|
assert "scanned 1 file(s); 0 advisory warning(s)" in p.stdout
|
|
|
|
|
|
# --- line numbers, against truth read off disk ---------------------------------------------------
|
|
|
|
|
|
def test_a_file_with_NO_trailing_newline_does_not_shift_later_lines(repo: Path) -> None:
|
|
"""`\\ No newline at end of file` is a marker, not a line of the new file."""
|
|
(repo / "docs" / "n.md").write_text("a\nb\nc", encoding="utf-8") # no trailing newline
|
|
commit(repo)
|
|
(repo / "docs" / "n.md").write_text(f"a\nb\nZ\n{NARRATIVE}\n", encoding="utf-8")
|
|
commit(repo)
|
|
assert hits(run(repo, "--diff", "HEAD~1").stdout) == truth(repo, "docs/n.md")
|
|
|
|
|
|
def test_an_added_line_whose_TEXT_starts_with_plus_plus_is_content_not_a_header(repo: Path) -> None:
|
|
"""Docs here embed diff output in fenced blocks, so `++ ` at column 0 is real."""
|
|
(repo / "docs" / "p.md").write_text("p\n", encoding="utf-8")
|
|
commit(repo)
|
|
(repo / "docs" / "p.md").write_text(f"p\n++ a fenced diff line\n{NARRATIVE}\n", encoding="utf-8")
|
|
commit(repo)
|
|
assert hits(run(repo, "--diff", "HEAD~1").stdout) == truth(repo, "docs/p.md")
|
|
|
|
|
|
def test_a_multi_hunk_file_numbers_every_hunk_from_its_own_header(repo: Path) -> None:
|
|
body = [f"l{i}" for i in range(1, 31)]
|
|
(repo / "docs" / "m.md").write_text("\n".join(body) + "\n", encoding="utf-8")
|
|
commit(repo)
|
|
body[4] = NARRATIVE # replace, early hunk
|
|
body.insert(15, NARRATIVE) # pure insert, middle hunk
|
|
body[-1] = NARRATIVE # replace, last hunk
|
|
(repo / "docs" / "m.md").write_text("\n".join(body) + "\n", encoding="utf-8")
|
|
commit(repo)
|
|
assert hits(run(repo, "--diff", "HEAD~1").stdout) == truth(repo, "docs/m.md")
|
|
|
|
|
|
def test_a_NON_ASCII_path_is_scanned_rather_than_silently_skipped(repo: Path) -> None:
|
|
"""`core.quotePath` quotes the path, and a quoted path matches no scope rule — it vanishes."""
|
|
(repo / "docs" / "café.md").write_text(f"x\n{NARRATIVE}\n", encoding="utf-8")
|
|
commit(repo)
|
|
assert hits(run(repo, "--all").stdout) == truth(repo, "docs/café.md")
|
|
assert hits(run(repo, "--diff", "HEAD~1").stdout) == truth(repo, "docs/café.md")
|
|
|
|
|
|
def test_a_final_UNTERMINATED_line_is_still_scanned_in_all_mode(repo: Path) -> None:
|
|
(repo / "docs" / "t.md").write_text(f"ok\n{NARRATIVE}", encoding="utf-8") # no trailing newline
|
|
commit(repo)
|
|
assert hits(run(repo, "--all").stdout) == truth(repo, "docs/t.md")
|
|
|
|
|
|
def test_a_RENAME_does_not_re_flag_the_whole_pre_existing_file(repo: Path) -> None:
|
|
"""Without rename detection a `git mv` reports every line of the file as newly added."""
|
|
(repo / "docs" / "r1.md").write_text("a\n" + f"{NARRATIVE}\n" + "b\n", encoding="utf-8")
|
|
commit(repo)
|
|
git(repo, "mv", "docs/r1.md", "docs/r2.md")
|
|
commit(repo)
|
|
assert hits(run(repo, "--diff", "HEAD~1").stdout) == set()
|
|
|
|
# `diff.renames` defaults to true, so the assertion above passes with or without the explicit
|
|
# `--find-renames`. Turning the config off is what makes that flag load-bearing and this test a
|
|
# real proof of it rather than a restatement of a git default.
|
|
git(repo, "config", "diff.renames", "false")
|
|
assert hits(run(repo, "--diff", "HEAD~1").stdout) == set(), "the --find-renames pin is not doing its job"
|
|
|
|
|
|
# --- the population ------------------------------------------------------------------------------
|
|
|
|
|
|
# Hand-written, one entry per exempt prefix. NOT derived from the script — a population read out of
|
|
# the code under test proves only that the code agrees with itself. The completeness guard below is
|
|
# what stops a new prefix from shipping without a case here.
|
|
EXEMPT_SAMPLES = {
|
|
"docs/decisions/": ("docs/decisions/records/x/y.md", "docs/decisions/archive/x/y.md"),
|
|
}
|
|
|
|
|
|
@pytest.mark.parametrize("mode", ["--all", "--diff"])
|
|
def test_every_exempt_genre_is_exempt_in_BOTH_modes(repo: Path, mode: str) -> None:
|
|
"""`docs/decisions/**` is exempt WHOLESALE (#784). Both modes, because an exemption that held
|
|
only for `--all` would still flag the file on the PR that adds it.
|
|
|
|
The CONTROLS are what make this a measurement. `hits(out) == set()` alone also passes when the
|
|
scan examined nothing at all — a scanned-zero run reads identical to a working exemption — so a
|
|
non-exempt file carrying the same marker must come back HIT in the same invocation.
|
|
|
|
Each exempt sample is also paired with a MINIMAL TWIN — the same path with the exempt directory
|
|
renamed — which raises the cost of a structural bypass but does NOT close it. Stated as a limit
|
|
rather than a guarantee, because three successive attempts to close it were each defeated by the
|
|
next round: a lone depth-1 control fell to a depth-1 population rule; a depth-3 control fell to a
|
|
`count("/") <= 3` cap; the twins themselves fell to a directory-NAME rule keyed on the renamed
|
|
SECOND segment (`path.count("/") > 1 and "-" not in path.split("/", 2)[1]`), which skipped both
|
|
samples and kept both twins. It reddens here now, but only because the `docs/superpowers/`
|
|
fixture below happens to have a hyphen-free second segment — luck, not design, and not a
|
|
property this test establishes.
|
|
|
|
So this test proves the scan EXAMINED something; it does NOT prove a sample's absence is
|
|
attributable to `EXEMPT_PREFIXES` rather than to its shape. Do not add a fourth finite control —
|
|
each of the three was locally correct and the sequence did not converge, the same shape as the
|
|
withdrawn `test_review_verdict_vocabulary_parity.py` (six rounds, then deleted) that
|
|
`docs.no-session-narrative` cites as the case for stopping. What DOES close it is a closed-form
|
|
proof rather than a fixture:
|
|
`test_the_population_agrees_with_an_INDEPENDENT_RESTATEMENT_over_the_REAL_corpus` below.
|
|
"""
|
|
for rels in EXEMPT_SAMPLES.values():
|
|
for rel in rels:
|
|
f = repo / rel
|
|
f.parent.mkdir(parents=True, exist_ok=True)
|
|
f.write_text(f"x\n{NARRATIVE}\n", encoding="utf-8")
|
|
# The twin of `docs/decisions/records/x/y.md` is `docs/decisions-not/records/x/y.md`: identical
|
|
# in every structural respect, and NOT under the exempt prefix.
|
|
twins = {rel.replace(prefix, prefix[:-1] + "-not/", 1) for prefix, rels in EXEMPT_SAMPLES.items() for rel in rels}
|
|
# `docs/superpowers/` is deliberately NOT exempt (#812), and re-exempting it is the specific
|
|
# regression this change argues against. It is named here rather than left to the closed-form
|
|
# oracle, because that oracle proves a property of `is_scanned_path`, not that the SCAN consults
|
|
# it: a `continue` added in `run_all`/`run_diff` reinstates the exemption with the function
|
|
# untouched. Only a path that reaches the OUTPUT witnesses the wiring.
|
|
twins = twins | {"docs/superpowers/plans/2026-01-01-x.md"}
|
|
for rel in twins | {"docs/control.md"}:
|
|
control = repo / rel
|
|
control.parent.mkdir(parents=True, exist_ok=True)
|
|
control.write_text(f"x\n{NARRATIVE}\n", encoding="utf-8")
|
|
commit(repo)
|
|
out = run(repo, mode, "HEAD~1").stdout if mode == "--diff" else run(repo, mode).stdout
|
|
assert hits(out) == {(rel, 2) for rel in twins | {"docs/control.md"}}
|
|
|
|
|
|
def test_the_population_agrees_with_an_INDEPENDENT_RESTATEMENT_over_the_REAL_corpus() -> None:
|
|
"""CLOSED-FORM. This is what three successive finite controls could not buy, and it is why the
|
|
retraction in the exemption test's docstring stops at "not another fixture" rather than at
|
|
"unprovable".
|
|
|
|
The rule as the record states it — `docs/**/*.md` minus `docs/decisions/**`, plus root-level
|
|
`*.md` — is restated below over path SEGMENTS rather than string prefixes, so the two are not
|
|
one expression copy-pasted, then compared against `is_scanned_path` on every tracked Markdown
|
|
path in this repo. A finite fixture can only witness the shapes someone thought to write down;
|
|
this witnesses every real path, so any population change that alters a verdict on one of them
|
|
reddens — the depth-1 rule and the directory-NAME rule among them.
|
|
|
|
It is NOT sufficient on its own, and the arm below says why with the measurement: a
|
|
`count("/") <= 3` cap changes no real path's verdict today, so within this test only the
|
|
synthetic arm catches it. Do not trim that arm as belt-and-braces; it is the half that sees a
|
|
rule the corpus has no instance of yet. (The exemption test above happens to redden on that cap
|
|
too — its two minimal twins sit at four slashes — but that is incidental, not the arm carrying the claim.)
|
|
"""
|
|
|
|
def restated(path: str) -> bool:
|
|
parts = path.split("/")
|
|
if not parts[-1].endswith(".md"):
|
|
return False
|
|
if len(parts) == 1:
|
|
return True # root-level *.md
|
|
if parts[0] != "docs":
|
|
return False # nested markdown outside docs/ is out of scope
|
|
return parts[1] != "decisions" # docs/** minus docs/decisions/**
|
|
|
|
spec = importlib.util.spec_from_file_location("_cdn_pop", SCRIPT)
|
|
assert spec is not None and spec.loader is not None
|
|
mod = importlib.util.module_from_spec(spec)
|
|
spec.loader.exec_module(mod)
|
|
|
|
rc = subprocess.run(["git", "ls-files", "-z", "--", "*.md"], cwd=REPO, capture_output=True, text=True, check=True)
|
|
paths = [p for p in rc.stdout.split("\0") if p]
|
|
assert len(paths) > 100, f"corpus looks wrong: {len(paths)} paths"
|
|
|
|
disagree = [p for p in paths if mod.is_scanned_path(p) != restated(p)]
|
|
assert not disagree, f"population disagrees with the restated rule on {len(disagree)}: {disagree[:8]}"
|
|
|
|
# Anti-vacuity: an oracle that says False everywhere would agree with a scan-nothing mutant.
|
|
scanned = [p for p in paths if restated(p)]
|
|
exempt = [p for p in paths if p.startswith("docs/decisions/")]
|
|
assert len(scanned) > 50 and len(exempt) > 50, f"{len(scanned)} scanned / {len(exempt)} exempt"
|
|
|
|
# The real corpus alone cannot see a rule that differs only on a shape it does not currently
|
|
# contain — no scanned path carries more than three slashes today, so a `count("/") <= 3` cap is
|
|
# a no-op against it and would ship latent. So the same oracle is also compared over a cross
|
|
# product of FOUR dimensions a population rule has been observed to key on here — depth, first
|
|
# segment, second segment, extension — widened with basename, case and dotted directories after
|
|
# a review round found a mutant keyed on each. This ENUMERATES; it is not a universal over the
|
|
# space of rules, and a dimension on neither list has simply not been measured.
|
|
heads = ["docs", "Docs", "web", "scripts", ".github", ""]
|
|
seconds = ["decisions", "superpowers", "other", "decisions-not", "Decisions", ".hidden", ""]
|
|
leaves = ["f", "index", "README"]
|
|
exts = [".md", ".markdown", ".MD", ".txt", ""]
|
|
synthetic = set()
|
|
for ext in exts:
|
|
for leaf in leaves:
|
|
for head in heads:
|
|
for second in seconds:
|
|
for depth in range(0, 3):
|
|
segs = [p for p in (head, second) if p] + ["x"] * depth
|
|
synthetic.add("/".join(segs + [leaf + ext]))
|
|
bad = [p for p in sorted(synthetic) if mod.is_scanned_path(p) != restated(p)]
|
|
assert not bad, f"population disagrees with the restated rule on {len(bad)} shapes: {bad[:8]}"
|
|
|
|
|
|
def test_every_exempt_prefix_has_a_case_above() -> None:
|
|
"""Completeness: adding a prefix to `EXEMPT_PREFIXES` without a sample above reddens here, so the
|
|
exemption test cannot silently stop covering the population it claims to cover."""
|
|
spec = importlib.util.spec_from_file_location("_cdn", SCRIPT)
|
|
assert spec is not None and spec.loader is not None
|
|
mod = importlib.util.module_from_spec(spec)
|
|
spec.loader.exec_module(mod)
|
|
assert set(EXEMPT_SAMPLES) == set(mod.EXEMPT_PREFIXES)
|
|
for prefix, rels in EXEMPT_SAMPLES.items():
|
|
assert all(r.startswith(prefix) for r in rels), prefix
|
|
|
|
|
|
def test_markdown_outside_the_stated_population_is_not_scanned(repo: Path) -> None:
|
|
"""The population is `docs/**/*.md` minus `EXEMPT_PREFIXES`, plus root-level `*.md`. A skill
|
|
under `.claude/` is out of scope, and the record's `mechanics:` says so — out of the DETECTOR,
|
|
still bound by the RULE."""
|
|
(repo / ".claude" / "skills" / "s").mkdir(parents=True)
|
|
(repo / ".claude" / "skills" / "s" / "SKILL.md").write_text(f"{NARRATIVE}\n", encoding="utf-8")
|
|
(repo / "README.md").write_text(f"{NARRATIVE}\n", encoding="utf-8")
|
|
commit(repo)
|
|
assert hits(run(repo, "--all").stdout) == {("README.md", 1)}
|
|
|
|
|
|
# --- the detector actually detects ---------------------------------------------------------------
|
|
|
|
|
|
def test_a_MULTI_FILE_diff_scans_every_file_not_just_the_first(repo: Path) -> None:
|
|
"""Every real CI run is multi-file. Without the `in_hunk` reset on `diff --git`, the parser stays
|
|
inside the previous file's hunk and silently drops every file after the first."""
|
|
for name in ("a", "b", "c"):
|
|
(repo / "docs" / f"{name}.md").write_text("x\n", encoding="utf-8")
|
|
commit(repo)
|
|
for name in ("a", "b", "c"):
|
|
(repo / "docs" / f"{name}.md").write_text(f"x\n{NARRATIVE}\n", encoding="utf-8")
|
|
commit(repo)
|
|
p = run(repo, "--diff", "HEAD~1")
|
|
assert hits(p.stdout) == {(f"docs/{n}.md", 2) for n in ("a", "b", "c")}
|
|
assert "scanned 3 file(s); 3 advisory warning(s)" in p.stdout
|
|
|
|
|
|
def test_the_reported_POPULATION_COUNT_matches_the_files_actually_scanned(repo: Path) -> None:
|
|
"""The count is the observable that made every false clean in this file's history visible. A
|
|
mutant that never populated the scanned set reported `scanned 0 file(s); 3 warning(s)` — green,
|
|
and self-contradictory."""
|
|
(repo / "docs" / "one.md").write_text("x\n", encoding="utf-8")
|
|
(repo / "docs" / "two.md").write_text("x\n", encoding="utf-8")
|
|
commit(repo)
|
|
(repo / "docs" / "one.md").write_text(f"x\n{NARRATIVE}\n", encoding="utf-8")
|
|
(repo / "docs" / "two.md").write_text("x\nharmless\n", encoding="utf-8")
|
|
commit(repo)
|
|
assert "scanned 2 file(s); 1 advisory warning(s)" in run(repo, "--diff", "HEAD~1").stdout
|
|
|
|
|
|
def test_a_NON_UTF8_LOCALE_does_not_silently_empty_the_scan(repo: Path) -> None:
|
|
"""`subprocess.run(errors="replace")` is the other half of the locale channel: without it a
|
|
UTF-8 doc under an ascii locale raises inside `git()`, the bare `except` catches it, and the
|
|
whole scan degrades to nothing while still exiting 0."""
|
|
import os
|
|
|
|
(repo / "docs" / "u8.md").write_text(f"héllo — em dash\n{NARRATIVE}\n", encoding="utf-8")
|
|
commit(repo)
|
|
env = dict(os.environ, LC_ALL="C", PYTHONUTF8="0", PYTHONIOENCODING="utf-8")
|
|
p = subprocess.run(
|
|
[sys.executable, str(SCRIPT), "--diff", "HEAD~1"],
|
|
cwd=repo,
|
|
capture_output=True,
|
|
text=True,
|
|
env=env,
|
|
)
|
|
assert p.returncode == 0, p.stderr
|
|
assert "internal error" not in p.stdout, p.stdout
|
|
assert hits(p.stdout) == {("docs/u8.md", 2)}
|
|
|
|
|
|
def test_the_pattern_is_case_insensitive(repo: Path) -> None:
|
|
(repo / "docs" / "case.md").write_text("AN EARLIER DRAFT of this said otherwise\n", encoding="utf-8")
|
|
commit(repo)
|
|
assert hits(run(repo, "--all").stdout) == {("docs/case.md", 1)}
|
|
|
|
|
|
def test_MUTATION_neutering_the_pattern_makes_a_known_hit_go_quiet(repo: Path, tmp_path: Path) -> None:
|
|
"""The clause-level disarm: with PATTERNS unable to match, a file that IS flagged stops being
|
|
flagged. Without this, every assertion above is satisfied by a detector that finds nothing."""
|
|
(repo / "docs" / "d.md").write_text(f"x\n{NARRATIVE}\n", encoding="utf-8")
|
|
commit(repo)
|
|
assert hits(run(repo, "--all").stdout) == truth(repo, "docs/d.md")
|
|
|
|
disarmed = tmp_path / "disarmed.py"
|
|
src = SCRIPT.read_text(encoding="utf-8")
|
|
marker = "PATTERNS = re.compile("
|
|
assert src.count(marker) == 1
|
|
disarmed.write_text(
|
|
src.replace(marker, 'PATTERNS = re.compile(r"(?!x)x" # disarmed\n or ', 1), encoding="utf-8"
|
|
)
|
|
p = subprocess.run([sys.executable, str(disarmed), "--all"], cwd=repo, capture_output=True, text=True)
|
|
assert p.returncode == 0, p.stderr
|
|
assert hits(p.stdout) == set(), "the disarmed detector still flagged something — the mutation did not take"
|