ersatztv/scripts/check-doc-narrative.py

#!/usr/bin/env python3
"""ersatztv#784 — ADVISORY nudge for `docs.no-session-narrative`.

A doc records the END STATE; the path to it belongs in the commit message, not the artifact.

THIS NEVER FAILS. Every path returns exit 0 — including a bad argument, an unresolvable base ref,
an unreadable file and an unhandled exception. That is a design constraint, not an oversight: a
narrative detector is a string predicate over prose, and `docs/defect-shapes-773.md` §4 plus
`testing.guard-derives-population-from-source` both argue that class must not be load-bearing (the
withdrawn `test_review_verdict_vocabulary_parity.py` — six review rounds, then deleted — is the
empirical case). Do not convert this into a gate; the decision record says no in as many words.

WHY PYTHON AND NOT SHELL. Hand-parsing `git diff -U0` output in bash by matching line prefixes
produced four separate defects in that one parser: the
`\\ No newline at end of file` marker was counted as content, an added line whose own text began
`++ ` was eaten by the `+++ ` header arm, `core.quotePath` hid non-ASCII paths, and `read` dropped a
final unterminated line. Those are four instances of one mistake — deciding what a diff line IS from
its prefix alone, with no hunk state. Patching them one at a time is the shape this repo has
recorded as never converging, so the mechanism was replaced rather than the sites.

  check-doc-narrative.py --diff <base-ref>   scan lines ADDED against <base-ref> (the CI mode)
  check-doc-narrative.py --all               scan the whole tracked corpus (deliberate sweep)

Scanning only ADDED lines in CI is what keeps the existing corpus of legitimate history out of the
output; `--all` deliberately reports all of it, for a human to apply the who-benefits test to.
"""

from __future__ import annotations

import os
import re
import subprocess
import sys

# The never-fails invariant must not depend on the ambient locale. Both the summary line and the
# last-resort handler below carry non-ASCII text, so under ascii/latin-1 stdio the very code meant to
# guarantee exit 0 is what raises. Degrade unencodable characters instead of failing on them.
try:
    sys.stdout.reconfigure(errors="replace")  # type: ignore[union-attr]
    sys.stderr.reconfigure(errors="replace")  # type: ignore[union-attr]
except Exception:  # noqa: S110 — a stdout that cannot be reconfigured is not a reason to fail
    pass  # deliberate: this is the never-fails invariant's own setup, so it cannot itself raise

# `docs/decisions/**` is exempt WHOLESALE — a decision record narrating how a rule was got wrong is
# carrying the rationale it exists to carry, so a detector that flagged it would fight the
# convention it serves.
#
# `docs/superpowers/**` was CONSIDERED for this list in #812 and deliberately REJECTED. Recorded here
# so it is not re-proposed on plausibility: the argument was that those 35 dated plans are frozen
# (measured 2026-08-29, before this change: nothing edited there since 2026-07-23), so a sweep over
# them yields only carve-out false positives. But
# `--diff` — the mode CI runs — scans ADDED lines, and a frozen file contributes none, so exempting
# them buys nothing there. What it WOULD suppress is any plan being WRITTEN OR REVISED now — 15 of
# the 35 have more than one commit, so revisions do happen — which is precisely the case where the
# rule's remedy (move it to the commit message) is still available. The exemption costs the only
# reach the detector has and buys only quiet in a sweep a person runs deliberately.
#
# `scripts/tests/fixtures/` is exempt for the same reason one level down (#876): it holds COPIES of
# decision records and other test DATA, which carry whatever phrasing the test under them needs and
# are not an artifact anyone edits for a reader. Exempting it is also what keeps the depth
# measurement in the record true — its record copies sit at four to six slashes.
EXEMPT_PREFIXES = ("docs/decisions/", "scripts/tests/fixtures/")

# The PROCESS corpus (#876): hooks, workflows, scripts and their tests, scanned regardless of
# extension because the artifacts there are shell, YAML, Python and jq, not Markdown. It is in the
# population on the #812 argument run forward — `--diff` sees only ADDED lines, and this is where
# narrative is being ADDED: 287 of the 453 sites the #876 sweep found outside the docs corpus were
# under 30 days old (measured 2026-09-03 at `fb5592971`), against a `docs/superpowers/**` that had
# not moved since 2026-07-23. `web/` and the C# projects are left out on the same measurement: 3 of
# the 74 PATTERNS-matching sites lived there, across roughly 4,600 tracked files.
PROCESS_PREFIXES = (".claude/", ".gitea/", ".husky/", "scripts/")

# Exempt by NAME, not by prefix: this file and its test carry the phrasings as pattern and fixture,
# so they would be permanent hits — the false positive that makes an advisory check stop being read.
EXEMPT_FILES = ("scripts/check-doc-narrative.py", "scripts/tests/test_check_doc_narrative.py")

# Session-narrative phrasings. Deliberately narrow: each is first person or names a revision of THIS
# artifact. Broad words that also appear in legitimate dated history ("previously", "was wrong") are
# absent on purpose — a false positive on a carved-out case is what makes an advisory check stop
# being read.
PATTERNS = re.compile(
    r"an earlier draft"
    r"|earlier drafts"
    r"|the (?:first|previous|original) (?:version|draft) of (?:this|the)"
    r"|my first attempt"
    r"|I (?:initially|first|originally|then) (?:thought|assumed|tried|wrote|found)"
    r"|we (?:then|initially) (?:found|thought|realis|realiz)"
    r"|it turned out that"
    r"|earlier today"
    r"|as of just now"
    r"|currently investigating",
    re.IGNORECASE,
)

WARNING = (
    "{path}:{line} reads as session narrative — a reader coming cold never saw the earlier draft. "
    "Answer the review finding in the COMMIT MESSAGE and let only the corrected claim enter the "
    "doc (docs.no-session-narrative). Keep it only if a reader would ACT differently knowing it "
    "(dated measurement, stated snapshot boundary, tested-and-rejected result, a trap and its "
    "consequence). Line: {text}"
)


def is_scanned_path(path: str) -> bool:
    """The population: `docs/**/*.md` minus `docs/decisions/**`, plus root-level `*.md`, plus every
    tracked file under `PROCESS_PREFIXES` regardless of extension, minus `EXEMPT_FILES`.

    Stated positively and in one place so the record's `mechanics:` can quote it exactly. `web/`,
    C# source, and nested markdown outside `docs/` and the process prefixes are deliberately NOT in
    scope. This is a PATH predicate; `run_all` additionally restricts the population to tracked
    regular files, because a symlink's content is its target, not the artifact.
    """
    if path in EXEMPT_FILES or any(path.startswith(p) for p in EXEMPT_PREFIXES):
        return False
    if any(path.startswith(p) for p in PROCESS_PREFIXES):
        return True
    if not path.endswith(".md"):
        return False
    return path.startswith("docs/") or "/" not in path


# Git's OUTPUT FORMAT is configurable, and this script reads paths and line numbers out of that
# format. Three separate knobs were each demonstrated turning a real hit into `scanned 0 file(s)` —
# `core.quotePath` hiding non-ASCII paths, `diff.dstPrefix` rewriting the header, `color.diff=always`
# injecting ANSI escapes. Pinning them one at a time is refuting variants, not clearing the channel,
# so the channel is closed at both ends: the user's and the system's config files are taken out of
# the picture entirely (which also covers knobs nobody has thought of yet), and the handful that a
# REPO-local config could still set are pinned explicitly on the command line, where they win.
GIT_ENV_OVERRIDES = {
    "GIT_CONFIG_GLOBAL": os.devnull,
    "GIT_CONFIG_SYSTEM": os.devnull,
    "GIT_CONFIG_NOSYSTEM": "1",
}
# `core.quotePath=false` is witnessed by a test. The env overrides above are NOT, and cannot be: they
# exist for the knob nobody has named yet, which is exactly what three rounds of naming one knob at a
# time argued for. Every knob that IS named is pinned on the command line, where it also beats a
# repo-local config, and has a row in FORMAT_KNOBS in the test file.
GIT_CONFIG_PINS = ("-c", "core.quotePath=false")


def git(*args: str) -> tuple[int, str]:
    """Run git with its output format pinned. Returns (returncode, stdout) and never raises."""
    try:
        p = subprocess.run(
            ["git", *GIT_CONFIG_PINS, *args],
            capture_output=True,
            text=True,
            errors="replace",
            env={**os.environ, **GIT_ENV_OVERRIDES},
        )
    except OSError as exc:  # git missing, or not a repo we can exec in
        return 1, f"{exc}"
    if p.returncode != 0:
        sys.stderr.write(p.stderr)
    return p.returncode, p.stdout


def scan_line(path: str, lineno: int, text: str, out: list[str]) -> None:
    if PATTERNS.search(text):
        detail = WARNING.format(path=path, line=lineno, text=text[:160])
        out.append(f"::warning file={path}::{detail}")


def added_lines(diff: str):
    """Yield (path, lineno, text) for every ADDED line in a unified diff.

    A line's meaning comes from HUNK STATE, not from its prefix: `+++ ` is a header only before the
    first `@@` of a file, and inside a hunk it is content whose own text starts `++ `. That
    distinction is the whole reason this is not a prefix match.
    """
    path = None
    in_hunk = False
    lineno = 0
    for raw in diff.split("\n"):
        if raw.startswith("diff --git "):
            path, in_hunk = None, False
        elif raw.startswith("@@"):
            m = re.match(r"@@ -\d+(?:,\d+)? \+(\d+)(?:,\d+)? @@", raw)
            if not m:
                in_hunk = False
                continue
            in_hunk = True
            lineno = int(m.group(1)) - 1
        elif not in_hunk:
            if raw.startswith("+++ "):
                p = raw[4:]
                # `/dev/null` on the new side means the file was DELETED. Stated honestly: this arm
                # is DEFENSIVE, not load-bearing — a deletion contributes no `+` lines, so nothing is
                # yielded for it either way. Measured rather than asserted, and re-measured on every
                # run: `test_check_doc_narrative.py` stays green with this arm removed, a declared
                # claim in `scripts/tests/mutation_manifest.py` (ersatztv#881). The named file is the
                # scope that is actually executed — a claim over every test in the repository would
                # be wider than anything re-taking it. The arm is kept because `path` should never
                # name a file the added lines do not belong to. A
                # `--diff-filter=d` on the git call was removed rather than kept beside it: a second
                # mechanism nobody can witness failing is how a duplicate guard hides its twin.
                # Git terminates the `+++` filename with a TAB when the path contains a space, and a
                # path carrying that tab matches nothing in `is_scanned_path` — the file is dropped
                # SILENTLY, the same scanned-0 channel the config pins above close. Witnessed by a
                # positive control with a space in the name.
                p = p.split("\t", 1)[0]
                path = None if p == "/dev/null" else (p[2:] if p.startswith("b/") else p)
        elif raw.startswith("+"):
            lineno += 1
            if path is not None:
                yield path, lineno, raw[1:]
        elif raw.startswith("-") or raw.startswith("\\"):
            pass  # a removed line, or the no-trailing-newline marker: neither advances the new file
        else:
            lineno += 1  # context (absent at -U0, but harmless and correct if -U grows)


def run_diff(base: str, out: list[str]) -> int:
    rc, diff = git(
        "diff",
        "-U0",
        # Pinned, not decorative: `diff.renames=false` in a developer's gitconfig turns a `git mv`
        # into a whole-file add and re-flags every pre-existing line. Same channel as the prefixes.
        "--find-renames",
        # Pin the header shape the path is parsed out of. `diff.noprefix`, `diff.srcPrefix` and
        # `diff.dstPrefix` each rewrite it from a developer's gitconfig, and `diff.external` replaces
        # the output entirely — a configured prefix silently produced a scanned-0-files clean run.
        "--src-prefix=a/",
        "--dst-prefix=b/",
        "--no-ext-diff",
        "--no-color",
        f"{base}...HEAD",
    )
    if rc != 0:
        print(
            f"doc-narrative: could not diff against '{base}' — SCANNED NOTHING. "
            "This is reported rather than swallowed: a silent zero-file scan is indistinguishable "
            "from a clean one, which is the failure `ci.required-job-step-execution-markers` exists for."
        )
        return -1
    scanned = set()
    for path, lineno, text in added_lines(diff):
        if not is_scanned_path(path):
            continue
        scanned.add(path)
        scan_line(path, lineno, text, out)
    return len(scanned)


def run_all(out: list[str]) -> int:
    # Population from `git ls-files`, never a filesystem walk — an untracked scratch file is not
    # part of the corpus (#778). The STAGE listing, because the population is tracked REGULAR files
    # (mode 100644/100755): a tracked symlink (120000) is a path the predicate admits whose worktree
    # content is its TARGET — a directory (`.claude/skills/jellyfin`, a cross-repo skill link) or a
    # file that may sit outside the population — and a gitlink (160000) is a submodule pointer with
    # no content of its own. `--diff` carries no such filter: a gitlink's diff content is a sha, and
    # a symlink's is its target PATH, scanned like any other added line — a phrasing inside a path
    # name would draw an advisory warning on the link, which is harmless. The stage listing emits
    # one row PER STAGE for an unmerged path, so modes are gathered per path first: a path is opened
    # once, and only when EVERY stage is a regular blob — in an add/add conflict between a symlink
    # and a regular file, the worktree holds one of them and a per-row test would let the regular
    # row authorise opening the other.
    rc, listing = git("ls-files", "-s", "-z")
    if rc != 0:
        print("doc-narrative: could not list tracked files — SCANNED NOTHING.")
        return -1
    modes: dict[str, set[str]] = {}
    for entry in listing.split("\0"):
        if not entry:
            continue
        meta, _, path = entry.partition("\t")
        modes.setdefault(path, set()).add(meta.split(" ", 1)[0])
    scanned = 0
    for path, path_modes in modes.items():
        if not all(m.startswith("100") for m in path_modes) or not is_scanned_path(path):
            continue
        try:
            with open(path, encoding="utf-8", errors="replace") as fh:
                text = fh.read()
        except OSError as exc:
            # A tracked-but-deleted doc is an ordinary working state, not a reason to fail.
            print(f"doc-narrative: skipped {path} ({exc.strerror}).")
            continue
        scanned += 1
        # splitlines() keeps a final unterminated line, which `read`-per-line dropped.
        for i, line in enumerate(text.splitlines(), start=1):
            scan_line(path, i, line, out)
    return scanned


def main(argv: list[str]) -> int:
    mode = argv[1] if len(argv) > 1 else "--all"
    out: list[str] = []

    if mode == "--diff":
        if len(argv) < 3 or not argv[2]:
            print("doc-narrative: --diff needs a base ref — SCANNED NOTHING. (advisory; not a failure)")
            return 0
        scanned = run_diff(argv[2], out)
    elif mode == "--all":
        scanned = run_all(out)
    else:
        print(f"doc-narrative: unknown mode '{mode}'. usage: {argv[0]} [--all | --diff <base-ref>]")
        return 0

    for line in out:
        print(line)
    if scanned < 0:
        return 0
    print(
        f"doc-narrative: scanned {scanned} file(s); {len(out)} advisory warning(s). "
        "NON-BLOCKING — this check never fails a run."
    )
    return 0


if __name__ == "__main__":
    try:
        sys.exit(main(sys.argv))
    # The never-fails constraint outranks a clean traceback, so this catch is deliberately blind.
    except Exception as exc:
        print(f"doc-narrative: internal error ({exc!r}) — SCANNED NOTHING. Advisory; not a failure.")
        sys.exit(0)