ersatztv/scripts/migrate_decisions_split.py

#!/usr/bin/env python3
"""One-shot migration (ersatztv#610): split the decision corpus into one file per record.

    docs/decisions.md                  ->  docs/decisions/records/<area>/<topic>.md   (one per record)
    docs/decisions/<topic-file>.md         + the source file KEPT, stripped to its narrative
    docs/decisions/archive/*.md        ->  docs/decisions/archive/<area>/<topic>.md

Run from the repo root:  PYTHONPATH=. python3 scripts/migrate_decisions_split.py [--dry-run]

Two properties this script exists to guarantee, both checked by
`scripts/tests/test_migration_equivalence.py`:

1. **Nothing is rewritten.** Every record's rationale body is copied byte-for-byte, and every
   metadata field round-trips unchanged. The migration is a MOVE, so a field-level before/after
   comparison is a complete correctness proof.
2. **No prose is lost.** A few hundred lines of the corpus are NOT inside any record — file
   preambles that explain why a topic file exists, cross-reference notes, and decisions.md's
   lifecycle schema header. Those files are kept and stripped to exactly that prose, never deleted.
   (They cannot be filed per-area: topic files hold several areas, and 4 of 23 areas span several
   files.) The one thing deliberately dropped is same-file anchor bullets, which the split makes
   *wrong* rather than merely stale — the generated index replaces them with links that resolve.
   Run with `--dry-run` for the current counts rather than trusting a number written here.
"""

from __future__ import annotations

import argparse
import os
import re
from pathlib import Path

import yaml

import scripts.decisions_lib as dl

# Emitted in this order so every record file reads the same way.
_FIELD_ORDER = [
    ("key", "key"),
    ("title", None),  # from Record.heading
    ("status", "status"),
    ("since", "since"),
    ("supersedes", "supersedes"),
    ("superseded-by", "superseded_by"),
    ("stale-after", "stale_after"),
    ("rule", "rule"),
    ("signals", "signals"),
    ("mechanics", "mechanics"),
    ("sources", "sources"),
]


def record_path(rec: dl.Record) -> Path:
    """`records/<area>/<topic>.md` for an active record, `archive/<area>/<topic>.md` otherwise.

    Identity stays the KEY; the path is derived from it, and the validator enforces the two agree.
    Every key in the corpus is exactly two dotted segments, so this is unambiguous — a key with
    inner dots would need a rule and there are none.
    """
    area, _, topic = (rec.key or "").partition(".")
    root = dl.ARCHIVE_DIR if rec.status in ("superseded", "retired") else dl.RECORDS_DIR
    return root / area / f"{topic}.md"


def render_record(rec: dl.Record) -> str:
    """YAML frontmatter + the rationale body, copied verbatim."""
    meta: dict[str, object] = {}
    for fm_key, field in _FIELD_ORDER:
        val = rec.heading if fm_key == "title" else getattr(rec, field)
        if val is None:
            continue  # absent stays absent; "" is emitted so empty-vs-absent survives
        meta[fm_key] = val
    fm = yaml.safe_dump(
        meta,
        sort_keys=False,
        allow_unicode=True,
        default_flow_style=False,
        width=10**9,  # never fold: keep every scalar on one line so diffs stay readable
    )
    return f"---\n{fm}---\n\n{_rationale_body(rec)}\n"


def _rationale_body(rec: dl.Record) -> str:
    """The record body with its contiguous metadata block stripped — the prose, byte-for-byte.

    Mirrors decisions_validate._rationale's contiguous-block rule, but preserves interior blank
    lines and indentation (that function normalizes whitespace for comparison; here the bytes are
    the payload).
    """
    lines = rec.body.splitlines()
    i = 0
    while i < len(lines) and not lines[i].strip():
        i += 1
    if i < len(lines) and lines[i].strip().startswith("`key:"):
        j = i + 1
        while j < len(lines) and lines[j].strip():
            j += 1
        lines = lines[:i] + lines[j:]
    return "\n".join(lines).strip()


_MOVED_HEADING = "## Records formerly in this file"


def _moved_index(records: list[dl.Record], src: Path) -> str:
    """A generated 'where did it go' index appended to each stripped file.

    Live docs point into the corpus BY DATE — "see `decisions.md` 2026-07-10" — roughly 32 such
    references across 12 files. The split would dangle every one of them. This index keeps them
    resolvable: the reader lands on the file the pointer names and finds the date, the title, and
    the record's new path. Cheaper and far less error-prone than rewriting 32 references by hand,
    and it degrades gracefully for the historical issue comments that use the same form.
    """
    if not records:
        return ""
    lines = [
        _MOVED_HEADING,
        "",
        "Each record below moved to its own file under `records/` (ersatztv#610); the rationale is",
        "unchanged. Resolve by **key** — that is the stable identity. A date-based pointer from",
        "another doc or an old issue comment should land here and then follow the link.",
        "",
    ]
    for r in sorted(records, key=lambda x: (x.heading, x.key or "")):
        # Relative to the SOURCE FILE'S OWN PARENT, not to TOPIC_DIR. The stripped files sit at
        # three different depths — docs/decisions.md in `docs/`, the topic files in
        # `docs/decisions/`, the archive files in `docs/decisions/archive/` — so a single fixed
        # base produces links that resolve correctly from exactly one of them and dangle from the
        # other two. That is how 105 links, including every one in the corpus's primary entry
        # file, pointed at nonexistent paths.
        dest = os.path.relpath(record_path(r), src.parent)
        lines.append(f"- {r.heading} — [`{r.key}`]({dest})")
    return "\n".join(lines)


# A bullet whose link is a bare same-file anchor, e.g. `- [2026-07-10 — Title](#2026-07-10--title)`.
# decisions.md's "In this file:" index is ~97 of these. Once the records move out, every one of them
# points at a heading that no longer exists in that file — and the generated "Records formerly in
# this file" section replaces them with links that actually resolve. Anything else keeps its bullet.
_SAME_FILE_ANCHOR_BULLET = re.compile(r"^\s*[-*]\s*\[[^\]]*\]\(#[^)]*\)\s*$")


def _preamble(path: Path, records: list[dl.Record]) -> str:
    """Everything in `path` that is not inside a record — kept, never deleted.

    Two exceptions are dropped rather than kept, because the split makes them *wrong* rather than
    merely stale: same-file anchor bullets (see above).
    """
    text = path.read_text(encoding="utf-8")
    lines = text.splitlines()
    drop: set[int] = set()
    for r in records:
        start = r.lineno - 1  # heading line, 1-indexed in Record
        for n in range(start, start + 1 + len(r.body.splitlines())):
            drop.add(n)
    kept = [ln for n, ln in enumerate(lines) if n not in drop and not _SAME_FILE_ANCHOR_BULLET.match(ln)]
    # collapse the blank runs left behind where a bullet list used to be
    out: list[str] = []
    for ln in kept:
        if not ln.strip() and out and not out[-1].strip():
            continue
        out.append(ln)
    while out and not out[-1].strip():
        out.pop()
    return "\n".join(out)


def plan() -> tuple[list[tuple[dl.Record, Path]], dict[Path, str], dict[Path, int]]:
    """(record -> destination) pairs, and (source file -> preamble to keep)."""
    sources = [p for p in dl.active_files() if p.exists()]
    sources += sorted(p for p in dl.ARCHIVE_DIR.glob("*.md") if p.name not in dl._NON_DECISION_FILES)
    moves: list[tuple[dl.Record, Path]] = []
    preambles: dict[Path, str] = {}
    kept_prose: dict[Path, int] = {}  # ORIGINAL prose only, excluding the generated index
    for src in sources:
        if dl.RECORDS_DIR in src.parents or dl.has_frontmatter(src.read_text(encoding="utf-8")):
            continue  # already migrated
        recs = [r for r in dl.parse_file(src) if r.heading not in dl.SKIP_HEADINGS]
        keyed = [r for r in recs if r.key]
        if len(keyed) != len(recs):
            unkeyed = [r.heading for r in recs if not r.key]
            raise SystemExit(f"{src}: {len(unkeyed)} record(s) without a key, cannot file by key: {unkeyed}")
        moves += [(r, record_path(r)) for r in keyed]
        body = _preamble(src, recs)
        idx = _moved_index(keyed, src)
        kept_prose[src] = len(body.splitlines())
        preambles[src] = f"{body.rstrip()}\n\n{idx}" if idx else body
    return moves, preambles, kept_prose


def main(argv=None) -> int:
    ap = argparse.ArgumentParser()
    ap.add_argument("--dry-run", action="store_true", help="report the plan, write nothing")
    args = ap.parse_args(argv)

    moves, preambles, kept_prose = plan()

    dests = [d for _, d in moves]
    if len(set(dests)) != len(dests):
        dupes = sorted({str(d) for d in dests if dests.count(d) > 1})
        raise SystemExit(f"destination collision — two records map to the same path: {dupes}")

    print(f"{len(moves)} records -> {len({d.parent for d in dests})} directories")
    idx_lines = sum(len(v.splitlines()) for v in preambles.values()) - sum(kept_prose.values())
    print(
        f"{len(preambles)} source files kept: {sum(kept_prose.values())} lines of original prose "
        f"preserved, plus {idx_lines} generated index lines"
    )
    if args.dry_run:
        for rec, dest in sorted(moves, key=lambda m: str(m[1]))[:5]:
            print(f"   {rec.key}  ->  {dest.relative_to(dl.REPO_ROOT)}")
        print("   …")
        return 0

    for rec, dest in moves:
        dest.parent.mkdir(parents=True, exist_ok=True)
        dest.write_text(render_record(rec), encoding="utf-8")
    for src, text in preambles.items():
        src.write_text(text.rstrip("\n") + "\n", encoding="utf-8")
    print("migration written")
    return 0


if __name__ == "__main__":
    raise SystemExit(main())