feat(610): migration script + field-level equivalence harness
The migration is a MOVE, so correctness is provable rather than reviewable: parse the corpus before, migrate, parse after, assert the Record sets are identical field-for-field with byte-identical rationale. scripts/migrate_decisions_split.py 166 records -> docs/decisions/records/<area>/<topic>.md (active) or docs/decisions/archive/<area>/<topic>.md (superseded/retired), 26 directories. Path is DERIVED from the key, so identity stays the key. Refuses to run if any record lacks a key, and aborts on a destination collision. Source files are KEPT, stripped to their narrative -- not deleted. 506 lines of the corpus sit outside any record: decisions.md's lifecycle-schema header (the status vocabulary, supersession rules, the edit-token contract) and each topic file's preamble explaining why those records exist. For most of it that is the only copy. It also cannot be filed per-area -- topic files hold several areas and 4 of 23 areas span several files -- so the files themselves stay. scripts/tests/test_migration_equivalence.py Runs the real migration against a COPY of the real corpus in tmp_path, never the working tree. Asserts: all 166 records survive with the same keys, every metadata field round-trips, titles round-trip from headings, every rationale body is byte-identical, path matches key, archived records land in the archive wing, the legacy files keep their narrative, and no parseable record is left behind in them. Proven non-vacuous: corrupting one migrated record's prose is caught by the byte-identical check, and deleting one is caught by the survival check. One test-authoring note: an early assertion string-matched "## " to prove no records were left in decisions.md. That is wrong -- the schema header quotes an illustrative "## 2026-07-17 ..." example in prose. Whether records remain is a PARSING question, so the parser-based leftover test is the real invariant.
This commit is contained in:
@@ -0,0 +1,163 @@
|
||||
#!/usr/bin/env python3
|
||||
"""One-shot migration (ersatztv#610): split the decision corpus into one file per record.
|
||||
|
||||
docs/decisions.md -> docs/decisions/records/<area>/<topic>.md (one per record)
|
||||
docs/decisions/<topic-file>.md + the source file KEPT, stripped to its narrative
|
||||
docs/decisions/archive/*.md -> docs/decisions/archive/<area>/<topic>.md
|
||||
|
||||
Run from the repo root: PYTHONPATH=. python3 scripts/migrate_decisions_split.py [--dry-run]
|
||||
|
||||
Two properties this script exists to guarantee, both checked by
|
||||
`scripts/tests/test_migration_equivalence.py`:
|
||||
|
||||
1. **Nothing is rewritten.** Every record's rationale body is copied byte-for-byte, and every
|
||||
metadata field round-trips unchanged. The migration is a MOVE, so a field-level before/after
|
||||
comparison is a complete correctness proof.
|
||||
2. **No prose is lost.** 506 lines of the corpus are NOT inside any record — file preambles that
|
||||
explain why a topic file exists, cross-reference notes, and decisions.md's lifecycle schema
|
||||
header. Those files are kept and stripped to exactly that prose, never deleted. (They cannot be
|
||||
filed per-area: topic files hold several areas, and 4 of 23 areas span several files.)
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
from pathlib import Path
|
||||
|
||||
import yaml
|
||||
|
||||
import scripts.decisions_lib as dl
|
||||
|
||||
# Emitted in this order so every record file reads the same way.
|
||||
_FIELD_ORDER = [
|
||||
("key", "key"),
|
||||
("title", None), # from Record.heading
|
||||
("status", "status"),
|
||||
("since", "since"),
|
||||
("supersedes", "supersedes"),
|
||||
("superseded-by", "superseded_by"),
|
||||
("stale-after", "stale_after"),
|
||||
("rule", "rule"),
|
||||
("signals", "signals"),
|
||||
("mechanics", "mechanics"),
|
||||
("sources", "sources"),
|
||||
]
|
||||
|
||||
|
||||
def record_path(rec: dl.Record) -> Path:
|
||||
"""`records/<area>/<topic>.md` for an active record, `archive/<area>/<topic>.md` otherwise.
|
||||
|
||||
Identity stays the KEY; the path is derived from it, and the validator enforces the two agree.
|
||||
Every key in the corpus is exactly two dotted segments, so this is unambiguous — a key with
|
||||
inner dots would need a rule and there are none.
|
||||
"""
|
||||
area, _, topic = (rec.key or "").partition(".")
|
||||
root = dl.ARCHIVE_DIR if rec.status in ("superseded", "retired") else dl.RECORDS_DIR
|
||||
return root / area / f"{topic}.md"
|
||||
|
||||
|
||||
def render_record(rec: dl.Record) -> str:
|
||||
"""YAML frontmatter + the rationale body, copied verbatim."""
|
||||
meta: dict[str, object] = {}
|
||||
for fm_key, field in _FIELD_ORDER:
|
||||
val = rec.heading if fm_key == "title" else getattr(rec, field)
|
||||
if val is None:
|
||||
continue # absent stays absent; "" is emitted so empty-vs-absent survives
|
||||
meta[fm_key] = val
|
||||
fm = yaml.safe_dump(
|
||||
meta,
|
||||
sort_keys=False,
|
||||
allow_unicode=True,
|
||||
default_flow_style=False,
|
||||
width=10**9, # never fold: keep every scalar on one line so diffs stay readable
|
||||
)
|
||||
return f"---\n{fm}---\n\n{_rationale_body(rec)}\n"
|
||||
|
||||
|
||||
def _rationale_body(rec: dl.Record) -> str:
|
||||
"""The record body with its contiguous metadata block stripped — the prose, byte-for-byte.
|
||||
|
||||
Mirrors decisions_validate._rationale's contiguous-block rule, but preserves interior blank
|
||||
lines and indentation (that function normalizes whitespace for comparison; here the bytes are
|
||||
the payload).
|
||||
"""
|
||||
lines = rec.body.splitlines()
|
||||
i = 0
|
||||
while i < len(lines) and not lines[i].strip():
|
||||
i += 1
|
||||
if i < len(lines) and lines[i].strip().startswith("`key:"):
|
||||
j = i + 1
|
||||
while j < len(lines) and lines[j].strip():
|
||||
j += 1
|
||||
lines = lines[:i] + lines[j:]
|
||||
return "\n".join(lines).strip()
|
||||
|
||||
|
||||
def _preamble(path: Path, records: list[dl.Record]) -> str:
|
||||
"""Everything in `path` that is not inside a record — kept, never deleted."""
|
||||
text = path.read_text(encoding="utf-8")
|
||||
lines = text.splitlines()
|
||||
drop: set[int] = set()
|
||||
for r in records:
|
||||
start = r.lineno - 1 # heading line, 1-indexed in Record
|
||||
for n in range(start, start + 1 + len(r.body.splitlines())):
|
||||
drop.add(n)
|
||||
kept = [ln for n, ln in enumerate(lines) if n not in drop]
|
||||
while kept and not kept[-1].strip():
|
||||
kept.pop()
|
||||
return "\n".join(kept)
|
||||
|
||||
|
||||
def plan() -> tuple[list[tuple[dl.Record, Path]], dict[Path, str]]:
|
||||
"""(record -> destination) pairs, and (source file -> preamble to keep)."""
|
||||
sources = [p for p in dl.active_files() if p.exists()]
|
||||
sources += sorted(p for p in dl.ARCHIVE_DIR.glob("*.md") if p.name not in dl._NON_DECISION_FILES)
|
||||
moves: list[tuple[dl.Record, Path]] = []
|
||||
preambles: dict[Path, str] = {}
|
||||
for src in sources:
|
||||
if dl.RECORDS_DIR in src.parents or dl.has_frontmatter(src.read_text(encoding="utf-8")):
|
||||
continue # already migrated
|
||||
recs = [r for r in dl.parse_file(src) if r.heading not in ("Index", "Active catalog", "Contents")]
|
||||
keyed = [r for r in recs if r.key]
|
||||
if len(keyed) != len(recs):
|
||||
unkeyed = [r.heading for r in recs if not r.key]
|
||||
raise SystemExit(f"{src}: {len(unkeyed)} record(s) without a key, cannot file by key: {unkeyed}")
|
||||
moves += [(r, record_path(r)) for r in keyed]
|
||||
preambles[src] = _preamble(src, recs)
|
||||
return moves, preambles
|
||||
|
||||
|
||||
def main(argv=None) -> int:
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--dry-run", action="store_true", help="report the plan, write nothing")
|
||||
args = ap.parse_args(argv)
|
||||
|
||||
moves, preambles = plan()
|
||||
|
||||
dests = [d for _, d in moves]
|
||||
if len(set(dests)) != len(dests):
|
||||
dupes = sorted({str(d) for d in dests if dests.count(d) > 1})
|
||||
raise SystemExit(f"destination collision — two records map to the same path: {dupes}")
|
||||
|
||||
print(f"{len(moves)} records -> {len({d.parent for d in dests})} directories")
|
||||
print(
|
||||
f"{len(preambles)} source files kept, stripped to narrative "
|
||||
f"({sum(len(v.splitlines()) for v in preambles.values())} lines preserved)"
|
||||
)
|
||||
if args.dry_run:
|
||||
for rec, dest in sorted(moves, key=lambda m: str(m[1]))[:5]:
|
||||
print(f" {rec.key} -> {dest.relative_to(dl.REPO_ROOT)}")
|
||||
print(" …")
|
||||
return 0
|
||||
|
||||
for rec, dest in moves:
|
||||
dest.parent.mkdir(parents=True, exist_ok=True)
|
||||
dest.write_text(render_record(rec), encoding="utf-8")
|
||||
for src, text in preambles.items():
|
||||
src.write_text(text.rstrip("\n") + "\n", encoding="utf-8")
|
||||
print("migration written")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Reference in New Issue
Block a user