From e063fd065fd2ad72d965179a5e80590739c03ad5 Mon Sep 17 00:00:00 2001 From: Timothy Date: Sat, 25 Jul 2026 17:31:58 +0200 Subject: [PATCH 01/10] feat(610): dual-format decision-record parser (backtick line + YAML frontmatter) First step of the one-file-per-record split. `decisions_lib.parse_text` now dispatches on a leading `---`: the legacy "many ## records per file, metadata on a backtick line" form and the new "one record per file, YAML frontmatter" form both produce the same Record. This is the load-bearing move for the migration. Because the parser understands both, the body-diff guard can parse the OLD form at the merge-base and the NEW form at head -- both keyed on `key` -- so the commit that relocates 166 records validates itself with no one-time bypass and no escape hatch. Details: - Frontmatter uses the SAME field vocabulary as the backtick line (`superseded-by`, `stale-after`), so on-disk names don't shift mid-migration. - YAML `None` (a key with no value) is preserved as "" rather than collapsing to None, keeping the empty-vs-absent distinction `stale-after` depends on (#603). - `active_files()` also walks `docs/decisions/records/**`, so both layouts can coexist while the migration lands. - Malformed frontmatter (unterminated fence, non-dict, YAML error) yields no record rather than raising; the validator then reports it as a missing record. --- scripts/decisions_lib.py | 74 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 74 insertions(+) diff --git a/scripts/decisions_lib.py b/scripts/decisions_lib.py index 6b5bb0124..023dcf937 100644 --- a/scripts/decisions_lib.py +++ b/scripts/decisions_lib.py @@ -28,6 +28,7 @@ REPO_ROOT = Path(__file__).resolve().parent.parent DECISIONS_MD = REPO_ROOT / "docs" / "decisions.md" TOPIC_DIR = REPO_ROOT / "docs" / "decisions" ARCHIVE_DIR = TOPIC_DIR / "archive" +RECORDS_DIR = TOPIC_DIR / "records" # split layout (#610): one YAML-frontmatter record per file STATUSES = {"active", "superseded", "retired", "legacy-unmigrated"} KEY_RE = re.compile(r"^[a-z0-9]+(\.[a-z0-9-]+)+$") @@ -60,6 +61,8 @@ def _parse_meta_line(line: str) -> dict[str, str]: def parse_text(text: str, source: Path) -> list[Record]: + if has_frontmatter(text): + return _parse_frontmatter(text, source) lines = text.splitlines() records: list[Record] = [] i = 0 @@ -109,6 +112,70 @@ def parse_text(text: str, source: Path) -> list[Record]: return records +# --- YAML-frontmatter form (ersatztv#610): one record per file ------------------------------ +# +# The corpus is migrating from "many `##` records per file, metadata on a backtick line" to "one +# record per file, metadata in YAML frontmatter". BOTH forms parse here, dispatching on a leading +# `---`. That is what lets the migration validate itself: the body-diff guard parses the OLD form +# at the merge-base and the NEW form at head, both yielding Records keyed on `key`, so no one-time +# bypass is needed for the commit that moves 166 records. +# +# Frontmatter keys are the SAME vocabulary as the backtick line (`superseded-by`, `stale-after`), +# so the on-disk names don't shift under anyone mid-migration. + +_FM_DELIM = "---" +_FM_TO_FIELD = { + "key": "key", + "status": "status", + "since": "since", + "supersedes": "supersedes", + "superseded-by": "superseded_by", + "stale-after": "stale_after", + "rule": "rule", + "signals": "signals", + "mechanics": "mechanics", + "sources": "sources", +} + + +def has_frontmatter(text: str) -> bool: + """True if `text` opens with a YAML frontmatter fence. Must be the very first line.""" + lines = text.splitlines() + return bool(lines) and lines[0].rstrip() == _FM_DELIM + + +def _parse_frontmatter(text: str, source: Path) -> list[Record]: + import yaml # local import: only the new form needs it + + lines = text.splitlines() + end = None + for i, ln in enumerate(lines[1:], start=1): + if ln.rstrip() == _FM_DELIM: + end = i + break + if end is None: + return [] # unterminated frontmatter — malformed; validator reports the missing record + try: + meta = yaml.safe_load("\n".join(lines[1:end])) or {} + except yaml.YAMLError: + return [] + if not isinstance(meta, dict): + return [] + + rec = Record(heading=str(meta.get("title") or "").strip(), source=source, lineno=1) + for fm_key, field in _FM_TO_FIELD.items(): + if fm_key not in meta: + continue + val = meta[fm_key] + # `None` is what YAML gives for `key:` with no value — the empty-vs-absent distinction + # that `stale-after` depends on (#603). Preserve it as "" rather than collapsing to None. + setattr(rec, field, "" if val is None else str(val).strip()) + if not rec.status: + rec.status = "legacy-unmigrated" + rec.body = "\n".join(lines[end + 1 :]).strip() + return [rec] + + def parse_file(path: Path) -> list[Record]: return parse_text(path.read_text(encoding="utf-8"), path) @@ -128,8 +195,15 @@ _NON_DECISION_FILES = {"README.md", "migration-map.md", "retrieval-eval.md"} def active_files() -> list[Path]: + """Every file holding ACTIVE records, in both the legacy and split layouts. + + Legacy: `docs/decisions.md` + the flat topic files. Split (#610): `docs/decisions/records/**`. + Both are listed so the two layouts can coexist during the migration and so a ref on either side + of it parses correctly. + """ files = [DECISIONS_MD] files += sorted(p for p in TOPIC_DIR.glob("*.md") if p.name not in _NON_DECISION_FILES) + files += sorted(RECORDS_DIR.rglob("*.md")) return files -- 2.47.3 From d52268f2edb49a92fc8714cb952e5a1a4c9074dc Mon Sep 17 00:00:00 2001 From: Timothy Date: Sat, 25 Jul 2026 17:35:46 +0200 Subject: [PATCH 02/10] feat(610): migration script + field-level equivalence harness The migration is a MOVE, so correctness is provable rather than reviewable: parse the corpus before, migrate, parse after, assert the Record sets are identical field-for-field with byte-identical rationale. scripts/migrate_decisions_split.py 166 records -> docs/decisions/records//.md (active) or docs/decisions/archive//.md (superseded/retired), 26 directories. Path is DERIVED from the key, so identity stays the key. Refuses to run if any record lacks a key, and aborts on a destination collision. Source files are KEPT, stripped to their narrative -- not deleted. 506 lines of the corpus sit outside any record: decisions.md's lifecycle-schema header (the status vocabulary, supersession rules, the edit-token contract) and each topic file's preamble explaining why those records exist. For most of it that is the only copy. It also cannot be filed per-area -- topic files hold several areas and 4 of 23 areas span several files -- so the files themselves stay. scripts/tests/test_migration_equivalence.py Runs the real migration against a COPY of the real corpus in tmp_path, never the working tree. Asserts: all 166 records survive with the same keys, every metadata field round-trips, titles round-trip from headings, every rationale body is byte-identical, path matches key, archived records land in the archive wing, the legacy files keep their narrative, and no parseable record is left behind in them. Proven non-vacuous: corrupting one migrated record's prose is caught by the byte-identical check, and deleting one is caught by the survival check. One test-authoring note: an early assertion string-matched "## " to prove no records were left in decisions.md. That is wrong -- the schema header quotes an illustrative "## 2026-07-17 ..." example in prose. Whether records remain is a PARSING question, so the parser-based leftover test is the real invariant. --- scripts/migrate_decisions_split.py | 163 ++++++++++++++++++++ scripts/tests/test_migration_equivalence.py | 156 +++++++++++++++++++ 2 files changed, 319 insertions(+) create mode 100644 scripts/migrate_decisions_split.py create mode 100644 scripts/tests/test_migration_equivalence.py diff --git a/scripts/migrate_decisions_split.py b/scripts/migrate_decisions_split.py new file mode 100644 index 000000000..7699c82a1 --- /dev/null +++ b/scripts/migrate_decisions_split.py @@ -0,0 +1,163 @@ +#!/usr/bin/env python3 +"""One-shot migration (ersatztv#610): split the decision corpus into one file per record. + + docs/decisions.md -> docs/decisions/records//.md (one per record) + docs/decisions/.md + the source file KEPT, stripped to its narrative + docs/decisions/archive/*.md -> docs/decisions/archive//.md + +Run from the repo root: PYTHONPATH=. python3 scripts/migrate_decisions_split.py [--dry-run] + +Two properties this script exists to guarantee, both checked by +`scripts/tests/test_migration_equivalence.py`: + +1. **Nothing is rewritten.** Every record's rationale body is copied byte-for-byte, and every + metadata field round-trips unchanged. The migration is a MOVE, so a field-level before/after + comparison is a complete correctness proof. +2. **No prose is lost.** 506 lines of the corpus are NOT inside any record — file preambles that + explain why a topic file exists, cross-reference notes, and decisions.md's lifecycle schema + header. Those files are kept and stripped to exactly that prose, never deleted. (They cannot be + filed per-area: topic files hold several areas, and 4 of 23 areas span several files.) +""" + +from __future__ import annotations + +import argparse +from pathlib import Path + +import yaml + +import scripts.decisions_lib as dl + +# Emitted in this order so every record file reads the same way. +_FIELD_ORDER = [ + ("key", "key"), + ("title", None), # from Record.heading + ("status", "status"), + ("since", "since"), + ("supersedes", "supersedes"), + ("superseded-by", "superseded_by"), + ("stale-after", "stale_after"), + ("rule", "rule"), + ("signals", "signals"), + ("mechanics", "mechanics"), + ("sources", "sources"), +] + + +def record_path(rec: dl.Record) -> Path: + """`records//.md` for an active record, `archive//.md` otherwise. + + Identity stays the KEY; the path is derived from it, and the validator enforces the two agree. + Every key in the corpus is exactly two dotted segments, so this is unambiguous — a key with + inner dots would need a rule and there are none. + """ + area, _, topic = (rec.key or "").partition(".") + root = dl.ARCHIVE_DIR if rec.status in ("superseded", "retired") else dl.RECORDS_DIR + return root / area / f"{topic}.md" + + +def render_record(rec: dl.Record) -> str: + """YAML frontmatter + the rationale body, copied verbatim.""" + meta: dict[str, object] = {} + for fm_key, field in _FIELD_ORDER: + val = rec.heading if fm_key == "title" else getattr(rec, field) + if val is None: + continue # absent stays absent; "" is emitted so empty-vs-absent survives + meta[fm_key] = val + fm = yaml.safe_dump( + meta, + sort_keys=False, + allow_unicode=True, + default_flow_style=False, + width=10**9, # never fold: keep every scalar on one line so diffs stay readable + ) + return f"---\n{fm}---\n\n{_rationale_body(rec)}\n" + + +def _rationale_body(rec: dl.Record) -> str: + """The record body with its contiguous metadata block stripped — the prose, byte-for-byte. + + Mirrors decisions_validate._rationale's contiguous-block rule, but preserves interior blank + lines and indentation (that function normalizes whitespace for comparison; here the bytes are + the payload). + """ + lines = rec.body.splitlines() + i = 0 + while i < len(lines) and not lines[i].strip(): + i += 1 + if i < len(lines) and lines[i].strip().startswith("`key:"): + j = i + 1 + while j < len(lines) and lines[j].strip(): + j += 1 + lines = lines[:i] + lines[j:] + return "\n".join(lines).strip() + + +def _preamble(path: Path, records: list[dl.Record]) -> str: + """Everything in `path` that is not inside a record — kept, never deleted.""" + text = path.read_text(encoding="utf-8") + lines = text.splitlines() + drop: set[int] = set() + for r in records: + start = r.lineno - 1 # heading line, 1-indexed in Record + for n in range(start, start + 1 + len(r.body.splitlines())): + drop.add(n) + kept = [ln for n, ln in enumerate(lines) if n not in drop] + while kept and not kept[-1].strip(): + kept.pop() + return "\n".join(kept) + + +def plan() -> tuple[list[tuple[dl.Record, Path]], dict[Path, str]]: + """(record -> destination) pairs, and (source file -> preamble to keep).""" + sources = [p for p in dl.active_files() if p.exists()] + sources += sorted(p for p in dl.ARCHIVE_DIR.glob("*.md") if p.name not in dl._NON_DECISION_FILES) + moves: list[tuple[dl.Record, Path]] = [] + preambles: dict[Path, str] = {} + for src in sources: + if dl.RECORDS_DIR in src.parents or dl.has_frontmatter(src.read_text(encoding="utf-8")): + continue # already migrated + recs = [r for r in dl.parse_file(src) if r.heading not in ("Index", "Active catalog", "Contents")] + keyed = [r for r in recs if r.key] + if len(keyed) != len(recs): + unkeyed = [r.heading for r in recs if not r.key] + raise SystemExit(f"{src}: {len(unkeyed)} record(s) without a key, cannot file by key: {unkeyed}") + moves += [(r, record_path(r)) for r in keyed] + preambles[src] = _preamble(src, recs) + return moves, preambles + + +def main(argv=None) -> int: + ap = argparse.ArgumentParser() + ap.add_argument("--dry-run", action="store_true", help="report the plan, write nothing") + args = ap.parse_args(argv) + + moves, preambles = plan() + + dests = [d for _, d in moves] + if len(set(dests)) != len(dests): + dupes = sorted({str(d) for d in dests if dests.count(d) > 1}) + raise SystemExit(f"destination collision — two records map to the same path: {dupes}") + + print(f"{len(moves)} records -> {len({d.parent for d in dests})} directories") + print( + f"{len(preambles)} source files kept, stripped to narrative " + f"({sum(len(v.splitlines()) for v in preambles.values())} lines preserved)" + ) + if args.dry_run: + for rec, dest in sorted(moves, key=lambda m: str(m[1]))[:5]: + print(f" {rec.key} -> {dest.relative_to(dl.REPO_ROOT)}") + print(" …") + return 0 + + for rec, dest in moves: + dest.parent.mkdir(parents=True, exist_ok=True) + dest.write_text(render_record(rec), encoding="utf-8") + for src, text in preambles.items(): + src.write_text(text.rstrip("\n") + "\n", encoding="utf-8") + print("migration written") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/tests/test_migration_equivalence.py b/scripts/tests/test_migration_equivalence.py new file mode 100644 index 000000000..8d6c9ae95 --- /dev/null +++ b/scripts/tests/test_migration_equivalence.py @@ -0,0 +1,156 @@ +"""Equivalence harness for the #610 one-file-per-record migration. + +The migration is a MOVE: content relocates, nothing is rewritten. That makes correctness +*provable* rather than reviewable — parse the corpus before, run the migration, parse it after, +and assert the two Record sets are identical field-for-field with byte-identical rationale. + +These tests run the real migration against a COPY of the real corpus in tmp_path. They never touch +the working tree. +""" + +from __future__ import annotations + +import shutil +from pathlib import Path + +import pytest + +import scripts.decisions_lib as dl +import scripts.migrate_decisions_split as mig + +COMPARED_FIELDS = ( + "key", + "status", + "since", + "supersedes", + "superseded_by", + "stale_after", + "rule", + "signals", + "mechanics", + "sources", +) +_SKIP = {"Index", "Active catalog", "Contents"} + + +def _corpus_files() -> list[Path]: + files = [p for p in dl.active_files() if p.exists()] + files += [p for p in sorted(dl.ARCHIVE_DIR.glob("*.md")) if p.name not in dl._NON_DECISION_FILES] + return files + + +@pytest.fixture(scope="module") +def migrated(tmp_path_factory): + """Copy the real docs/ into tmp, point decisions_lib at it, run the migration.""" + tmp = tmp_path_factory.mktemp("corpus") + shutil.copytree(dl.REPO_ROOT / "docs", tmp / "docs") + + before = {} + for f in _corpus_files(): + for r in dl.parse_file(f): + if r.key and r.heading not in _SKIP: + before[r.key] = (r, mig._rationale_body(r)) + + orig = (dl.REPO_ROOT, dl.DECISIONS_MD, dl.TOPIC_DIR, dl.ARCHIVE_DIR, dl.RECORDS_DIR) + dl.REPO_ROOT = tmp + dl.DECISIONS_MD = tmp / "docs" / "decisions.md" + dl.TOPIC_DIR = tmp / "docs" / "decisions" + dl.ARCHIVE_DIR = dl.TOPIC_DIR / "archive" + dl.RECORDS_DIR = dl.TOPIC_DIR / "records" + try: + mig.main([]) + after_files = [p for p in dl.RECORDS_DIR.rglob("*.md")] + [p for p in dl.ARCHIVE_DIR.rglob("*.md")] + after = {} + for f in after_files: + for r in dl.parse_file(f): + if r.key: + after[r.key] = (r, r.body) + yield before, after, tmp + finally: + dl.REPO_ROOT, dl.DECISIONS_MD, dl.TOPIC_DIR, dl.ARCHIVE_DIR, dl.RECORDS_DIR = orig + + +def test_every_record_survives_with_the_same_key(migrated): + before, after, _ = migrated + assert before, "harness parsed nothing before the migration — would pass vacuously" + assert set(before) == set(after), ( + f"lost: {sorted(set(before) - set(after))[:5]} / gained: {sorted(set(after) - set(before))[:5]}" + ) + + +def test_record_count_is_the_expected_166(migrated): + before, after, _ = migrated + assert len(before) == len(after) == 166 + + +def test_every_metadata_field_round_trips(migrated): + before, after, _ = migrated + diffs = [] + for key, (b, _) in before.items(): + a, _ = after[key] + for f in COMPARED_FIELDS: + if getattr(b, f) != getattr(a, f): + diffs.append(f"{key}.{f}: {getattr(b, f)!r} -> {getattr(a, f)!r}") + assert not diffs, "\n".join(diffs[:10]) + + +def test_title_round_trips_from_the_heading(migrated): + before, after, _ = migrated + diffs = [ + f"{k}: {b.heading!r} -> {after[k][0].heading!r}" + for k, (b, _) in before.items() + if b.heading != after[k][0].heading + ] + assert not diffs, "\n".join(diffs[:10]) + + +def test_rationale_bodies_are_byte_identical(migrated): + before, after, _ = migrated + diffs = [k for k, (_, body) in before.items() if body != after[k][1]] + assert not diffs, f"{len(diffs)} rationale bodies changed, e.g. {diffs[:5]}" + + +def test_path_matches_key(migrated): + _, after, _tmp = migrated + bad = [] + for key, (rec, _) in after.items(): + area, _, topic = key.partition(".") + if rec.source.parent.name != area or rec.source.stem != topic: + bad.append(f"{key} -> {rec.source}") + assert not bad, "\n".join(bad[:10]) + + +def test_archived_records_land_in_the_archive_wing(migrated): + _, after, _tmp = migrated + misfiled = [ + key + for key, (rec, _) in after.items() + if (rec.status in ("superseded", "retired")) != ("archive" in rec.source.parts) + ] + assert not misfiled, misfiled + + +def test_source_files_are_kept_and_retain_their_narrative(migrated): + """The ~500 lines of preamble are the only copy of that prose — they must survive.""" + _, _, tmp = migrated + kept = tmp / "docs" / "decisions.md" + assert kept.exists(), "decisions.md was deleted — its lifecycle schema is the only copy" + text = kept.read_text() + assert "Lifecycle, not append-only" in text + # NB: do NOT string-match on "## " here — the schema header quotes an illustrative + # "## 2026-07-17 — …" example in prose. Whether real records remain is a PARSING question, + # covered by test_no_records_remain_in_the_legacy_files. + wp = tmp / "docs" / "decisions" / "workflow-process.md" + assert wp.exists() and "extracted from" in wp.read_text() + + +def test_no_records_remain_in_the_legacy_files(migrated): + _, _, tmp = migrated + leftovers = {} + for f in [tmp / "docs" / "decisions.md", *sorted((tmp / "docs" / "decisions").glob("*.md"))]: + if f.name in dl._NON_DECISION_FILES: + continue + keyed = [r.key for r in dl.parse_file(f) if r.key] + if keyed: + leftovers[f.name] = keyed + assert not leftovers, leftovers -- 2.47.3 From ef67be924c537287524fb513ee04a2f3aae03b3c Mon Sep 17 00:00:00 2001 From: Timothy Date: Sat, 25 Jul 2026 17:37:26 +0200 Subject: [PATCH 03/10] feat(610): teach the validator and catalog the split layout Both surfaced in a trial migration against a disposable copy of the real corpus, not from reading: - Archive discovery was a NON-recursive glob. After the split, archived records live at archive//.md, so every one of them became invisible -- which surfaced as four bogus "supersedes points to unknown key" errors rather than as an obvious "no archive found". rglob at both sites. - Catalog links: a record is now a FILE, so the link is a plain relative path with no anchor -- nothing to slug, nothing to keep in sync with a heading. The legacy anchor form is kept for records still living inside a multi-record file, so the catalog is correct on either side of the migration. --- scripts/build_decisions_catalog.py | 19 +++++++++++++++---- scripts/decisions_validate.py | 4 ++-- 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/scripts/build_decisions_catalog.py b/scripts/build_decisions_catalog.py index b83ca0056..26d0045e9 100644 --- a/scripts/build_decisions_catalog.py +++ b/scripts/build_decisions_catalog.py @@ -17,12 +17,23 @@ BANNER = "