solvecraft/scripts/split_archives.py

#!/usr/bin/env python3
"""Split the frozen archives plan/findings.md and plan/upstream-issues.md into one file
per record (ADR style), and prove the split lossless.

One-off migration (2026-10-25). The archives are read from git at SOURCE_REV (the last
main commit that has them), so the output is the same wherever and whenever it is run.

    python3 -I scripts/split_archives.py write   # write the per-record files and the
                                                 # README archive blocks, delete the archives
    python3 -I scripts/split_archives.py check   # verify the files on disk: byte-identical
                                                 # to what `write` produces, and lossless

Findings: every `## ` section becomes plan/findings/<unit or general>--<slug>.md, holding
`# <heading>` and the section body verbatim. The text before the first `## ` goes, quoted,
into an archive block of plan/findings/README.md.

Upstream issues: every `#### ` entry becomes plan/upstream-issues/<ID>--<slug>.md holding
the `#### ` line verbatim, `- **Severity:** <## heading>`, `- **Area:** <### heading>` and
the body verbatim. The text before the first `## ` and the `## Note: duplicate IDs`
section go verbatim into archive blocks of plan/upstream-issues/README.md; the other odd
`## ` section (a hang report without a `#### ` entry) gets its own file. Entries under the
two odd `## ` headings take the severity of their ID's series. The second entry with an
ID already used gets a `--dup2` name suffix (heading unchanged).

The lossless check rebuilds each archive's non-blank lines, in archive order, from the
files on disk (undoing only the heading changes) and compares them with the archive.
The issues archive's `## `/`### ` headings are checked separately: each must appear as a
Severity/Area value, or verbatim in a README block or its own file.
"""

import re
import subprocess
import sys
from pathlib import Path

ROOT = Path(__file__).resolve().parent.parent
SOURCE_REV = "4fe6f4d51676792b2551ae18ccdbf38a5e10852f"
FINDINGS = "plan/findings.md"
ISSUES = "plan/upstream-issues.md"
FINDINGS_DIR = "plan/findings"
ISSUES_DIR = "plan/upstream-issues"

SEVERITIES = ("Crash / undefined behaviour", "Hang", "Nondeterminism", "Wrong result", "Docs")
SERIES_SEVERITY = {"UB": "Crash / undefined behaviour", "HANG": "Hang", "ND": "Nondeterminism",
                   "WR": "Wrong result", "DOC": "Docs", "FAIL": "Wrong result",
                   "ISSUE": "Wrong result"}
ID_PREFIX = r"^(?:[A-Z]+-\d+(?: \([^)]*\))?|[A-Z]+-new|[A-Z]+ \([^)]*\)|Issue #\d+)\.\s*"
UNIT_RE = r"[a-z][a-z0-9_]*\.[a-z][a-z0-9_]*"
# The only archive lines not carried over: the issues archive's own "Frozen archive" notice
# (stale once the archive is gone, and its relative link no longer resolves). The check
# lists them and leaves them out of the comparison.
ISSUES_INTRO_DROPPED = (
    "> **Frozen archive (2026-10-25).** Do not add entries here: every new issue is its own",
    "> file under [`plan/upstream-issues/`](upstream-issues/README.md), so parallel PRs never",
    "> conflict. The site renders both.",
)
BEGIN = "<!-- archive:{} begin (generated by scripts/split_archives.py) -->"
END = "<!-- archive:{} end -->"


def git(*args):
    return subprocess.run(["git", "-C", str(ROOT), *args], check=True,
                          capture_output=True, text=True).stdout


def archive(path):
    return git("show", f"{SOURCE_REV}:{path}").split("\n")


def existing(directory):
    """Record files already in the directory at SOURCE_REV (generated names avoid them)."""
    out = git("ls-tree", "--name-only", SOURCE_REV, directory + "/")
    return {Path(l).name for l in out.split() if l.endswith(".md")}


def slugify(text, limit=60):
    s = re.sub(r"[^a-z0-9]+", "-", text.lower()).strip("-")
    if len(s) > limit:
        cut = s[:limit + 1]
        s = cut[:cut.rfind("-")] if "-" in cut else s[:limit]
    return s.strip("-")


def trim(lines):
    """Drop trailing blank lines."""
    lines = list(lines)
    while lines and not lines[-1].strip():
        lines.pop()
    return lines


def unique(name, taken):
    stem, n = name, 2
    while f"{name}.md" in taken:
        name = f"{stem}-{n}"
        n += 1
    taken.add(f"{name}.md")
    return f"{name}.md"


def nonblank(lines):
    return [l for l in lines if l.strip()]


# ---------------------------------------------------------------- findings

def split_findings():
    """Returns ([(path, text)] in archive order, {readme: {block: text}})."""
    taken = existing(FINDINGS_DIR)
    lines = archive(FINDINGS)
    assert lines[0] == "# Findings", lines[:1]
    intro, secs, cur, fence = [], [], None, False
    for line in lines[1:]:
        if line.startswith("```"):
            fence = not fence
        if not fence and line.startswith("## "):
            cur = (line, [])
            secs.append(cur)
        elif cur is None:
            intro.append(line)
        else:
            cur[1].append(line)
    files = []
    for head, body in secs:
        title = head[3:].strip()
        m = re.match(rf"({UNIT_RE})((?:\s*(?:,|/|\band\b)\s*{UNIT_RE})*)(.*)$", title)
        if m:
            unit, rest = m.group(1), re.sub(r"^[\s:;,—–-]+", "", m.group(3))
        else:
            unit, rest = "general", title
        name = unique(f"{unit}--{slugify(rest) or 'notes'}", taken)
        files.append((f"{FINDINGS_DIR}/{name}", "\n".join([f"# {title}"] + trim(body)) + "\n"))
    intro = trim(intro)
    while intro and not intro[0].strip():
        intro.pop(0)
    block = "\n".join(("> " + l) if l.strip() else ">" for l in intro)
    return files, {f"{FINDINGS_DIR}/README.md": {"intro": block}}


# ---------------------------------------------------------------- upstream issues

def entry_id(title):
    """The ID the site uses (`X-N`, else the whole title) and the ID for the file name."""
    m = re.match(r"([A-Z]+)-(\d+)\.\s*", title)
    site_id = f"{m.group(1)}-{m.group(2)}" if m else title
    for pat, fmt in ((r"([A-Z]+-\d+)\b", "{0}"), (r"Issue #(\d+)", "ISSUE-{0}"),
                     (r"([A-Z]+-new)\b", "{0}"), (r"([A-Z]+) \(", "{0}-unnumbered")):
        m = re.match(pat, title)
        if m:
            return site_id, fmt.format(m.group(1))
    return site_id, "UNNUMBERED"


def split_issues():
    """Returns (odd-section files, {readme: {block: text}}, pieces, entries).

    pieces: the odd `## ` sections in archive order, ("block", readme, name) or
    ("file", path). entries: (heading line, path, text) in archive order."""
    taken = existing(ISSUES_DIR)
    lines = archive(ISSUES)
    assert lines[0] == "# Upstream SMLib issues", lines[:1]
    readme = f"{ISSUES_DIR}/README.md"
    files, blocks, pieces, raw, intro = [], {readme: {}}, [], [], []
    severity = area = ""
    cur = odd_sec = None
    fence = seen_h2 = False

    def close():
        nonlocal cur, odd_sec
        if cur is not None:
            raw.append(cur)
        if odd_sec is not None:
            target, name, body = odd_sec
            if name is None:
                files.append((target, "\n".join(trim(body)) + "\n"))
            else:
                blocks[readme][name] = "\n".join(trim(body))
        cur = odd_sec = None

    for line in lines[1:]:
        if line.startswith("```"):
            fence = not fence
        h = None if fence else re.match(r"(#{2,4}) (.*)$", line)
        if h:
            level, title = len(h.group(1)), h.group(2).strip()
            close()
            if level == 2:
                seen_h2 = True
                severity = title if title in SEVERITIES else ""
                area = ""
                if title.startswith("Note: duplicate IDs"):
                    odd_sec = (readme, "note-duplicate-ids", [line])
                    pieces.append(("block", readme, "note-duplicate-ids"))
                elif not severity:
                    path = f"{ISSUES_DIR}/{unique('HANG-unnumbered--' + slugify(title), taken)}"
                    odd_sec = (path, None, [line])
                    pieces.append(("file", path))
            elif level == 3:
                area = title
            else:
                cur = (line, [], severity, area)
            continue
        if cur is not None:
            cur[1].append(line)
        elif odd_sec is not None:
            odd_sec[2].append(line)
        elif not seen_h2:
            intro.append(line)
        else:
            assert not line.strip(), f"text outside any entry: {line!r}"
    close()
    intro = [l for l in intro if l not in ISSUES_INTRO_DROPPED]
    blocks[readme]["intro"] = "\n".join(trim(intro)).strip("\n")

    entries, seen = [], {}
    for head, body, sev, ar in raw:
        title = head[5:].strip()
        site_id, name_id = entry_id(title)
        seen[site_id] = seen.get(site_id, 0) + 1
        sev = sev or SERIES_SEVERITY.get(name_id.split("-")[0], "Wrong result")
        stem = f"{name_id}--{slugify(re.sub(ID_PREFIX, '', title))}"
        if seen[site_id] > 1:
            stem += f"--dup{seen[site_id]}"
        text = "\n".join([head, f"- **Severity:** {sev}",
                          f"- **Area:** {ar}" if ar else "- **Area:**"] + trim(body)) + "\n"
        entries.append((head, f"{ISSUES_DIR}/{unique(stem, taken)}", text))
    return files, blocks, pieces, entries


# ---------------------------------------------------------------- write / check

def fill_blocks(path, blocks):
    text = (ROOT / path).read_text()
    for name, content in blocks.items():
        b, e = BEGIN.format(name), END.format(name)
        i, j = text.find(b), text.find(e)
        assert 0 <= i < j, f"{path}: missing markers for {name}"
        text = text[:i + len(b)] + "\n" + content + "\n" + text[j:]
    return text


def read_block(text, name):
    b, e = BEGIN.format(name), END.format(name)
    i, j = text.find(b), text.find(e)
    assert 0 <= i < j, f"missing markers for {name}"
    return text[i + len(b):j].strip("\n").split("\n")


def all_outputs():
    ffiles, fblocks = split_findings()
    ifiles, iblocks, _, ientries = split_issues()
    outs = dict(ffiles)
    outs.update(ifiles)
    outs.update((p, t) for _, p, t in ientries)
    for path, blocks in list(fblocks.items()) + list(iblocks.items()):
        outs[path] = fill_blocks(path, blocks)
    return outs


def write():
    outs = all_outputs()
    for path, text in outs.items():
        (ROOT / path).write_text(text)
    for path in (FINDINGS, ISSUES):
        (ROOT / path).unlink(missing_ok=True)
    print(f"wrote {len(outs)} files, removed {FINDINGS} and {ISSUES}")


def report(label, want, got, nfiles):
    if want == got:
        print(f"{label}: lossless, {len(want)} non-blank archive lines found in order "
              f"in {nfiles} files")
        return True
    i = next((k for k, (a, b) in enumerate(zip(want, got)) if a != b), min(len(want), len(got)))
    print(f"{label}: MISMATCH at non-blank line {i}: archive={want[i:i + 1]} "
          f"files={got[i:i + 1]} ({len(want)} vs {len(got)} lines)")
    return False


def check():
    ok = True
    for path, text in all_outputs().items():
        p = ROOT / path
        if not p.exists() or p.read_text() != text:
            print(f"MISMATCH: {path} differs from what `write` generates")
            ok = False
    for path in (FINDINGS, ISSUES):
        if (ROOT / path).exists():
            print(f"FAIL: {path} still exists")
            ok = False

    # findings: README block (unquoted), then every file, in archive order, from disk
    ffiles, _ = split_findings()
    readme = (ROOT / FINDINGS_DIR / "README.md").read_text()
    stream = [re.sub(r"^> ?", "", l) for l in read_block(readme, "intro")]
    for path, _ in ffiles:
        lines = (ROOT / path).read_text().split("\n")
        assert lines[0].startswith("# "), path
        stream += ["#" + lines[0]] + lines[1:]
    ok &= report("findings", nonblank(archive(FINDINGS)[1:]), nonblank(stream), len(ffiles) + 1)

    # upstream issues: walk the archive; at each #### take the next entry file, at each
    # odd ## the next README block or file, all read from disk
    ifiles, _, pieces, entries = split_issues()
    alines = archive(ISSUES)
    ireadme = (ROOT / ISSUES_DIR / "README.md").read_text()
    stream = read_block(ireadme, "intro")
    on_disk = set()
    entry_it, piece_it = iter(entries), iter(pieces)
    fence = False
    for line in alines[1:]:
        if line.startswith("```"):
            fence = not fence
        if fence:
            continue
        if line.startswith("#### "):
            head, path, _ = next(entry_it)
            lines = (ROOT / path).read_text().split("\n")
            sev = re.match(r"- \*\*Severity:\*\* (.*)$", lines[1])
            area = re.match(r"- \*\*Area:\*\*\s*(.*)$", lines[2])
            assert lines[0] == head and sev and area, path
            on_disk.update({sev.group(1), area.group(1)})
            stream += [lines[0]] + lines[3:]
        elif line.startswith("## ") and line[3:].strip() not in SEVERITIES:
            kind, path, *name = next(piece_it)
            text = (ROOT / path).read_text()
            block = read_block(text, name[0]) if kind == "block" else text.split("\n")
            on_disk.update(l[3:].strip() for l in block if l.startswith("## "))
            stream += block
    assert next(entry_it, None) is None and next(piece_it, None) is None
    drop_heads = lambda ls: [l for l in nonblank(ls) if not re.match(r"#{1,3} ", l)]
    want = drop_heads(alines)
    for line in ISSUES_INTRO_DROPPED:  # exactly once each, in the intro
        assert want.count(line) == 1 and line not in stream, line
        want.remove(line)
    print(f"upstream-issues: {len(ISSUES_INTRO_DROPPED)} archive lines left out on purpose "
          "(the archive's own frozen notice):")
    for line in ISSUES_INTRO_DROPPED:
        print(f"    {line}")
    ok &= report("upstream-issues", want, drop_heads(stream), len(entries) + len(ifiles) + 1)
    heads = [l.lstrip("#").strip() for l in alines if re.match(r"#{2,3} ", l)]
    missing = [h for h in heads if h not in on_disk]
    print(f"upstream-issues: {len(heads)} section headings, {len(missing)} not on disk"
          + (f": {missing}" if missing else ""))
    ok &= not missing
    print(f"{len(ffiles)} finding files, {len(entries)} issue entry files")
    print("OK" if ok else "FAILED")
    return ok


if __name__ == "__main__":
    cmd = sys.argv[1] if len(sys.argv) > 1 else ""
    if cmd == "write":
        write()
    elif cmd == "check":
        sys.exit(0 if check() else 1)
    else:
        sys.exit(__doc__)