mirror of
https://github.com/bherbruck/solidcraft.git
synced 2026-11-08 04:52:10 +00:00
bherbruck
9c6f3218c4
Keeps #983's known_mismatches.toml guard next to the archive guards in check-frozen-archives.sh, and both sides' wording in AGENTS.md and plan/findings/README.md. split_archives.py leaves the issues archive's own "Frozen archive" notice (3 lines, dead relative link) out of the README; the check lists those lines and compares the rest. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01QaDuBSC1mqS9DnQoEsr7LJ
359 lines
14 KiB
Python
Executable File
359 lines
14 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""Split the frozen archives plan/findings.md and plan/upstream-issues.md into one file
|
|
per record (ADR style), and prove the split lossless.
|
|
|
|
One-off migration (2026-10-25). The archives are read from git at SOURCE_REV (the last
|
|
main commit that has them), so the output is the same wherever and whenever it is run.
|
|
|
|
python3 -I scripts/split_archives.py write # write the per-record files and the
|
|
# README archive blocks, delete the archives
|
|
python3 -I scripts/split_archives.py check # verify the files on disk: byte-identical
|
|
# to what `write` produces, and lossless
|
|
|
|
Findings: every `## ` section becomes plan/findings/<unit or general>--<slug>.md, holding
|
|
`# <heading>` and the section body verbatim. The text before the first `## ` goes, quoted,
|
|
into an archive block of plan/findings/README.md.
|
|
|
|
Upstream issues: every `#### ` entry becomes plan/upstream-issues/<ID>--<slug>.md holding
|
|
the `#### ` line verbatim, `- **Severity:** <## heading>`, `- **Area:** <### heading>` and
|
|
the body verbatim. The text before the first `## ` and the `## Note: duplicate IDs`
|
|
section go verbatim into archive blocks of plan/upstream-issues/README.md; the other odd
|
|
`## ` section (a hang report without a `#### ` entry) gets its own file. Entries under the
|
|
two odd `## ` headings take the severity of their ID's series. The second entry with an
|
|
ID already used gets a `--dup2` name suffix (heading unchanged).
|
|
|
|
The lossless check rebuilds each archive's non-blank lines, in archive order, from the
|
|
files on disk (undoing only the heading changes) and compares them with the archive.
|
|
The issues archive's `## `/`### ` headings are checked separately: each must appear as a
|
|
Severity/Area value, or verbatim in a README block or its own file.
|
|
"""
|
|
|
|
import re
|
|
import subprocess
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
ROOT = Path(__file__).resolve().parent.parent
|
|
SOURCE_REV = "4fe6f4d51676792b2551ae18ccdbf38a5e10852f"
|
|
FINDINGS = "plan/findings.md"
|
|
ISSUES = "plan/upstream-issues.md"
|
|
FINDINGS_DIR = "plan/findings"
|
|
ISSUES_DIR = "plan/upstream-issues"
|
|
|
|
SEVERITIES = ("Crash / undefined behaviour", "Hang", "Nondeterminism", "Wrong result", "Docs")
|
|
SERIES_SEVERITY = {"UB": "Crash / undefined behaviour", "HANG": "Hang", "ND": "Nondeterminism",
|
|
"WR": "Wrong result", "DOC": "Docs", "FAIL": "Wrong result",
|
|
"ISSUE": "Wrong result"}
|
|
ID_PREFIX = r"^(?:[A-Z]+-\d+(?: \([^)]*\))?|[A-Z]+-new|[A-Z]+ \([^)]*\)|Issue #\d+)\.\s*"
|
|
UNIT_RE = r"[a-z][a-z0-9_]*\.[a-z][a-z0-9_]*"
|
|
# The only archive lines not carried over: the issues archive's own "Frozen archive" notice
|
|
# (stale once the archive is gone, and its relative link no longer resolves). The check
|
|
# lists them and leaves them out of the comparison.
|
|
ISSUES_INTRO_DROPPED = (
|
|
"> **Frozen archive (2026-10-25).** Do not add entries here: every new issue is its own",
|
|
"> file under [`plan/upstream-issues/`](upstream-issues/README.md), so parallel PRs never",
|
|
"> conflict. The site renders both.",
|
|
)
|
|
BEGIN = "<!-- archive:{} begin (generated by scripts/split_archives.py) -->"
|
|
END = "<!-- archive:{} end -->"
|
|
|
|
|
|
def git(*args):
|
|
return subprocess.run(["git", "-C", str(ROOT), *args], check=True,
|
|
capture_output=True, text=True).stdout
|
|
|
|
|
|
def archive(path):
|
|
return git("show", f"{SOURCE_REV}:{path}").split("\n")
|
|
|
|
|
|
def existing(directory):
|
|
"""Record files already in the directory at SOURCE_REV (generated names avoid them)."""
|
|
out = git("ls-tree", "--name-only", SOURCE_REV, directory + "/")
|
|
return {Path(l).name for l in out.split() if l.endswith(".md")}
|
|
|
|
|
|
def slugify(text, limit=60):
|
|
s = re.sub(r"[^a-z0-9]+", "-", text.lower()).strip("-")
|
|
if len(s) > limit:
|
|
cut = s[:limit + 1]
|
|
s = cut[:cut.rfind("-")] if "-" in cut else s[:limit]
|
|
return s.strip("-")
|
|
|
|
|
|
def trim(lines):
|
|
"""Drop trailing blank lines."""
|
|
lines = list(lines)
|
|
while lines and not lines[-1].strip():
|
|
lines.pop()
|
|
return lines
|
|
|
|
|
|
def unique(name, taken):
|
|
stem, n = name, 2
|
|
while f"{name}.md" in taken:
|
|
name = f"{stem}-{n}"
|
|
n += 1
|
|
taken.add(f"{name}.md")
|
|
return f"{name}.md"
|
|
|
|
|
|
def nonblank(lines):
|
|
return [l for l in lines if l.strip()]
|
|
|
|
|
|
# ---------------------------------------------------------------- findings
|
|
|
|
def split_findings():
|
|
"""Returns ([(path, text)] in archive order, {readme: {block: text}})."""
|
|
taken = existing(FINDINGS_DIR)
|
|
lines = archive(FINDINGS)
|
|
assert lines[0] == "# Findings", lines[:1]
|
|
intro, secs, cur, fence = [], [], None, False
|
|
for line in lines[1:]:
|
|
if line.startswith("```"):
|
|
fence = not fence
|
|
if not fence and line.startswith("## "):
|
|
cur = (line, [])
|
|
secs.append(cur)
|
|
elif cur is None:
|
|
intro.append(line)
|
|
else:
|
|
cur[1].append(line)
|
|
files = []
|
|
for head, body in secs:
|
|
title = head[3:].strip()
|
|
m = re.match(rf"({UNIT_RE})((?:\s*(?:,|/|\band\b)\s*{UNIT_RE})*)(.*)$", title)
|
|
if m:
|
|
unit, rest = m.group(1), re.sub(r"^[\s:;,—–-]+", "", m.group(3))
|
|
else:
|
|
unit, rest = "general", title
|
|
name = unique(f"{unit}--{slugify(rest) or 'notes'}", taken)
|
|
files.append((f"{FINDINGS_DIR}/{name}", "\n".join([f"# {title}"] + trim(body)) + "\n"))
|
|
intro = trim(intro)
|
|
while intro and not intro[0].strip():
|
|
intro.pop(0)
|
|
block = "\n".join(("> " + l) if l.strip() else ">" for l in intro)
|
|
return files, {f"{FINDINGS_DIR}/README.md": {"intro": block}}
|
|
|
|
|
|
# ---------------------------------------------------------------- upstream issues
|
|
|
|
def entry_id(title):
|
|
"""The ID the site uses (`X-N`, else the whole title) and the ID for the file name."""
|
|
m = re.match(r"([A-Z]+)-(\d+)\.\s*", title)
|
|
site_id = f"{m.group(1)}-{m.group(2)}" if m else title
|
|
for pat, fmt in ((r"([A-Z]+-\d+)\b", "{0}"), (r"Issue #(\d+)", "ISSUE-{0}"),
|
|
(r"([A-Z]+-new)\b", "{0}"), (r"([A-Z]+) \(", "{0}-unnumbered")):
|
|
m = re.match(pat, title)
|
|
if m:
|
|
return site_id, fmt.format(m.group(1))
|
|
return site_id, "UNNUMBERED"
|
|
|
|
|
|
def split_issues():
|
|
"""Returns (odd-section files, {readme: {block: text}}, pieces, entries).
|
|
|
|
pieces: the odd `## ` sections in archive order, ("block", readme, name) or
|
|
("file", path). entries: (heading line, path, text) in archive order."""
|
|
taken = existing(ISSUES_DIR)
|
|
lines = archive(ISSUES)
|
|
assert lines[0] == "# Upstream SMLib issues", lines[:1]
|
|
readme = f"{ISSUES_DIR}/README.md"
|
|
files, blocks, pieces, raw, intro = [], {readme: {}}, [], [], []
|
|
severity = area = ""
|
|
cur = odd_sec = None
|
|
fence = seen_h2 = False
|
|
|
|
def close():
|
|
nonlocal cur, odd_sec
|
|
if cur is not None:
|
|
raw.append(cur)
|
|
if odd_sec is not None:
|
|
target, name, body = odd_sec
|
|
if name is None:
|
|
files.append((target, "\n".join(trim(body)) + "\n"))
|
|
else:
|
|
blocks[readme][name] = "\n".join(trim(body))
|
|
cur = odd_sec = None
|
|
|
|
for line in lines[1:]:
|
|
if line.startswith("```"):
|
|
fence = not fence
|
|
h = None if fence else re.match(r"(#{2,4}) (.*)$", line)
|
|
if h:
|
|
level, title = len(h.group(1)), h.group(2).strip()
|
|
close()
|
|
if level == 2:
|
|
seen_h2 = True
|
|
severity = title if title in SEVERITIES else ""
|
|
area = ""
|
|
if title.startswith("Note: duplicate IDs"):
|
|
odd_sec = (readme, "note-duplicate-ids", [line])
|
|
pieces.append(("block", readme, "note-duplicate-ids"))
|
|
elif not severity:
|
|
path = f"{ISSUES_DIR}/{unique('HANG-unnumbered--' + slugify(title), taken)}"
|
|
odd_sec = (path, None, [line])
|
|
pieces.append(("file", path))
|
|
elif level == 3:
|
|
area = title
|
|
else:
|
|
cur = (line, [], severity, area)
|
|
continue
|
|
if cur is not None:
|
|
cur[1].append(line)
|
|
elif odd_sec is not None:
|
|
odd_sec[2].append(line)
|
|
elif not seen_h2:
|
|
intro.append(line)
|
|
else:
|
|
assert not line.strip(), f"text outside any entry: {line!r}"
|
|
close()
|
|
intro = [l for l in intro if l not in ISSUES_INTRO_DROPPED]
|
|
blocks[readme]["intro"] = "\n".join(trim(intro)).strip("\n")
|
|
|
|
entries, seen = [], {}
|
|
for head, body, sev, ar in raw:
|
|
title = head[5:].strip()
|
|
site_id, name_id = entry_id(title)
|
|
seen[site_id] = seen.get(site_id, 0) + 1
|
|
sev = sev or SERIES_SEVERITY.get(name_id.split("-")[0], "Wrong result")
|
|
stem = f"{name_id}--{slugify(re.sub(ID_PREFIX, '', title))}"
|
|
if seen[site_id] > 1:
|
|
stem += f"--dup{seen[site_id]}"
|
|
text = "\n".join([head, f"- **Severity:** {sev}",
|
|
f"- **Area:** {ar}" if ar else "- **Area:**"] + trim(body)) + "\n"
|
|
entries.append((head, f"{ISSUES_DIR}/{unique(stem, taken)}", text))
|
|
return files, blocks, pieces, entries
|
|
|
|
|
|
# ---------------------------------------------------------------- write / check
|
|
|
|
def fill_blocks(path, blocks):
|
|
text = (ROOT / path).read_text()
|
|
for name, content in blocks.items():
|
|
b, e = BEGIN.format(name), END.format(name)
|
|
i, j = text.find(b), text.find(e)
|
|
assert 0 <= i < j, f"{path}: missing markers for {name}"
|
|
text = text[:i + len(b)] + "\n" + content + "\n" + text[j:]
|
|
return text
|
|
|
|
|
|
def read_block(text, name):
|
|
b, e = BEGIN.format(name), END.format(name)
|
|
i, j = text.find(b), text.find(e)
|
|
assert 0 <= i < j, f"missing markers for {name}"
|
|
return text[i + len(b):j].strip("\n").split("\n")
|
|
|
|
|
|
def all_outputs():
|
|
ffiles, fblocks = split_findings()
|
|
ifiles, iblocks, _, ientries = split_issues()
|
|
outs = dict(ffiles)
|
|
outs.update(ifiles)
|
|
outs.update((p, t) for _, p, t in ientries)
|
|
for path, blocks in list(fblocks.items()) + list(iblocks.items()):
|
|
outs[path] = fill_blocks(path, blocks)
|
|
return outs
|
|
|
|
|
|
def write():
|
|
outs = all_outputs()
|
|
for path, text in outs.items():
|
|
(ROOT / path).write_text(text)
|
|
for path in (FINDINGS, ISSUES):
|
|
(ROOT / path).unlink(missing_ok=True)
|
|
print(f"wrote {len(outs)} files, removed {FINDINGS} and {ISSUES}")
|
|
|
|
|
|
def report(label, want, got, nfiles):
|
|
if want == got:
|
|
print(f"{label}: lossless, {len(want)} non-blank archive lines found in order "
|
|
f"in {nfiles} files")
|
|
return True
|
|
i = next((k for k, (a, b) in enumerate(zip(want, got)) if a != b), min(len(want), len(got)))
|
|
print(f"{label}: MISMATCH at non-blank line {i}: archive={want[i:i + 1]} "
|
|
f"files={got[i:i + 1]} ({len(want)} vs {len(got)} lines)")
|
|
return False
|
|
|
|
|
|
def check():
|
|
ok = True
|
|
for path, text in all_outputs().items():
|
|
p = ROOT / path
|
|
if not p.exists() or p.read_text() != text:
|
|
print(f"MISMATCH: {path} differs from what `write` generates")
|
|
ok = False
|
|
for path in (FINDINGS, ISSUES):
|
|
if (ROOT / path).exists():
|
|
print(f"FAIL: {path} still exists")
|
|
ok = False
|
|
|
|
# findings: README block (unquoted), then every file, in archive order, from disk
|
|
ffiles, _ = split_findings()
|
|
readme = (ROOT / FINDINGS_DIR / "README.md").read_text()
|
|
stream = [re.sub(r"^> ?", "", l) for l in read_block(readme, "intro")]
|
|
for path, _ in ffiles:
|
|
lines = (ROOT / path).read_text().split("\n")
|
|
assert lines[0].startswith("# "), path
|
|
stream += ["#" + lines[0]] + lines[1:]
|
|
ok &= report("findings", nonblank(archive(FINDINGS)[1:]), nonblank(stream), len(ffiles) + 1)
|
|
|
|
# upstream issues: walk the archive; at each #### take the next entry file, at each
|
|
# odd ## the next README block or file, all read from disk
|
|
ifiles, _, pieces, entries = split_issues()
|
|
alines = archive(ISSUES)
|
|
ireadme = (ROOT / ISSUES_DIR / "README.md").read_text()
|
|
stream = read_block(ireadme, "intro")
|
|
on_disk = set()
|
|
entry_it, piece_it = iter(entries), iter(pieces)
|
|
fence = False
|
|
for line in alines[1:]:
|
|
if line.startswith("```"):
|
|
fence = not fence
|
|
if fence:
|
|
continue
|
|
if line.startswith("#### "):
|
|
head, path, _ = next(entry_it)
|
|
lines = (ROOT / path).read_text().split("\n")
|
|
sev = re.match(r"- \*\*Severity:\*\* (.*)$", lines[1])
|
|
area = re.match(r"- \*\*Area:\*\*\s*(.*)$", lines[2])
|
|
assert lines[0] == head and sev and area, path
|
|
on_disk.update({sev.group(1), area.group(1)})
|
|
stream += [lines[0]] + lines[3:]
|
|
elif line.startswith("## ") and line[3:].strip() not in SEVERITIES:
|
|
kind, path, *name = next(piece_it)
|
|
text = (ROOT / path).read_text()
|
|
block = read_block(text, name[0]) if kind == "block" else text.split("\n")
|
|
on_disk.update(l[3:].strip() for l in block if l.startswith("## "))
|
|
stream += block
|
|
assert next(entry_it, None) is None and next(piece_it, None) is None
|
|
drop_heads = lambda ls: [l for l in nonblank(ls) if not re.match(r"#{1,3} ", l)]
|
|
want = drop_heads(alines)
|
|
for line in ISSUES_INTRO_DROPPED: # exactly once each, in the intro
|
|
assert want.count(line) == 1 and line not in stream, line
|
|
want.remove(line)
|
|
print(f"upstream-issues: {len(ISSUES_INTRO_DROPPED)} archive lines left out on purpose "
|
|
"(the archive's own frozen notice):")
|
|
for line in ISSUES_INTRO_DROPPED:
|
|
print(f" {line}")
|
|
ok &= report("upstream-issues", want, drop_heads(stream), len(entries) + len(ifiles) + 1)
|
|
heads = [l.lstrip("#").strip() for l in alines if re.match(r"#{2,3} ", l)]
|
|
missing = [h for h in heads if h not in on_disk]
|
|
print(f"upstream-issues: {len(heads)} section headings, {len(missing)} not on disk"
|
|
+ (f": {missing}" if missing else ""))
|
|
ok &= not missing
|
|
print(f"{len(ffiles)} finding files, {len(entries)} issue entry files")
|
|
print("OK" if ok else "FAILED")
|
|
return ok
|
|
|
|
|
|
if __name__ == "__main__":
|
|
cmd = sys.argv[1] if len(sys.argv) > 1 else ""
|
|
if cmd == "write":
|
|
write()
|
|
elif cmd == "check":
|
|
sys.exit(0 if check() else 1)
|
|
else:
|
|
sys.exit(__doc__)
|