solvecraft/harness/callgraph/write_closure.py

#!/usr/bin/env python3
"""Write plan/closure.md (and harness/callgraph/unused_functions.txt) from the call-graph outputs.

Run after extract.py, build_graph.py and plan_units.py.
"""
import collections
import glob
import json
import os
import re
import sys

HERE = os.path.dirname(os.path.abspath(__file__))
ROOT = os.path.abspath(os.path.join(HERE, "..", ".."))
D = os.path.join(ROOT, "bin", "callgraph")
UP = os.path.join(ROOT, "upstream")
sys.path.insert(0, HERE)
from entrypoints import ENTRY_GROUPS, KERNEL_ROOTS, OUT_OF_SCOPE_ENTRY  # noqa: E402
from pp import inactive_lines  # noqa: E402

g = json.load(open(os.path.join(D, "graph.json")))
cl = json.load(open(os.path.join(D, "closure.json")))
FILES = json.load(open(os.path.join(D, "files.json")))
S = json.load(open(os.path.join(D, "plan_summary.json")))
UNITS = json.load(open(os.path.join(HERE, "units.json")))
F = g["funcs"]
CHA, RTA = set(cl["cha"]), set(cl["rta"])


def L(f):
    return f["end"] - f["line"] + 1


src_cache = {}


def src(path):
    if path not in src_cache:
        with open(os.path.join(UP, path), errors="replace") as fh:
            src_cache[path] = fh.read().split("\n")
    return src_cache[path]


out = []
w = out.append
tot_lines = sum(v["lines"] for v in FILES.values())
tot_func = sum(v["func_lines"] for v in FILES.values())
cha_lines = sum(v["cha_lines"] for v in FILES.values())
rta_lines = sum(v["rta_lines"] for v in FILES.values())
inscope_files = sorted(k for k, v in FILES.items() if v["cha_lines"] > 0)
unused_files = sorted(k for k, v in FILES.items() if v["cha_lines"] == 0)

w("# In-scope closure of SMLib for solidcore")
w("")
w("Generated by `harness/callgraph/write_closure.py` (pipeline: `extract.py` -> `build_graph.py` ->")
w("`plan_units.py` -> `write_closure.py`; see `harness/callgraph/README.md`). Upstream commit c9c5979.")
w("")
w("## Summary")
w("")
w("| quantity | value |")
w("|---|---:|")
w("| upstream SMLib + SM_API lines (src + inc) | %d |" % tot_lines)
w("| lines inside function bodies (all) | %d |" % tot_func)
w("| **in-scope function lines (CHA closure)** | **%d** |" % cha_lines)
w("| in-scope function lines under RTA (informational) | %d |" % rta_lines)
w("| in-scope functions (CHA) | %d |" % len(CHA))
w("| idiom-mapped lines (containers/memory/error stacks -> Rust std, not ported as units) | %d |" % S["idiom_lines"])
w("| lines assigned to port units | %d |" % S["inscope_lines"])
w("| port units / layers | %d / %d |" % (S["units"], S["layers"]))
w("| in-scope files (>=1 reachable function) | %d |" % len(inscope_files))
w("| files with no reachable function | %d |" % len(unused_files))
w("")
w("## Method")
w("")
w("* **AST source.** libclang 21 (Python bindings) over `bin/cpp/compile_commands.json`, i.e. the exact")
w("  flags of the reference C++ build (`-DNDEBUG -DNOMINMAX -DSM_GRAPHICS_CALLBACKS=1 -std=c++17`, so")
w("  `SM_DEBUG_CODE`, `SM_USE_TBB`, `SM_USE_GLOBAL_CACHE` are *off*; `SM_GFX_CODE` is on). All 224")
w("  SMLib + 13 SM_API TUs (plus the SM_API_test / prog_test TUs for test attribution) parse with zero errors.")
w("* **Edges.** Every `CALL_EXPR`, function/method reference (`DECL_REF_EXPR`/`MEMBER_REF_EXPR`, which")
w("  also captures function pointers and callbacks), `new` -> constructor, `delete` -> destructor of the")
w("  static type (and of all derived classes), and constructor -> own destructor. Template members map")
w("  to their primary template by qualified name.")
w("* **Virtual calls (conservative).** An unqualified call to a virtual method reaches the method and")
w("  *every* override in every (transitively) derived class (class-hierarchy analysis, CHA). Qualified")
w("  calls (`Base::f()`) are non-virtual. The pip `libclang` bindings lack `clang_getOverriddenCursors`,")
w("  so overrides are matched by USR method signature along the class hierarchy. RTA (dispatch only to")
w("  classes whose constructor is reachable) is reported for information only; the CHA set is the plan.")
w("* **Dead debug branches.** `if` statements whose condition is provably false in this build are not")
w("  followed: never-written locals/file statics initialised to `FALSE`/`0` (the ubiquitous")
w("  `SmBoolean bDebugMe = FALSE; if (bDebugMe) {...}` pattern), `*Debug*` flags and `DebugLevel()`")
w("  comparisons against non-negative literals, combined with `!`/`&&`/`||`. 1135 such branches were")
w("  pruned; they only draw/dump/print (`Draw`, `smgfx_*`, `Dump`, file I/O of debug snapshots).")
w("* **Roots.** The SM_API entry points listed below plus `SmBrep::ReadFromFile` (the native .smb/.brp")
w("  reader that every prog_test and pt_TestFiles fixture goes through; needed to feed difftests).")
w("")
w("## Entry points")
w("")
for grp in list(ENTRY_GROUPS) + list(KERNEL_ROOTS):
    names = sorted(set(cl["entry_found"].get(grp, [])))
    w("* **%s** (%d): %s" % (grp, len(names), ", ".join("`%s`" % n for n in names)))
w("")
w("Out of scope (not roots; reachable code is only included if an in-scope root also reaches it):")
w(", ".join("`%s`" % x for x in OUT_OF_SCOPE_ENTRY) + ".")
w("")
w("Per-group closures (CHA, function lines; groups overlap heavily):")
w("")
gl = {}
for grp, rs in cl["group_reach"].items():
    gl[grp] = sum(L(F[u]) for u in rs if F[u]["file"].startswith("source/"))
w("| group | in-scope function lines |")
w("|---|---:|")
for grp in sorted(gl, key=lambda k: -gl[k]):
    w("| %s | %d |" % (grp, gl[grp]))
w("")
w("Note on tessellation: `SmTess`/`NL_Tessellate` are **not** needed by mass properties or any other")
w("in-scope operation (mass properties integrate exactly over faces, `SmBrep::ComputePreciseProperties`).")
w("The only reachable `SmTess.cpp`/`SmPoly*.cpp` code is CHA residue: `SmAttribute::Copy` dispatching")
w("to polygon/decimation attribute types, `SmBrepCache::BuildTrees` dispatching `CalculateBoundingBox`")
w("to `SmCPolyFace`, and `SmApiTransform` accepting a `SmPolyBrep`. Those units (`query.poly`,")
w("`query.tess`) are flagged low priority in `plan/modules.toml`.")
w("")
w("## Rust module totals")
w("")
w("| module | units | est. lines |")
w("|---|---:|---:|")
for m in sorted(S["module_lines"], key=lambda m: -S["module_lines"][m]):
    w("| %s | %d | %d |" % (m, S["module_units"][m], S["module_lines"][m]))
w("")
w("Smallest NLib leaf: `nlib.knots` (`nurbs::knots`, %d lines, no dependencies): %s." % (
    S["leaf_lines"], ", ".join("`%s`" % x for x in S["leaf_funcs"])))
w("")
w("## In-scope files")
w("")
w("`file lines` = physical lines; `fn lines` = lines inside function definitions; `in-scope` = CHA-reachable")
w("function lines; `RTA` = informational. Inline header functions are counted in the header.")
w("")
w("| file | file lines | fn lines | in-scope | RTA | groups |")
w("|---|---:|---:|---:|---:|---|")
for k in sorted(inscope_files, key=lambda k: -FILES[k]["cha_lines"]):
    v = FILES[k]
    w("| `%s` | %d | %d | %d | %d | %s |" % (k, v["lines"], v["func_lines"], v["cha_lines"], v["rta_lines"],
                                         ",".join(v["groups"])))
w("")
w("## Provably unused")
w("")
w("No function in these files is reachable from any in-scope root under CHA (header-only type/macro")
w("files with no function bodies are included; they may still be needed for declarations/constants):")
w("")
w("| file | lines | fn lines |")
w("|---|---:|---:|")
for k in sorted(unused_files, key=lambda k: (-FILES[k]["func_lines"], k)):
    v = FILES[k]
    w("| `%s` | %d | %d |" % (k, v["lines"], v["func_lines"]))
w("")
partial = sorted(((k, v["func_lines"] - v["cha_lines"], v["funcs"] - v["cha_funcs"]) for k, v in FILES.items()
                  if v["cha_lines"] > 0 and v["func_lines"] > v["cha_lines"]), key=lambda t: -t[1])
w("Unreachable functions inside in-scope files: %d functions, %d lines. Largest:" % (
    sum(t[2] for t in partial), sum(t[1] for t in partial)))
w("")
w("| file | unreachable fn lines | functions |")
w("|---|---:|---:|")
for k, ln, n in partial[:40]:
    w("| `%s` | %d | %d |" % (k, ln, n))
w("")
w("The full list of unreachable functions (file, line, name) is in `harness/callgraph/unused_functions.txt`.")
unused_fns = sorted((f["file"], f["line"], f["name"], L(f)) for u, f in F.items()
                    if f["file"].startswith("source/") and u not in CHA)
with open(os.path.join(HERE, "unused_functions.txt"), "w") as fh:
    fh.write("# file\tline\tlines\tfunction  (not reachable from in-scope roots under CHA)\n")
    for f_, ln, nm, n in unused_fns:
        fh.write("%s\t%d\t%d\t%s\n" % (f_, ln, n, nm))
w("")

# ---------------------------------------------------------------------------------------------
w("## Global state and nondeterminism hazards")
w("")
w("Only code in the in-scope closure is listed. Rust replacements: everything mutable moves into the")
w("`Kernel` value (tolerances, RNG, caches, counters) or into explicit per-operation context structs;")
w("pointer-keyed maps become arena-id keyed maps with insertion-ordered iteration (or `BTreeMap` on ids).")
w("")
H = S["hazards"]


def hz_table(kind, title, note):
    rows = H.get(kind, [])
    w("### %s (%d sites)" % (title, len(rows)))
    w("")
    w(note)
    w("")
    if not rows:
        return
    agg = collections.OrderedDict()
    for unit, fn, f_, ln, txt in rows:
        agg.setdefault((unit, fn, f_), []).append((ln, txt))
    w("| unit | function | file:lines | example |")
    w("|---|---|---|---|")
    for (unit, fn, f_), sites in agg.items():
        w("| `%s` | `%s` | `%s:%s` | `%s` |" % (unit, fn, f_.split("/")[-1], ",".join(str(s[0]) for s in sites[:6]),
                                              sites[0][1].replace("|", "\\|")[:90]))
    w("")


hz_table("rng", "rand()/srand(time)",
         "`SmFace::GetPointsInFace` reseeds the C RNG with `time(NULL)` and dithers sample V-parameters when "
         "`lPointCount > 1`. Results are **nondeterministic** in upstream for that path; callers in the closure "
         "(`SmAssertArray`, `SmFeatureExecutive`) request one point (deterministic branch). Rust: `Kernel` RNG "
         "(seeded); difftests must avoid `lPointCount > 1` or compare only invariants (record in findings.md).")
hz_table("clock", "clock()/time()",
         "Only the `time(NULL)` RNG seed above survives preprocessing. The `clock()` timing blocks in "
         "`SmMerge::MergeAddedTopology` and `SmTrimmingTools::CheckFace` sit inside `#ifdef SM_DEBUG_CODE` and are "
         "compiled out of the reference build. Lines inside inactive `#if` blocks are excluded from every scan here.")
hz_table("ptr-hash", "Pointer-keyed hash containers",
         "`SmMapPtrToPtr`/`std::unordered_*` keyed by object addresses. Lookups are order-independent, but any "
         "iteration (`GetAllKeys`, `GetAllValues`, `GetAllKeyValuePairs`, `Get1stKeyFor`, range-for) is "
         "address-ordered and therefore run-to-run unstable. Rust: key by arena ids and iterate in insertion "
         "order (`indexmap`-style) or sorted id order; where upstream iterates, record any ordering-driven "
         "divergence in findings.md.")
hz_table("ptr-iter", "Iteration over pointer-keyed containers (address-ordered)",
         "These sites enumerate a pointer-keyed map/set, so the visiting order (and anything order-dependent "
         "downstream: created topology order, tie-breaks, first-match results) depends on heap addresses. "
         "Each is a candidate determinism divergence vs upstream; the Rust port uses insertion order of arena "
         "ids. Difftests touching these paths should compare order-insensitively where upstream is unstable.")
hz_table("ptr-order", "Sorting / ordering that may involve pointers",
         "Sites using `qsort`/`std::sort`/pointer-to-integer casts inside the closure. Each needs a check "
         "that the comparator does not order by address.")
hz_table("thread_local", "Thread-local storage",
         "`SmThreadLocalStorage` (pthread TLS) tracks `operator new` placement bookkeeping and the error "
         "callback. Not needed in Rust (no placement new; errors are `Result`).")
w("### Mutable globals and statics")
w("")
w("Non-const namespace-scope variables, static members, and function-local statics in in-scope code")
w("(the ~%d `static NL_STRING rname = \"...\"` routine-name strings and the `sAssert*_list` validator" %
  sum(1 for a, fn, gl_ in S["mut_globals"] if gl_["name"] == "rname"))
w("label tables are omitted: never written after initialisation). `written` = the variable is assigned")
w("after its initialiser somewhere in its function (static locals) — i.e. state carried across calls.")
w("")
w("| unit | function | variable | type | file:line | written |")
w("|---|---|---|---|---|---|")
FILE_UNIT = {}
for uid, un in sorted(UNITS.items()):
    for fpath in un["upstream"]:
        FILE_UNIT.setdefault(fpath, uid)
for a, fn, gv in S["mut_globals"]:
    if not a:
        a = FILE_UNIT.get(gv["file"], "(not in a unit)")
    if gv["name"] == "rname" or gv["name"].startswith("sAssert") or gv["name"].startswith("sTrimming"):
        continue
    written = ""
    if gv["kind"] == "static_local" and gv["in"] in F:
        f = F[gv["in"]]
        rx = re.compile(r"(\b%s\s*(=(?!=)|\+=|-=|\*=|/=|\|=|&=|\+\+|--))|((\+\+|--|&)\s*%s\b)" % (
            re.escape(gv["name"]), re.escape(gv["name"])))
        lines = src(f["file"])[f["line"] - 1:f["end"]]
        n = 0
        dead_pp = inactive_lines(f["file"])
        for i, ln in enumerate(lines):
            if f["line"] + i == gv["line"] or (f["line"] + i) in dead_pp:
                continue
            code = ln.split("//")[0]
            if rx.search(code):
                n += 1
        written = "yes (%d)" % n if n else "no"
        decl = " ".join(src(gv["file"])[gv["line"] - 1:gv["line"] + 2])
        mo = re.search(r"\b%s\s*=\s*([^;,]*)" % re.escape(gv["name"]), decl)
        if mo and re.search(r"(->|\.[A-Za-z_]|\b[rp][A-Z]\w*|\bm_\w+|\w+\s*\()", mo.group(1)):
            written = (written + "; " if written else "") + "**runtime-initialised on first call**"
    w("| `%s` | `%s` | `%s` | `%s` | `%s:%d` | %s |" % (a, fn, gv["name"], gv["type"], gv["file"].split("/")[-1],
                                                 gv["line"], written))
w("")
w("Highlights:")
w("")
w("* `SmTol.cpp`: `s_ZoneTol3d`, `s_TolGapGain`, `s_ModelSizeEstimate`, `s_LargeSmallSizeRatio` are")
w("  process-wide tolerance state -> fields of `Kernel`.")
w("* `SmApiGeneral.cpp`: `pGlobalContext` (lazily created default `SmContext`) -> callers pass `&mut Kernel`.")
w("* NLib file statics used as hidden parameters between a driver and its callback (`NL_Geom.cpp`")
w("  `ST_mmxaps_*`, `NL_Math.cpp` `ST_EvalIso*`, `NL_Spiral.cpp` spiral parameters, `NL_SrfFit.cpp`")
w("  `pts2/p2/glo_np`, `NL_SrfOffset.cpp` `slm/totalmax`) -> explicit closure/context structs.")
w("* Static iteration/statistics counters (`itl`, `nok`, `s_lVolEvalCount`, `SM_AIH_FACE_NUMBER`) ->")
w("  dropped or moved into `Kernel` if they influence results.")
w("* `SmMessages.cpp` log flags and UI/escape callbacks -> not ported (errors are `Result`, no I/O).")
w("* Context traversal marks (`SmContext::m_lCurrentMark*`, see upstream `.agents/operations/concurrency.md`)")
w("  are per-context mutable state used by topology traversals -> `Kernel`/per-operation mark sets.")
w("")
w("## Threading (TBB)")
w("")
w("TBB appears only behind `SM_USE_TBB` / `SM_USE_TBB_SCALABLE`, which the reference build does not define")
w("(upstream docs: operations are single-threaded in the shipped build). Only the guard lines remain in")
w("compiled code; the `tbb::parallel_for` / `tbb::task_arena` bodies are preprocessed out. Guard sites:")
w("")
for unit, fn, f_, ln, txt in H.get("tbb", []):
    w("* `%s:%d` in `%s` (%s): `%s`" % (f_.split("/")[-1], ln, fn, unit, txt.replace("|", "\\|")))
w("")
w("The parallel paths in `SmTopologyIntersector::IntersectInsertRelate` / `IntersectTopology` and")
w("`SmBrep::ShrinkGeometry` have serial `#else` branches, which are what the reference build runs and what")
w("the Rust port reproduces (deterministic order). `SmMemory.cpp`/`NL_Util.cpp` only switch the allocator.")
w("")
w("## Caveats")
w("")
w("* CHA over-approximates: e.g. `SmSurface::Evaluate`-style dispatch makes every surface subclass")
w("  (including volume-deformation `SmCrvInVolume`/`SmSrfInVolume`/`SmBendVolume` types) in scope. Units")
w("  whose code mostly fails RTA are flagged in their notes.")
w("* Calls through macros are seen after expansion (e.g. `SER(...)`, `NER(...)`); calls inside")
w("  `#ifdef SM_DEBUG_CODE` are not compiled and not seen, matching the reference build.")
w("* Implicit destructor calls of stack objects are approximated by constructor -> destructor edges.")
w("")
open(os.path.join(ROOT, "plan", "closure.md"), "w").write("\n".join(out))
print("closure.md written:", len(out), "lines; unused fns", len(unused_fns))