kernel-cases: a regression rig that grades each kernel against the case's own answer

Every geometry case the old kernel got wrong becomes tests/kernel-cases/<area>/<id>/: an
engine script or oracle recipe, and expected.json with the right answer, from analysis or a
hand calculation, or else the invariants. Never a kernel's output.

- solidcraft-cli kernel-cases <dir>...: runs a case and grades it. The checks are the outcome
  (succeeds, or refuses cleanly), the body count, and one closed shell per body that passes the
  record's structure and geometry checks and, with SMLib built, SMLib's check(All). Counts,
  volume, area, centroid and box are graded where the case gives them. Each run is pass, fail,
  not supported yet or crash.
- cargo xtask kernel-cases [--kernel default|smlib|both]: one process per case, so a native crash
  or a hang (--timeout) is that case's result. It writes docs/kernel-cases.md: coverage per area
  and source, results per kernel, and the sources looked at and skipped (<area>/skipped.json).
- solidcraft_kernel::validity and smlib_built, which the rig's checks use.

tests/kernel-cases/README.md gives the format.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

apps/solidcraft-cli/src/kernel_cases.rs

@@ -0,0 +1,194 @@
+//! The kernel regression rig: replay `tests/kernel-cases/<area>/<id>/recipe.json` (an oracle
+//! recipe or a command script) and grade the result against the case's own `expected.json`:
+//! closed-form numbers where the case has them, and always the invariants (the operation's
+//! outcome, one closed valid solid per body, the expected counts). One kernel per run; kernels
+//! are never compared with each other. See tests/kernel-cases/README.md.
+
+use serde_json::{Value, json};
+
+use crate::oracle::{ABS_FRAC, Checks, REL_TOL, totals, v3};
+use crate::{read_json, recipe};
+
+/// The kernel this process runs booleans with (`SOLIDCRAFT_BOOLEANS`, set by the caller).
+fn kernel() -> String {
+    std::env::var("SOLIDCRAFT_BOOLEANS").ok().filter(|k| !k.is_empty()).unwrap_or_else(|| "default".into())
+}
+
+/// The case's commands: a script as is, or an oracle recipe translated; `$CASE` in a string is
+/// the case directory (for input files such as `input.step`).
+fn script(dir: &str, recipe: &Value) -> Result<Value, String> {
+    let s = if recipe.get("commands").is_some() || recipe.is_array() { recipe.clone() } else { recipe::to_script(recipe)? };
+    let text = serde_json::to_string(&s).map_err(|e| e.to_string())?;
+    let dir_json = serde_json::to_string(dir).map_err(|e| e.to_string())?;
+    let dir_inner = dir_json.get(1..dir_json.len().saturating_sub(1)).unwrap_or("");
+    serde_json::from_str(&text.replace("$CASE", dir_inner)).map_err(|e| e.to_string())
+}
+
+/// An error that means "this kernel can't do it yet", not a wrong answer.
+fn unsupported(e: &str) -> bool {
+    e.contains("not supported yet")
+}
+
+/// A panic caught by the engine or the kernel guard.
+fn crashed(e: &str) -> bool {
+    e.contains("internal error") || e.contains("panicked")
+}
+
+fn case_name(dir: &str) -> (String, String) {
+    let p = std::path::Path::new(dir);
+    let name = |q: Option<&std::path::Path>| q.and_then(|q| q.file_name()).map(|x| x.to_string_lossy().to_string()).unwrap_or_default();
+    (name(p.parent()), name(Some(p)))
+}
+
+fn check_case(dir: &str) -> Value {
+    let (area, name) = case_name(dir);
+    let mut out = json!({"case": name, "area": area, "kernel": kernel()});
+    let want = match read_json(&format!("{dir}/expected.json")) {
+        Ok(w) => w,
+        Err(e) => {
+            out["status"] = json!("fail");
+            out["error"] = json!(format!("expected.json: {e}"));
+            return out;
+        }
+    };
+    out["sources"] = want["sources"].clone();
+    let commands = match read_json(&format!("{dir}/recipe.json")).and_then(|r| script(dir, &r)) {
+        Ok(s) => s,
+        Err(e) => {
+            out["status"] = json!("fail");
+            out["error"] = json!(format!("recipe.json: {e}"));
+            return out;
+        }
+    };
+    if kernel() == "smlib" && !solidcraft_engine::kernel::smlib_built() {
+        out["status"] = json!("unsupported");
+        out["error"] = json!("this build has no SMLib: build solidcraft-cli with --features solidcraft-kernel/smlib");
+        return out;
+    }
+    let refused = want["outcome"].as_str() == Some("refused");
+    let mut s = solidcraft_engine::Session::default();
+    let ran = s.run_script(&commands);
+    match (ran, refused) {
+        (Err(e), _) if crashed(&e.to_string()) => {
+            out["status"] = json!("crash");
+            out["error"] = json!(e.to_string());
+            return out;
+        }
+        (Err(e), true) => {
+            // The right answer is a clean refusal.
+            out["status"] = json!("pass");
+            out["refused_with"] = json!(e.to_string());
+            return out;
+        }
+        (Err(e), false) => {
+            let e = e.to_string();
+            out["status"] = json!(if unsupported(&e) { "unsupported" } else { "fail" });
+            out["error"] = json!(e);
+            return out;
+        }
+        (Ok(_), true) => {
+            out["status"] = json!("fail");
+            out["error"] = json!("succeeded where the case expects a refusal");
+            return out;
+        }
+        (Ok(_), false) => {}
+    }
+
+    // Every body in the world, imported components included.
+    let world = s.world_state();
+    let mut bodies = Vec::new();
+    let mut c = Checks { list: Vec::new(), pass: true };
+    let want_valid = want["valid"].as_bool().unwrap_or(true);
+    for b in &world.bodies {
+        let m = match solidcraft_engine::kernel::measure(&b.body) {
+            Ok(m) => m,
+            Err(e) => {
+                c.push("measure", json!(format!("{}: {e}", b.name)), json!("measurable"), false);
+                continue;
+            }
+        };
+        bodies.push(json!({
+            "volume_mm3": m.volume, "area_mm2": m.area, "center_of_mass": m.centroid,
+            "bbox": {"min": m.bbox.min, "max": m.bbox.max},
+            "faces": m.merged.faces, "edges": m.merged.edges, "vertices": m.merged.vertices,
+        }));
+        if want_valid {
+            // One closed shell, and every validity check this build has.
+            c.push("shells", json!(format!("{}: {}", b.name, m.shells)), json!(1), m.shells == 1);
+            let v = solidcraft_engine::kernel::validity(&b.body);
+            c.push("record_valid", json!(v.record.clone().err().unwrap_or_else(|| "ok".into())), json!("ok"), v.record.is_ok());
+            if let Some(sm) = &v.smlib {
+                c.push("smlib_check", json!(sm.clone().err().unwrap_or_else(|| "ok".into())), json!("ok"), sm.is_ok());
+            }
+        }
+    }
+    let got = totals(&json!({"bodies": bodies}));
+    let (edges, vertices): (u64, u64) =
+        bodies.iter().fold((0, 0), |(e, v), b| (e + b["edges"].as_u64().unwrap_or(0), v + b["vertices"].as_u64().unwrap_or(0)));
+    let has = |k: &str| want.get(k).is_some_and(|v| !v.is_null());
+    let rel = want["tolerance"]["rel"].as_f64().unwrap_or(REL_TOL);
+    let diag = |b: ([f64; 3], [f64; 3])| ((b.1[0] - b.0[0]).powi(2) + (b.1[1] - b.0[1]).powi(2) + (b.1[2] - b.0[2]).powi(2)).sqrt();
+    let want_bbox = v3(&want["bbox"]["min"]).zip(v3(&want["bbox"]["max"]));
+    let abs = want["tolerance"]["abs_mm"].as_f64().unwrap_or_else(|| ABS_FRAC * want_bbox.map(diag).unwrap_or_else(|| diag(got.bbox)).max(1.0));
+    let count = |c: &mut Checks, k: &str, g: u64| {
+        if has(k) {
+            c.push(k, json!(g), want[k].clone(), want[k].as_u64() == Some(g));
+        }
+    };
+    count(&mut c, "body_count", got.body_count as u64);
+    count(&mut c, "faces", got.faces);
+    count(&mut c, "edges", edges);
+    count(&mut c, "vertices", vertices);
+    if has("volume_mm3") {
+        c.rel("volume_mm3", got.volume, want["volume_mm3"].as_f64(), rel);
+    }
+    if has("area_mm2") {
+        c.rel("area_mm2", got.area, want["area_mm2"].as_f64(), rel);
+    }
+    if has("center_of_mass") {
+        c.vec("center_of_mass", got.com, v3(&want["center_of_mass"]), abs);
+    }
+    if has("bbox") {
+        c.vec("bbox.min", got.bbox.0, want_bbox.map(|b| b.0), abs);
+        c.vec("bbox.max", got.bbox.1, want_bbox.map(|b| b.1), abs);
+    }
+    out["status"] = json!(if c.pass { "pass" } else { "fail" });
+    out["checks"] = json!(c.list);
+    out
+}
+
+fn report(args: &[String], results: &[Value]) {
+    if args.iter().any(|a| a == "--json") {
+        println!("{}", serde_json::to_string_pretty(&Value::Array(results.to_vec())).unwrap_or_default());
+        return;
+    }
+    for r in results {
+        let detail: Vec<String> = r["checks"]
+            .as_array()
+            .into_iter()
+            .flatten()
+            .filter(|c| !c["ok"].as_bool().unwrap_or(false))
+            .map(|c| format!("{} got {} want {}", c["check"].as_str().unwrap_or(""), c["got"], c["want"]))
+            .collect();
+        let err = r["error"].as_str().map(|e| format!(" ({e})")).unwrap_or_default();
+        println!(
+            "{} {}/{} [{}]{err}{}",
+            r["status"].as_str().unwrap_or("?").to_uppercase(),
+            r["area"].as_str().unwrap_or(""),
+            r["case"].as_str().unwrap_or(""),
+            r["kernel"].as_str().unwrap_or(""),
+            if detail.is_empty() { String::new() } else { format!(": {}", detail.join("; ")) }
+        );
+    }
+}
+
+pub fn run(args: &[String]) -> Result<(), String> {
+    let dirs: Vec<&String> = args.iter().filter(|a| !a.starts_with("--")).collect();
+    if dirs.is_empty() {
+        return Err("kernel-cases: give one or more case directories (each with recipe.json and expected.json)".into());
+    }
+    let results: Vec<Value> = dirs.iter().map(|d| check_case(d)).collect();
+    report(args, &results);
+    let bad = results.iter().filter(|r| matches!(r["status"].as_str(), Some("fail" | "crash"))).count();
+    if bad > 0 { Err(format!("{bad} kernel case(s) failed")) } else { Ok(()) }
+}

apps/solidcraft-cli/src/main.rs

@@ -15,6 +15,7 @@
 //! solidcraft-cli commands                                     the command registry as JSON
 //! solidcraft-cli recipe <recipe.json>                         translate an oracle recipe to a script
 //! solidcraft-cli oracle <case-dir>...                         replay recipes and compare with expected.json
+//! solidcraft-cli kernel-cases [--json] <case-dir>...           the kernel regression rig (tests/kernel-cases)
 //! solidcraft-cli mcp [--headless] [--in design|script] [--connect HOST:PORT [--token T]] [--allow-path DIR]... [--allow-any-path]
 //!                    [--param-check off|warn|strict]
 //!                                                             MCP server on stdio (docs/mcp.md)
@@ -31,6 +32,7 @@ mod bench_agent;
 mod bench_assistant;
 mod crashes;
 mod fuzz;
+mod kernel_cases;
 mod logger;
 mod oracle;
 mod preview;
@@ -79,6 +81,8 @@ const USAGE_COMMANDS: &str = "  solidcraft-cli commands\n";
 const USAGE_RECIPE: &str = "  solidcraft-cli recipe <recipe.json>\n";
 /// `oracle`'s lines in the usage.
 const USAGE_ORACLE: &str = "  solidcraft-cli oracle <case-dir>... [--json]\n";
+/// `kernel-cases`'s lines in the usage.
+const USAGE_KERNEL_CASES: &str = "  solidcraft-cli kernel-cases <case-dir>... [--json]\n      (the kernel regression rig, tests/kernel-cases; SOLIDCRAFT_BOOLEANS picks the kernel)\n";
 /// `fuzz`'s lines in the usage.
 const USAGE_FUZZ: &str =
     "  solidcraft-cli fuzz [--from N] [--count N] [--jobs J] [--timeout S] [--steps N] [--minimise] [--out DIR] [--report FILE]\n";
@@ -116,6 +120,7 @@ const COMMANDS: &[Command] = &[
     Command { name: "commands", usage: USAGE_COMMANDS, listed: true },
     Command { name: "recipe", usage: USAGE_RECIPE, listed: true },
     Command { name: "oracle", usage: USAGE_ORACLE, listed: true },
+    Command { name: "kernel-cases", usage: USAGE_KERNEL_CASES, listed: true },
     Command { name: "fuzz", usage: USAGE_FUZZ, listed: true },
     Command { name: "fuzz-one", usage: USAGE_FUZZ_ONE, listed: false },
     Command { name: "mcp", usage: USAGE_MCP, listed: true },
@@ -164,6 +169,7 @@ fn main() -> ExitCode {
             recipe::to_script(&v).map(|s| println!("{}", pretty(&s)))
         }),
         Some("oracle") => oracle::run(&args[1..]),
+        Some("kernel-cases") => kernel_cases::run(&args[1..]),
         Some("fuzz") => fuzz::run(&args[1..]),
         Some("fuzz-one") => fuzz::one(&args[1..]),
         Some("mcp") => cmd_mcp(&args[1..]),

apps/solidcraft-cli/src/oracle.rs

@@ -13,28 +13,28 @@ use crate::{read_json, recipe};
 /// Default relative tolerance for volume and area (our curved-face tessellation is within ~2e-4).
 pub const REL_TOL: f64 = 1e-3;
 /// Default tolerance for centre of mass and bounding box, as a fraction of the bbox diagonal.
-const ABS_FRAC: f64 = 1e-3;
+pub(crate) const ABS_FRAC: f64 = 1e-3;
 /// Chord tolerance used to triangulate our part for the surface-distance check (mm).
 const MESH_TOL: f64 = 0.01;
 /// Relative tolerance on the principal second moments of the two triangulations.
 const INERTIA_REL: f64 = 2e-3;
 
 /// What SolidCraft built: totals over every body.
-struct Got {
-    body_count: usize,
-    volume: f64,
-    area: f64,
-    com: [f64; 3],
-    bbox: ([f64; 3], [f64; 3]),
-    faces: u64,
+pub(crate) struct Got {
+    pub(crate) body_count: usize,
+    pub(crate) volume: f64,
+    pub(crate) area: f64,
+    pub(crate) com: [f64; 3],
+    pub(crate) bbox: ([f64; 3], [f64; 3]),
+    pub(crate) faces: u64,
 }
 
-fn v3(v: &Value) -> Option<[f64; 3]> {
+pub(crate) fn v3(v: &Value) -> Option<[f64; 3]> {
     let a = v.as_array()?;
     Some([a.first()?.as_f64()?, a.get(1)?.as_f64()?, a.get(2)?.as_f64()?])
 }
 
-fn totals(m: &Value) -> Got {
+pub(crate) fn totals(m: &Value) -> Got {
     let bodies = m["bodies"].as_array().cloned().unwrap_or_default();
     let (mut volume, mut area, mut faces) = (0.0, 0.0, 0);
     let mut moment = [0.0; 3];
@@ -57,21 +57,21 @@ fn totals(m: &Value) -> Got {
     Got { body_count: bodies.len(), volume, area, com, bbox: (lo, hi), faces }
 }
 
-struct Checks {
-    list: Vec<Value>,
-    pass: bool,
+pub(crate) struct Checks {
+    pub(crate) list: Vec<Value>,
+    pub(crate) pass: bool,
 }
 
 impl Checks {
-    fn push(&mut self, what: &str, got: Value, want: Value, ok: bool) {
+    pub(crate) fn push(&mut self, what: &str, got: Value, want: Value, ok: bool) {
         self.pass &= ok;
         self.list.push(json!({"check": what, "got": got, "want": want, "ok": ok}));
     }
-    fn rel(&mut self, what: &str, g: f64, w: Option<f64>, rel: f64) {
+    pub(crate) fn rel(&mut self, what: &str, g: f64, w: Option<f64>, rel: f64) {
         let ok = w.is_some_and(|w| g.is_finite() && (g - w).abs() <= rel * w.abs().max(1.0));
         self.push(what, json!(g), json!(w), ok);
     }
-    fn vec(&mut self, what: &str, g: [f64; 3], w: Option<[f64; 3]>, tol: f64) {
+    pub(crate) fn vec(&mut self, what: &str, g: [f64; 3], w: Option<[f64; 3]>, tol: f64) {
         let ok = w.is_some_and(|w| g.iter().zip(w).all(|(g, w)| g.is_finite() && (g - w).abs() <= tol));
         self.push(what, json!(g), json!(w), ok);
     }

crates/kernel/src/lib.rs

@@ -152,6 +152,7 @@ mod touching_walls;
 mod tspline;
 #[cfg(test)]
 mod untouched_faces_tests;
+mod validity;
 pub mod watch;
 mod whole_face;
 
@@ -223,6 +224,7 @@ pub use sweep_path::{RailFit, SweepControl, SweepOrientation, sweep_path, sweep_
 pub use topo::{CylinderFace, cylinder_end_at, cylinder_face, cylinder_face_at};
 pub use topo::{TopoCounts, merged_topology, seam_flags};
 pub use tspline::tspline_sheet;
+pub use validity::{Validity, smlib_built, validity};
 pub use whole_face::{WholeFace, cut_takes_only_the_face, cut_tool, face_prism, matching_face};
 
 #[derive(Debug, Clone, PartialEq, thiserror::Error)]

crates/kernel/src/validity.rs

@@ -0,0 +1,49 @@
+//! Is a body one valid solid? The kernel regression rig (`tests/kernel-cases`) asks this of every
+//! result: the schema 4 record's structure and geometry (`Brep::validate`, `check_geometry`),
+//! and, with the `smlib` feature, SMLib's own check of the same record.
+
+use crate::{Body, brep_record};
+
+/// What the validity checks found for one body. `Err` holds why a check failed.
+#[derive(Clone, Debug)]
+pub struct Validity {
+    /// The record: structure, then geometry against topology within tolerances.
+    pub record: Result<(), String>,
+    /// SMLib's `check(All)` of the record; `None` in a build without SMLib.
+    pub smlib: Option<Result<(), String>>,
+}
+
+impl Validity {
+    pub fn ok(&self) -> bool {
+        self.record.is_ok() && self.smlib.as_ref().is_none_or(Result::is_ok)
+    }
+}
+
+/// Runs every validity check this build has on `body`.
+pub fn validity(body: &Body) -> Validity {
+    let record = brep_record(body).map_err(|e| e.to_string());
+    let checked = record.as_ref().map_err(Clone::clone).and_then(|r| {
+        r.validate().map_err(|e| format!("structure: {e}"))?;
+        r.check_geometry().map_err(|e| format!("geometry: {e}"))
+    });
+    Validity { record: checked, smlib: smlib_validity(body) }
+}
+
+#[cfg(feature = "smlib")]
+fn smlib_validity(body: &Body) -> Option<Result<(), String>> {
+    let checked = crate::smlib_body(body).map_err(|e| e.to_string()).and_then(|b| {
+        let r = b.check(solidcraft_smlib::CheckLevel::All).map_err(|e| format!("SMLib check: {e}"))?;
+        if r.is_valid() { Ok(()) } else { Err(format!("SMLib check: {r}")) }
+    });
+    Some(checked)
+}
+
+#[cfg(not(feature = "smlib"))]
+fn smlib_validity(_: &Body) -> Option<Result<(), String>> {
+    None
+}
+
+/// Whether this build links SMLib (the `smlib` feature).
+pub fn smlib_built() -> bool {
+    cfg!(feature = "smlib")
+}

tests/kernel-cases/README.md

@@ -0,0 +1,93 @@
+# Kernel regression rig
+
+Every geometry case the old kernel (truck) got wrong, recorded in an issue, a pull request or a
+crash report, is a use case here with its own right answer. The rig holds every kernel to all of
+them. `cargo xtask kernel-cases` runs them and writes [`docs/kernel-cases.md`](../../docs/kernel-cases.md).
+
+```sh
+cargo xtask kernel-cases                        # the default kernel
+cargo xtask kernel-cases --kernel both --cli target/release/solidcraft-cli   # and SMLib
+cargo run -p solidcraft-cli -- kernel-cases tests/kernel-cases/booleans/<id>     # one case
+```
+
+For the `smlib` column, build `solidcraft-cli` with `--features solidcraft-kernel/smlib` (after
+`scripts/fetch-smlib.sh`). The rig sets `SOLIDCRAFT_BOOLEANS=smlib` for those runs.
+
+## Rules
+
+- **The answer never comes from a kernel.** Not from truck's output, nor any commercial
+  kernel's. It comes from analysis or a hand calculation, or else from the invariants below.
+- **Kernels are graded against the case, never against each other.**
+- **No user designs.** A crash report or a repro design becomes a small synthetic recipe that
+  shows the same geometry. Check for private content before committing anything.
+- **No workaround on SolidCraft's side.** A case that fails on a kernel stays failing until that
+  kernel is fixed. SMLib defects go to tangency-labs/solidcore as issues (`priority: high`,
+  `bug`), and the fix goes into the fork.
+
+## Layout
+
+One directory per area, one per case: `tests/kernel-cases/<area>/<id>/`.
+
+| Area | Owner |
+|---|---|
+| `booleans` (combine, cuts, holes, patterns, split, mirror, draft) | booleans/shell/import/crash rig |
+| `shell` (shell, offset, thicken) | booleans/shell/import/crash rig |
+| `import` (STEP and IGES geometry) | booleans/shell/import/crash rig |
+| `crashes` (crashes and hangs inside kernel operations) | booleans/shell/import/crash rig |
+| `fillets`, `sweeps`, `threads`, `sheet-metal`, `tessellation` | history rig |
+
+`<id>` is `<source number>-<short-name>`, such as `1894-prism-cut-touching-walls`.
+
+Each case has:
+
+- `recipe.json`: an engine command script (`{"commands": [...]}`, as `solidcraft-cli run`
+  takes) or an oracle recipe (`tests/oracle/*/recipe.json`). `$CASE` in a string is the case
+  directory, for an input file such as `$CASE/input.step` (written by hand from the STEP or IGES
+  specification, never a user's file).
+- `expected.json`: the right answer, graded by `solidcraft-cli kernel-cases`.
+
+Each area also has `skipped.json`: every issue and PR looked at that did not become a case, with
+one line each saying why.
+
+```json
+[{"source": "tangency-labs/solidcraft#123", "why": "UI only: the selection highlight, no geometry"}]
+```
+
+## expected.json
+
+```json
+{
+  "sources": ["tangency-labs/solidcraft#1894", "tangency-labs/solidcraft#1629"],
+  "what": "a prism cut whose walls only touch the target's walls",
+  "source": "analytic",
+  "method": "box 40x30x20 minus the prism 10x30x5 flush with the side: V = 24000 - 1500",
+  "body_count": 1,
+  "volume_mm3": 22500.0,
+  "area_mm2": 5200.0,
+  "faces": 8
+}
+```
+
+| Field | Meaning |
+|---|---|
+| `sources` | The issues, PRs or crash reports the case comes from (`owner/repo#N`). |
+| `what` | One line: the geometry that went wrong. |
+| `source` | `analytic` (numbers worked out by hand; `method` gives the formulas) or `invariants` (no number is known: only the outcome, validity and counts are graded). |
+| `method` | How the numbers were derived, or which invariants hold and why. |
+| `outcome` | `solid` (default: the script succeeds) or `refused` (the right answer is a clean error, never a panic or a hang). |
+| `valid` | Default `true`: every body is one closed shell that passes the record's structure and geometry checks and, in builds with SMLib, SMLib's `check(All)`. `false` only where the result is legitimately not a solid. |
+| `body_count`, `faces`, `edges`, `vertices` | Exact counts, where the geometry fixes them. Faces and edges count whole analytic surfaces and curves once (a full cylinder side is one face). |
+| `volume_mm3`, `area_mm2`, `center_of_mass`, `bbox` | As in `tests/oracle`: totals over every body, relative tolerance 1e-3 for volume and area, 1e-3 of the box diagonal for positions. |
+| `tolerance` | Optional overrides: `rel`, `abs_mm`. |
+
+## Results
+
+Per case and kernel: **pass**; **fail** with the numbers; **not supported yet** (the kernel says
+so); **crash** (a panic caught by the guards, or the process died); **hang** (no result before
+the timeout).
+
+## Exporting to solidcore
+
+solidcore#349 proposes a graded case format for the Rust port. The cases here are SolidCraft's
+copy and the source; each one maps onto it field by field (`sources`, the derivation, the
+graded quantities and invariants), so an exporter can write them out for solidcore unchanged.

xtask/src/kernel_cases.rs

@@ -0,0 +1,311 @@
+//! `cargo xtask kernel-cases`: the kernel regression rig. Runs every case in
+//! `tests/kernel-cases/<area>/<id>/` through `solidcraft-cli kernel-cases`, one process per case
+//! (so a crash or a hang is that case's result, not the run's), with each kernel asked for, and
+//! writes `docs/kernel-cases.md`: results per case and kernel, coverage per area and per source,
+//! and the issues and PRs looked at and skipped. Each kernel is graded against the cases' own
+//! answers, never against the other kernel.
+
+use std::collections::BTreeMap;
+use std::path::{Path, PathBuf};
+use std::process::{Command, Stdio};
+use std::sync::{Arc, Mutex};
+use std::time::{Duration, Instant};
+
+use serde_json::{Value, json};
+
+use crate::{cargo, root};
+
+const DEFAULT_TIMEOUT_S: u64 = 180;
+
+/// Per kernel, how many cases ended in each status.
+type StatusCounts = BTreeMap<String, BTreeMap<String, usize>>;
+
+struct Options {
+    kernels: Vec<String>,
+    cli: Option<String>,
+    area: Option<String>,
+    timeout: Duration,
+    jobs: usize,
+}
+
+fn options(rest: &[&str]) -> Result<Options, String> {
+    let arg = |name: &str| rest.iter().position(|a| *a == name).and_then(|i| rest.get(i + 1)).map(|s| s.to_string());
+    let kernels = match arg("--kernel").as_deref() {
+        None | Some("default") => vec!["default".to_string()],
+        Some("smlib") => vec!["smlib".to_string()],
+        Some("both") => vec!["default".to_string(), "smlib".to_string()],
+        Some(k) => return Err(format!("--kernel {k}: default, smlib or both")),
+    };
+    let secs = arg("--timeout").map(|t| t.parse::<u64>().map_err(|e| format!("--timeout: {e}"))).transpose()?.unwrap_or(DEFAULT_TIMEOUT_S);
+    let jobs = arg("--jobs").map(|j| j.parse::<usize>().map_err(|e| format!("--jobs: {e}"))).transpose()?.unwrap_or(4).clamp(1, 32);
+    Ok(Options { kernels, cli: arg("--cli"), area: arg("--area"), timeout: Duration::from_secs(secs), jobs })
+}
+
+fn read(path: &Path) -> Value {
+    std::fs::read_to_string(path).ok().and_then(|t| serde_json::from_str(&t).ok()).unwrap_or(Value::Null)
+}
+
+/// Every case directory, sorted by area then id.
+fn cases(dir: &Path, only: Option<&str>) -> Result<Vec<PathBuf>, String> {
+    let mut out = Vec::new();
+    for area in std::fs::read_dir(dir).map_err(|e| format!("{}: {e}", dir.display()))?.flatten() {
+        let area = area.path();
+        if !area.is_dir() || only.is_some_and(|a| area.file_name().is_none_or(|n| n != a)) {
+            continue;
+        }
+        for case in std::fs::read_dir(&area).map_err(|e| format!("{}: {e}", area.display()))?.flatten() {
+            let case = case.path();
+            if case.join("recipe.json").is_file() && case.join("expected.json").is_file() {
+                out.push(case);
+            }
+        }
+    }
+    out.sort();
+    Ok(out)
+}
+
+/// One case on one kernel, in its own process, killed after the timeout.
+fn run_one(o: &Options, kernel: &str, case: &Path) -> Value {
+    let rel = case.strip_prefix(root()).unwrap_or(case).to_path_buf();
+    let mut cmd = match &o.cli {
+        Some(bin) => Command::new(bin),
+        None => {
+            let mut c = cargo();
+            c.args(["run", "-q", "--release", "-p", "solidcraft-cli", "--"]);
+            c
+        }
+    };
+    cmd.current_dir(root()).args(["kernel-cases", "--json"]).arg(&rel).stdout(Stdio::piped()).stderr(Stdio::piped());
+    if kernel == "default" {
+        cmd.env_remove("SOLIDCRAFT_BOOLEANS");
+    } else {
+        cmd.env("SOLIDCRAFT_BOOLEANS", kernel);
+    }
+    let base = |status: &str, error: String| {
+        let area = case.parent().and_then(|p| p.file_name()).map(|n| n.to_string_lossy().to_string()).unwrap_or_default();
+        let id = case.file_name().map(|n| n.to_string_lossy().to_string()).unwrap_or_default();
+        json!({"case": id, "area": area, "kernel": kernel, "status": status, "error": error})
+    };
+    let mut child = match cmd.spawn() {
+        Ok(c) => c,
+        Err(e) => return base("fail", format!("solidcraft-cli: {e}")),
+    };
+    let start = Instant::now();
+    loop {
+        match child.try_wait() {
+            Ok(Some(_)) => break,
+            Ok(None) if start.elapsed() > o.timeout => {
+                let _ = child.kill();
+                let _ = child.wait();
+                return base("hang", format!("no result after {} s", o.timeout.as_secs()));
+            }
+            Ok(None) => std::thread::sleep(Duration::from_millis(50)),
+            Err(e) => return base("fail", format!("wait: {e}")),
+        }
+    }
+    let out = match child.wait_with_output() {
+        Ok(o) => o,
+        Err(e) => return base("fail", format!("output: {e}")),
+    };
+    match serde_json::from_slice::<Value>(&out.stdout).ok().and_then(|v| v.as_array().and_then(|a| a.first().cloned())) {
+        Some(mut r) => {
+            r["seconds"] = json!(start.elapsed().as_secs_f64());
+            r
+        }
+        // No JSON: the process died (a native crash, an abort).
+        None => {
+            let err = String::from_utf8_lossy(&out.stderr);
+            let tail: String = err.lines().rev().take(3).collect::<Vec<_>>().into_iter().rev().collect::<Vec<_>>().join(" / ");
+            base("crash", format!("process exited ({}) without a result: {tail}", out.status))
+        }
+    }
+}
+
+fn run_all(o: &Options, list: &[PathBuf]) -> Vec<Value> {
+    let jobs: Vec<(String, PathBuf)> = o.kernels.iter().flat_map(|k| list.iter().map(move |c| (k.clone(), c.clone()))).collect();
+    let next = Arc::new(Mutex::new(0usize));
+    let results = Arc::new(Mutex::new(vec![Value::Null; jobs.len()]));
+    std::thread::scope(|s| {
+        for _ in 0..o.jobs.min(jobs.len().max(1)) {
+            let (next, results, jobs) = (Arc::clone(&next), Arc::clone(&results), &jobs);
+            s.spawn(move || {
+                loop {
+                    let i = {
+                        let Ok(mut n) = next.lock() else { return };
+                        let i = *n;
+                        *n += 1;
+                        i
+                    };
+                    let Some((kernel, case)) = jobs.get(i) else { return };
+                    let r = run_one(o, kernel, case);
+                    println!(
+                        "{:<11} {} [{kernel}]",
+                        r["status"].as_str().unwrap_or("?").to_uppercase(),
+                        case.strip_prefix(root()).unwrap_or(case).display()
+                    );
+                    if let Ok(mut all) = results.lock()
+                        && let Some(slot) = all.get_mut(i)
+                    {
+                        *slot = r;
+                    }
+                }
+            });
+        }
+    });
+    Arc::try_unwrap(results).ok().and_then(|m| m.into_inner().ok()).unwrap_or_default()
+}
+
+fn cell(r: Option<&Value>) -> String {
+    let Some(r) = r else { return String::new() };
+    let status = r["status"].as_str().unwrap_or("?");
+    let mut notes: Vec<String> = r["checks"]
+        .as_array()
+        .into_iter()
+        .flatten()
+        .filter(|c| !c["ok"].as_bool().unwrap_or(false))
+        .map(|c| format!("{} {} (want {})", c["check"].as_str().unwrap_or(""), short(&c["got"]), short(&c["want"])))
+        .collect();
+    if let Some(e) = r["error"].as_str() {
+        notes.push(e.chars().take(160).collect());
+    }
+    let s = match status {
+        "pass" => "pass".to_string(),
+        "unsupported" => "not supported yet".to_string(),
+        other => format!("**{}**", other.to_uppercase()),
+    };
+    if notes.is_empty() { s } else { format!("{s}: {}", notes.join("; ")).replace('|', "/").replace('\n', " ") }
+}
+
+fn short(v: &Value) -> String {
+    match v {
+        Value::Number(n) => n.as_f64().map(|x| format!("{x:.6}").trim_end_matches('0').trim_end_matches('.').to_string()).unwrap_or_default(),
+        Value::String(s) => s.chars().take(100).collect(),
+        Value::Array(a) => format!("[{}]", a.iter().map(short).collect::<Vec<_>>().join(", ")),
+        other => other.to_string(),
+    }
+}
+
+fn sources_of(want: &Value) -> Vec<String> {
+    want["sources"].as_array().into_iter().flatten().filter_map(|s| s.as_str().map(str::to_string)).collect()
+}
+
+/// `tangency-labs/solidcraft#123` as a link.
+fn link(source: &str) -> String {
+    match source.split_once('#') {
+        Some((repo, n)) if repo.contains('/') => format!("[{source}](https://github.com/{repo}/issues/{n})"),
+        _ => source.to_string(),
+    }
+}
+
+fn write_report(o: &Options, dir: &Path, list: &[PathBuf], results: &[Value]) -> Result<String, String> {
+    let key = |r: &Value| (r["area"].as_str().unwrap_or("").to_string(), r["case"].as_str().unwrap_or("").to_string());
+    let mut by_case: BTreeMap<(String, String), BTreeMap<String, Value>> = BTreeMap::new();
+    for r in results {
+        by_case.entry(key(r)).or_default().insert(r["kernel"].as_str().unwrap_or("").to_string(), r.clone());
+    }
+    // Coverage: per area, cases and their statuses per kernel; per source, its cases.
+    let mut areas: BTreeMap<String, (usize, StatusCounts)> = BTreeMap::new();
+    let mut per_source: BTreeMap<String, Vec<String>> = BTreeMap::new();
+    let mut kinds: BTreeMap<String, usize> = BTreeMap::new();
+    let mut rows = String::new();
+    for case in list {
+        let want = read(&case.join("expected.json"));
+        let area = case.parent().and_then(|p| p.file_name()).map(|n| n.to_string_lossy().to_string()).unwrap_or_default();
+        let id = case.file_name().map(|n| n.to_string_lossy().to_string()).unwrap_or_default();
+        let entry = areas.entry(area.clone()).or_default();
+        entry.0 += 1;
+        let runs = by_case.get(&(area.clone(), id.clone()));
+        for k in &o.kernels {
+            let status = runs.and_then(|r| r.get(k)).and_then(|r| r["status"].as_str()).unwrap_or("not run").to_string();
+            *entry.1.entry(k.clone()).or_default().entry(status).or_default() += 1;
+        }
+        let sources = sources_of(&want);
+        for s in &sources {
+            per_source.entry(s.clone()).or_default().push(format!("{area}/{id}"));
+        }
+        *kinds.entry(want["source"].as_str().unwrap_or("?").split_whitespace().next().unwrap_or("?").to_string()).or_default() += 1;
+        let cells: Vec<String> = o.kernels.iter().map(|k| cell(runs.and_then(|r| r.get(k)))).collect();
+        rows += &format!(
+            "| {area}/{id} | {} | {} | {} | {} |\n",
+            sources.iter().map(|s| link(s)).collect::<Vec<_>>().join(" "),
+            want["what"].as_str().unwrap_or("").replace('|', "/"),
+            want["source"].as_str().unwrap_or("?"),
+            cells.join(" | ")
+        );
+    }
+    let mut md = String::from(
+        "# Kernel regression rig\n\nGenerated by `cargo xtask kernel-cases`. Every geometry case the old kernel (truck) got wrong, as an issue, a PR or a crash report recorded it, is a use case in [`tests/kernel-cases/`](../tests/kernel-cases/) with its own right answer: numbers from analysis or a hand calculation (**analytic**) where the geometry has them, and otherwise the invariants (**invariants**): the operation succeeds, or refuses cleanly where that is the right answer; each body is one closed shell that passes the record's structure and geometry checks and, in builds with SMLib, SMLib's own check; the expected face, edge and vertex counts. Never a kernel's output, and never a commercial kernel's.\n\nEach kernel is graded against the case, never against the other kernel. **default** is the kernel SolidCraft ships today; **smlib** runs with `SOLIDCRAFT_BOOLEANS=smlib`. A failing case stays failing until its kernel is fixed; there is no workaround on SolidCraft's side. See [`tests/kernel-cases/README.md`](../tests/kernel-cases/README.md) for the format.\n\n",
+    );
+    md += "## Coverage\n\n| Area | Cases |";
+    for k in &o.kernels {
+        md += &format!(" {k} |");
+    }
+    md += "\n|---|---|";
+    md += &"---|".repeat(o.kernels.len());
+    md += "\n";
+    for (area, (n, statuses)) in &areas {
+        md += &format!("| {area} | {n} |");
+        for k in &o.kernels {
+            let s = statuses.get(k).map(|m| m.iter().map(|(s, c)| format!("{c} {s}")).collect::<Vec<_>>().join(", ")).unwrap_or_default();
+            md += &format!(" {s} |");
+        }
+        md += "\n";
+    }
+    md += &format!(
+        "\n{} cases from {} sources; answers: {}.\n\n## Results\n\n| Case | Sources | What went wrong | Answer |",
+        list.len(),
+        per_source.len(),
+        kinds.iter().map(|(k, v)| format!("{v} {k}")).collect::<Vec<_>>().join(", ")
+    );
+    for k in &o.kernels {
+        md += &format!(" {k} |");
+    }
+    md += "\n|---|---|---|---|";
+    md += &"---|".repeat(o.kernels.len());
+    md += "\n";
+    md += &rows;
+    md += "\n## Sources\n\n| Source | Cases |\n|---|---|\n";
+    for (s, cs) in &per_source {
+        md += &format!("| {} | {} |\n", link(s), cs.join(", "));
+    }
+    // What was looked at and skipped, area by area (`<area>/skipped.json`).
+    md += "\n## Looked at and skipped\n\n";
+    let mut skipped = 0;
+    let mut area_dirs: Vec<PathBuf> = std::fs::read_dir(dir).map_err(|e| e.to_string())?.flatten().map(|e| e.path()).filter(|p| p.is_dir()).collect();
+    area_dirs.sort();
+    for a in area_dirs {
+        let list = read(&a.join("skipped.json"));
+        let Some(items) = list.as_array().filter(|l| !l.is_empty()) else { continue };
+        md += &format!("### {}\n\n| Source | Why skipped |\n|---|---|\n", a.file_name().map(|n| n.to_string_lossy().to_string()).unwrap_or_default());
+        for it in items {
+            skipped += 1;
+            md += &format!("| {} | {} |\n", link(it["source"].as_str().unwrap_or("?")), it["why"].as_str().unwrap_or("").replace('|', "/"));
+        }
+        md += "\n";
+    }
+    if skipped == 0 {
+        md += "None yet.\n";
+    }
+    std::fs::write(root().join("docs/kernel-cases.md"), md).map_err(|e| format!("docs/kernel-cases.md: {e}"))?;
+    let summary: Vec<String> = o
+        .kernels
+        .iter()
+        .map(|k| {
+            let n = results.iter().filter(|r| r["kernel"] == k.as_str() && r["status"] == "pass").count();
+            format!("{k}: {n}/{} pass", list.len())
+        })
+        .collect();
+    Ok(format!("kernel-cases: {} → docs/kernel-cases.md ({skipped} skipped sources listed)", summary.join(", ")))
+}
+
+pub fn run(rest: &[&str]) -> Result<(), String> {
+    let o = options(rest)?;
+    let dir = root().join("tests/kernel-cases");
+    let list = cases(&dir, o.area.as_deref())?;
+    if list.is_empty() {
+        return Err(format!("no cases (recipe.json + expected.json) under {}", dir.display()));
+    }
+    let results = run_all(&o, &list);
+    println!("{}", write_report(&o, &dir, &list, &results)?);
+    Ok(())
+}

xtask/src/main.rs

@@ -10,6 +10,7 @@ mod external_corpus;
 mod features;
 mod i18n;
 mod icons;
+mod kernel_cases;
 mod layers;
 mod licences;
 mod lists;
@@ -45,6 +46,10 @@ commands:
                   replay the reference tests (tests/oracle/*/recipe.json), compare with each
                   case's expected.json and write docs/oracle.md (--cli: a solidcraft-cli
                   already built, instead of cargo run --release)
+  kernel-cases [--kernel default|smlib|both] [--area AREA] [--cli BIN] [--timeout S] [--jobs N]
+                  the kernel regression rig: run every case in tests/kernel-cases/<area>/<id>/ (one
+                  process each, killed after --timeout, default 180 s) with each kernel asked for,
+                  grade it against the case's own answer and write docs/kernel-cases.md
   action-matrix [--bless] [--results FILE] [--from DIR]
                   try every ordinary action on every face, edge, vertex and body of the design
                   corpus and compare with tests/action_matrix/baseline.json (--bless rewrites it;
@@ -102,6 +107,7 @@ fn main() -> ExitCode {
         Some("app-icons") => app_icons::run(&root()),
         Some("features") => features::run(&root()),
         Some("i18n") => i18n::run(&root(), &rest),
+        Some("kernel-cases") => kernel_cases::run(&rest),
         Some("oracle") => cmd_oracle(rest.iter().position(|a| *a == "--cli").and_then(|i| rest.get(i + 1)).copied()),
         Some("action-matrix") => {
             let arg = |name: &str| rest.iter().position(|a| *a == name).and_then(|i| rest.get(i + 1)).copied();