kernel-cases: shells as a graded count, no SMLib verdict where it can't take the record, $OUT

- `"shells": N` in expected.json grades the total number of shells: a body with a sealed void
  holds two, and some joins keep several lumps in one body. Those cases (#3175, #3455, #3526,
  #3730) are graded on validity again instead of being marked not valid.
- When SMLib's record conversion says "not supported yet" (revolved surfaces, traced intersection
  curves), its check gives no verdict. That's shown but not graded, since it isn't a fault of the
  body. solidcraft_kernel::SmlibCheck carries the three outcomes.
- `$OUT` in a recipe is a scratch directory outside the repository, for files a case writes. The
  two STEP-export crash cases write there instead of /tmp.
- cargo xtask kernel-cases --results FILE keeps every result as JSON.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

apps/solidcraft-cli/src/kernel_cases.rs

@@ -14,14 +14,24 @@ fn kernel() -> String {
     std::env::var("SOLIDCRAFT_BOOLEANS").ok().filter(|k| !k.is_empty()).unwrap_or_else(|| "default".into())
 }
 
-/// The case's commands: a script as is, or an oracle recipe translated; `$CASE` in a string is
-/// the case directory (for input files such as `input.step`).
-fn script(dir: &str, recipe: &Value) -> Result<Value, String> {
+/// The case's commands: a script as is, or an oracle recipe translated. In a string, `$CASE` is
+/// the case directory (input files such as `input.step`) and `$OUT` a scratch directory for files
+/// the case writes (an export), outside the repository.
+fn script(dir: &str, out: &str, recipe: &Value) -> Result<Value, String> {
     let s = if recipe.get("commands").is_some() || recipe.is_array() { recipe.clone() } else { recipe::to_script(recipe)? };
     let text = serde_json::to_string(&s).map_err(|e| e.to_string())?;
-    let dir_json = serde_json::to_string(dir).map_err(|e| e.to_string())?;
-    let dir_inner = dir_json.get(1..dir_json.len().saturating_sub(1)).unwrap_or("");
-    serde_json::from_str(&text.replace("$CASE", dir_inner)).map_err(|e| e.to_string())
+    let inner = |p: &str| -> Result<String, String> {
+        let j = serde_json::to_string(p).map_err(|e| e.to_string())?;
+        Ok(j.get(1..j.len().saturating_sub(1)).unwrap_or("").to_string())
+    };
+    serde_json::from_str(&text.replace("$CASE", &inner(dir)?).replace("$OUT", &inner(out)?)).map_err(|e| e.to_string())
+}
+
+/// A scratch directory for one case's output files.
+fn scratch(area: &str, name: &str) -> Result<String, String> {
+    let d = std::env::temp_dir().join(format!("solidcraft-kernel-cases-{}", std::process::id())).join(format!("{area}-{name}"));
+    std::fs::create_dir_all(&d).map_err(|e| format!("{}: {e}", d.display()))?;
+    Ok(d.to_string_lossy().to_string())
 }
 
 /// An error that means "this kernel can't do it yet", not a wrong answer.
@@ -52,7 +62,7 @@ fn check_case(dir: &str) -> Value {
         }
     };
     out["sources"] = want["sources"].clone();
-    let commands = match read_json(&format!("{dir}/recipe.json")).and_then(|r| script(dir, &r)) {
+    let commands = match scratch(&area, &name).and_then(|o| read_json(&format!("{dir}/recipe.json")).and_then(|r| script(dir, &o, &r))) {
         Ok(s) => s,
         Err(e) => {
             out["status"] = json!("fail");
@@ -105,6 +115,10 @@ fn check_case(dir: &str) -> Value {
     let mut bodies = Vec::new();
     let mut c = Checks { list: Vec::new(), pass: true };
     let want_valid = want["valid"].as_bool().unwrap_or(true);
+    // Shells: the total where the case gives it (a body with a void, or several lumps in one
+    // body), else one per body.
+    let want_shells = want["shells"].as_u64();
+    let mut shells = 0u64;
     for b in &world.bodies {
         let m = match solidcraft_engine::kernel::measure(&b.body) {
             Ok(m) => m,
@@ -118,13 +132,22 @@ fn check_case(dir: &str) -> Value {
             "bbox": {"min": m.bbox.min, "max": m.bbox.max},
             "faces": m.merged.faces, "edges": m.merged.edges, "vertices": m.merged.vertices,
         }));
+        shells += m.shells as u64;
         if want_valid {
-            // One closed shell, and every validity check this build has.
-            c.push("shells", json!(format!("{}: {}", b.name, m.shells)), json!(1), m.shells == 1);
+            // Closed shells, and every validity check this build has.
+            if want_shells.is_none() {
+                c.push("shells", json!(format!("{}: {}", b.name, m.shells)), json!(1), m.shells == 1);
+            }
             let v = solidcraft_engine::kernel::validity(&b.body);
             c.push("record_valid", json!(v.record.clone().err().unwrap_or_else(|| "ok".into())), json!("ok"), v.record.is_ok());
-            if let Some(sm) = &v.smlib {
-                c.push("smlib_check", json!(sm.clone().err().unwrap_or_else(|| "ok".into())), json!("ok"), sm.is_ok());
+            match &v.smlib {
+                Some(solidcraft_engine::kernel::SmlibCheck::Valid) => c.push("smlib_check", json!("ok"), json!("ok"), true),
+                Some(solidcraft_engine::kernel::SmlibCheck::Invalid(e)) => c.push("smlib_check", json!(e), json!("ok"), false),
+                // No verdict: the record can't be built in SMLib yet. Shown, not graded.
+                Some(solidcraft_engine::kernel::SmlibCheck::NotSupported(e)) => {
+                    c.push("smlib_check", json!(format!("no verdict: {e}")), json!("ok"), true)
+                }
+                None => {}
             }
         }
     }
@@ -142,6 +165,7 @@ fn check_case(dir: &str) -> Value {
         }
     };
     count(&mut c, "body_count", got.body_count as u64);
+    count(&mut c, "shells", shells);
     count(&mut c, "faces", got.faces);
     count(&mut c, "edges", edges);
     count(&mut c, "vertices", vertices);

crates/kernel/src/lib.rs

@@ -224,7 +224,7 @@ pub use sweep_path::{RailFit, SweepControl, SweepOrientation, sweep_path, sweep_
 pub use topo::{CylinderFace, cylinder_end_at, cylinder_face, cylinder_face_at};
 pub use topo::{TopoCounts, merged_topology, seam_flags};
 pub use tspline::tspline_sheet;
-pub use validity::{Validity, boolean_kernel, smlib_built, validity};
+pub use validity::{SmlibCheck, Validity, boolean_kernel, smlib_built, validity};
 pub use whole_face::{WholeFace, cut_takes_only_the_face, cut_tool, face_prism, matching_face};
 
 #[derive(Debug, Clone, PartialEq, thiserror::Error)]

crates/kernel/src/validity.rs

@@ -10,12 +10,22 @@ pub struct Validity {
     /// The record: structure, then geometry against topology within tolerances.
     pub record: Result<(), String>,
     /// SMLib's `check(All)` of the record; `None` in a build without SMLib.
-    pub smlib: Option<Result<(), String>>,
+    pub smlib: Option<SmlibCheck>,
+}
+
+/// SMLib's verdict on a body.
+#[derive(Clone, Debug)]
+pub enum SmlibCheck {
+    Valid,
+    Invalid(String),
+    /// The record can't be built in SMLib yet (a geometry kind the conversion doesn't take): no
+    /// verdict, and no fault of the body's.
+    NotSupported(String),
 }
 
 impl Validity {
     pub fn ok(&self) -> bool {
-        self.record.is_ok() && self.smlib.as_ref().is_none_or(Result::is_ok)
+        self.record.is_ok() && !matches!(self.smlib, Some(SmlibCheck::Invalid(_)))
     }
 }
 
@@ -30,16 +40,23 @@ pub fn validity(body: &Body) -> Validity {
 }
 
 #[cfg(feature = "smlib")]
-fn smlib_validity(body: &Body) -> Option<Result<(), String>> {
-    let checked = crate::smlib_body(body).map_err(|e| e.to_string()).and_then(|b| {
-        let r = b.check(solidcraft_smlib::CheckLevel::All).map_err(|e| format!("SMLib check: {e}"))?;
-        if r.is_valid() { Ok(()) } else { Err(format!("SMLib check: {r}")) }
-    });
-    Some(checked)
+fn smlib_validity(body: &Body) -> Option<SmlibCheck> {
+    let b = match crate::smlib_body(body) {
+        Ok(b) => b,
+        Err(e) => {
+            let e = e.to_string();
+            return Some(if e.contains("not supported yet") { SmlibCheck::NotSupported(e) } else { SmlibCheck::Invalid(e) });
+        }
+    };
+    Some(match b.check(solidcraft_smlib::CheckLevel::All) {
+        Ok(r) if r.is_valid() => SmlibCheck::Valid,
+        Ok(r) => SmlibCheck::Invalid(format!("SMLib check: {r}")),
+        Err(e) => SmlibCheck::Invalid(format!("SMLib check: {e}")),
+    })
 }
 
 #[cfg(not(feature = "smlib"))]
-fn smlib_validity(_: &Body) -> Option<Result<(), String>> {
+fn smlib_validity(_: &Body) -> Option<SmlibCheck> {
     None
 }
 

tests/kernel-cases/README.md

@@ -43,7 +43,8 @@ Each case has:
 - `recipe.json`: an engine command script (`{"commands": [...]}`, as `solidcraft-cli run`
   takes) or an oracle recipe (`tests/oracle/*/recipe.json`). `$CASE` in a string is the case
   directory, for an input file such as `$CASE/input.step` (written by hand from the STEP or IGES
-  specification, never a user's file).
+  specification, never a user's file), and `$OUT` a scratch directory outside the repository for
+  files the case writes (an export).
 - `expected.json`: the right answer, graded by `solidcraft-cli kernel-cases`.
 
 Each area also has `skipped.json` (or several, `skipped-<part>.json`, where an area is triaged in
@@ -75,7 +76,8 @@ parts): every issue and PR looked at that did not become a case, with one line e
 | `source` | `analytic` (numbers worked out by hand; `method` gives the formulas) or `invariants` (no number is known: only the outcome, validity and counts are graded). |
 | `method` | How the numbers were derived, or which invariants hold and why. |
 | `outcome` | `solid` (default: the script succeeds) or `refused` (the right answer is a clean error, never a panic or a hang). |
-| `valid` | Default `true`: every body is one closed shell that passes the record's structure and geometry checks and, in builds with SMLib, SMLib's `check(All)`. `false` only where the result is legitimately not a solid. |
+| `valid` | Default `true`: every body is one closed shell (or `shells` in all) that passes the record's structure and geometry checks and, in builds with SMLib, SMLib's `check(All)`. `false` only where the result is legitimately not a solid (a sheet). Where SMLib can't take the record yet (a surface kind its conversion lacks), its check gives no verdict and is not graded. |
+| `shells` | The total number of shells, where it isn't one per body: a body with a sealed void (2), several lumps kept in one body. |
 | `body_count`, `faces`, `edges`, `vertices` | Exact counts, where the geometry fixes them. Faces and edges count whole analytic surfaces and curves once (a full cylinder side is one face). |
 | `volume_mm3`, `area_mm2`, `center_of_mass`, `bbox` | As in `tests/oracle`: totals over every body, relative tolerance 1e-3 for volume and area, 1e-3 of the box diagonal for positions. |
 | `tolerance` | Optional overrides: `rel`, `abs_mm`. |

tests/kernel-cases/booleans/3175-cut-beside-sealed-void/expected.json

@@ -4,8 +4,8 @@
   ],
   "what": "a cut over the bottom edge of a revolved cylinder that holds a sealed spherical void on its axis: the body must not come out empty or refused",
   "source": "analytic",
-  "method": "revolved cylinder r24 h60 minus the sealed sphere r2 at z=30 minus the box x 20..30, y -10..10, z -5..10 where it meets the cylinder: {x>=20, |y|<=10, x^2+y^2<=24^2} is Int from -10 to 10 of (sqrt(576-y^2) - 20) dy = 10*sqrt(476) + 576*asin(10/24) - 400 = 65.725 over z in 0..10. V = pi*24^2*60 - 4/3*pi*2^3 - 10*area = 107882.68. One body that holds two shells (outer and the void), so valid is false: the rig reads more than one shell as not a solid",
-  "valid": false,
+  "method": "revolved cylinder r24 h60 minus the sealed sphere r2 at z=30 minus the box x 20..30, y -10..10, z -5..10 where it meets the cylinder: {x>=20, |y|<=10, x^2+y^2<=24^2} is Int from -10 to 10 of (sqrt(576-y^2) - 20) dy = 10*sqrt(476) + 576*asin(10/24) - 400 = 65.725 over z in 0..10. V = pi*24^2*60 - 4/3*pi*2^3 - 10*area = 107882.68. One body with two shells (outer and the void)",
   "body_count": 1,
+  "shells": 2,
   "volume_mm3": 107882.68287918373
 }

tests/kernel-cases/booleans/3455-join-of-64-separate-pegs/expected.json

@@ -4,8 +4,8 @@
   ],
   "what": "one combine of 64 separate pegs that do not touch: 36 s on truck, growing 4-9x per doubling; must finish in seconds",
   "source": "analytic",
-  "method": "64 non-touching 48-gon pegs of circumradius 2, 1 mm tall, 6 mm apart: each area 0.5*48*2^2*sin(2pi/48) = 12.5305, V = 64 * 12.5305 * 1. The join keeps them in one body (several lumps). A hang is a failure: time should grow with the number of bodies. valid is false only because the one body holds 64 separate shells, which the rig's one-shell check does not allow",
+  "method": "64 non-touching 48-gon pegs of circumradius 2, 1 mm tall, 6 mm apart: each area 0.5*48*2^2*sin(2pi/48) = 12.5305, V = 64 * 12.5305 * 1. The join keeps them in one body (several lumps). A hang is a failure: time should grow with the number of bodies. One body with 64 shells",
   "body_count": 1,
-  "volume_mm3": 801.9529,
-  "valid": false
-}
+  "shells": 64,
+  "volume_mm3": 801.9529
+}

tests/kernel-cases/booleans/3526-intersect-keeps-every-lump/expected.json

@@ -5,8 +5,8 @@
   ],
   "what": "intersect of a two-lump target (two spheres joined while apart) with a box holding the bottom half of each",
   "source": "analytic",
-  "method": "box x -10..40, y -10..10, z -10..0 holds the z<=0 half of each r=4 sphere (centres (0,0,0), (30,0,0)): V = 2 * (2/3) pi 4^3 = 268.083. The body must not be removed. valid is false only because the one body holds two separate shells, which the rig's one-shell check does not allow",
+  "method": "box x -10..40, y -10..10, z -10..0 holds the z<=0 half of each r=4 sphere (centres (0,0,0), (30,0,0)): V = 2 * (2/3) pi 4^3 = 268.083. The body must not be removed. One body with two shells",
   "body_count": 1,
-  "volume_mm3": 268.0826,
-  "valid": false
-}
+  "shells": 2,
+  "volume_mm3": 268.0826
+}

tests/kernel-cases/crashes/1985-step-export-enclosure-screw-hole/recipe.json

@@ -834,7 +834,7 @@
   {
    "command": "file.export",
    "params": {
-    "path": "/tmp/kernel-case-1985.step",
+    "path": "$OUT/kernel-case-1985.step",
     "format": "step"
    }
   },

tests/kernel-cases/crashes/2243-step-export-flanged-enclosure/recipe.json

@@ -1103,7 +1103,7 @@
   {
    "command": "file.export",
    "params": {
-    "path": "/tmp/kernel-case-2243.step",
+    "path": "$OUT/kernel-case-2243.step",
     "format": "step"
    }
   },

tests/kernel-cases/import/3730-void-oriented-shell-false/expected.json

@@ -1,16 +1,16 @@
 {
- "sources": [
-  "tangency-labs/solidcraft#3730",
-  "tangency-labs/solidcraft#3872"
- ],
- "what": "a block with a closed cavity written as BREP_WITH_VOIDS whose void shell (outward-facing faces) is turned by ORIENTED_CLOSED_SHELL .F.; the cavity must be taken out, not added",
- "source": "analytic",
- "method": "V = 40*30*12.5 - 20*6*5 = 15000 - 600; A = 2(40*30+40*12.5+30*12.5) + 2(20*6+20*5+6*5) = 4150 + 500; faces 6+6, edges 12+12, vertices 8+8; valid is false only because a body with a void holds two shells (outer and cavity), which the one-shell validity check does not allow",
- "valid": false,
- "body_count": 1,
- "volume_mm3": 14400.0,
- "area_mm2": 4650.0,
- "faces": 12,
- "edges": 24,
- "vertices": 16
+  "sources": [
+    "tangency-labs/solidcraft#3730",
+    "tangency-labs/solidcraft#3872"
+  ],
+  "what": "a block with a closed cavity written as BREP_WITH_VOIDS whose void shell (outward-facing faces) is turned by ORIENTED_CLOSED_SHELL .F.; the cavity must be taken out, not added",
+  "source": "analytic",
+  "method": "V = 40*30*12.5 - 20*6*5 = 15000 - 600; A = 2(40*30+40*12.5+30*12.5) + 2(20*6+20*5+6*5) = 4150 + 500; faces 6+6, edges 12+12, vertices 8+8; one body with two shells (outer and cavity)",
+  "body_count": 1,
+  "shells": 2,
+  "volume_mm3": 14400.0,
+  "area_mm2": 4650.0,
+  "faces": 12,
+  "edges": 24,
+  "vertices": 16
 }

xtask/src/kernel_cases.rs

@@ -26,6 +26,8 @@ struct Options {
     area: Option<String>,
     timeout: Duration,
     jobs: usize,
+    /// Where to keep every case's full result as JSON.
+    results: Option<String>,
 }
 
 fn options(rest: &[&str]) -> Result<Options, String> {
@@ -38,7 +40,7 @@ fn options(rest: &[&str]) -> Result<Options, String> {
     };
     let secs = arg("--timeout").map(|t| t.parse::<u64>().map_err(|e| format!("--timeout: {e}"))).transpose()?.unwrap_or(DEFAULT_TIMEOUT_S);
     let jobs = arg("--jobs").map(|j| j.parse::<usize>().map_err(|e| format!("--jobs: {e}"))).transpose()?.unwrap_or(4).clamp(1, 32);
-    Ok(Options { kernels, cli: arg("--cli"), area: arg("--area"), timeout: Duration::from_secs(secs), jobs })
+    Ok(Options { kernels, cli: arg("--cli"), area: arg("--area"), timeout: Duration::from_secs(secs), jobs, results: arg("--results") })
 }
 
 fn read(path: &Path) -> Value {
@@ -323,6 +325,10 @@ pub fn run(rest: &[&str]) -> Result<(), String> {
         return Err(format!("no cases (recipe.json + expected.json) under {}", dir.display()));
     }
     let results = run_all(&o, &list);
+    if let Some(path) = &o.results {
+        let text = serde_json::to_string_pretty(&results).map_err(|e| e.to_string())?;
+        std::fs::write(path, text).map_err(|e| format!("{path}: {e}"))?;
+    }
     println!("{}", write_report(&o, &dir, &list, &results)?);
     Ok(())
 }

xtask/src/main.rs

@@ -46,10 +46,11 @@ commands:
                   replay the reference tests (tests/oracle/*/recipe.json), compare with each
                   case's expected.json and write docs/oracle.md (--cli: a solidcraft-cli
                   already built, instead of cargo run --release)
-  kernel-cases [--kernel default|smlib|both] [--area AREA] [--cli BIN] [--timeout S] [--jobs N]
+  kernel-cases [--kernel default|smlib|both] [--area AREA] [--cli BIN] [--timeout S] [--jobs N] [--results FILE]
                   the kernel regression rig: run every case in tests/kernel-cases/<area>/<id>/ (one
                   process each, killed after --timeout, default 180 s) with each kernel asked for,
-                  grade it against the case's own answer and write docs/kernel-cases.md
+                  grade it against the case's own answer and write docs/kernel-cases.md (--results:
+                  every result as JSON too)
   action-matrix [--bless] [--results FILE] [--from DIR]
                   try every ordinary action on every face, edge, vertex and body of the design
                   corpus and compare with tests/action_matrix/baseline.json (--bless rewrites it;