diff --git a/apps/solidcraft-cli/src/kernel_cases.rs b/apps/solidcraft-cli/src/kernel_cases.rs new file mode 100644 index 0000000000..aac5612393 --- /dev/null +++ b/apps/solidcraft-cli/src/kernel_cases.rs @@ -0,0 +1,194 @@ +//! The kernel regression rig: replay `tests/kernel-cases///recipe.json` (an oracle +//! recipe or a command script) and grade the result against the case's own `expected.json`: +//! closed-form numbers where the case has them, and always the invariants (the operation's +//! outcome, one closed valid solid per body, the expected counts). One kernel per run; kernels +//! are never compared with each other. See tests/kernel-cases/README.md. + +use serde_json::{Value, json}; + +use crate::oracle::{ABS_FRAC, Checks, REL_TOL, totals, v3}; +use crate::{read_json, recipe}; + +/// The kernel this process runs booleans with (`SOLIDCRAFT_BOOLEANS`, set by the caller). +fn kernel() -> String { + std::env::var("SOLIDCRAFT_BOOLEANS").ok().filter(|k| !k.is_empty()).unwrap_or_else(|| "default".into()) +} + +/// The case's commands: a script as is, or an oracle recipe translated; `$CASE` in a string is +/// the case directory (for input files such as `input.step`). +fn script(dir: &str, recipe: &Value) -> Result { + let s = if recipe.get("commands").is_some() || recipe.is_array() { recipe.clone() } else { recipe::to_script(recipe)? }; + let text = serde_json::to_string(&s).map_err(|e| e.to_string())?; + let dir_json = serde_json::to_string(dir).map_err(|e| e.to_string())?; + let dir_inner = dir_json.get(1..dir_json.len().saturating_sub(1)).unwrap_or(""); + serde_json::from_str(&text.replace("$CASE", dir_inner)).map_err(|e| e.to_string()) +} + +/// An error that means "this kernel can't do it yet", not a wrong answer. +fn unsupported(e: &str) -> bool { + e.contains("not supported yet") +} + +/// A panic caught by the engine or the kernel guard. +fn crashed(e: &str) -> bool { + e.contains("internal error") || e.contains("panicked") +} + +fn case_name(dir: &str) -> (String, String) { + let p = std::path::Path::new(dir); + let name = |q: Option<&std::path::Path>| q.and_then(|q| q.file_name()).map(|x| x.to_string_lossy().to_string()).unwrap_or_default(); + (name(p.parent()), name(Some(p))) +} + +fn check_case(dir: &str) -> Value { + let (area, name) = case_name(dir); + let mut out = json!({"case": name, "area": area, "kernel": kernel()}); + let want = match read_json(&format!("{dir}/expected.json")) { + Ok(w) => w, + Err(e) => { + out["status"] = json!("fail"); + out["error"] = json!(format!("expected.json: {e}")); + return out; + } + }; + out["sources"] = want["sources"].clone(); + let commands = match read_json(&format!("{dir}/recipe.json")).and_then(|r| script(dir, &r)) { + Ok(s) => s, + Err(e) => { + out["status"] = json!("fail"); + out["error"] = json!(format!("recipe.json: {e}")); + return out; + } + }; + if kernel() == "smlib" && !solidcraft_engine::kernel::smlib_built() { + out["status"] = json!("unsupported"); + out["error"] = json!("this build has no SMLib: build solidcraft-cli with --features solidcraft-kernel/smlib"); + return out; + } + let refused = want["outcome"].as_str() == Some("refused"); + let mut s = solidcraft_engine::Session::default(); + let ran = s.run_script(&commands); + match (ran, refused) { + (Err(e), _) if crashed(&e.to_string()) => { + out["status"] = json!("crash"); + out["error"] = json!(e.to_string()); + return out; + } + (Err(e), true) => { + // The right answer is a clean refusal. + out["status"] = json!("pass"); + out["refused_with"] = json!(e.to_string()); + return out; + } + (Err(e), false) => { + let e = e.to_string(); + out["status"] = json!(if unsupported(&e) { "unsupported" } else { "fail" }); + out["error"] = json!(e); + return out; + } + (Ok(_), true) => { + out["status"] = json!("fail"); + out["error"] = json!("succeeded where the case expects a refusal"); + return out; + } + (Ok(_), false) => {} + } + + // Every body in the world, imported components included. + let world = s.world_state(); + let mut bodies = Vec::new(); + let mut c = Checks { list: Vec::new(), pass: true }; + let want_valid = want["valid"].as_bool().unwrap_or(true); + for b in &world.bodies { + let m = match solidcraft_engine::kernel::measure(&b.body) { + Ok(m) => m, + Err(e) => { + c.push("measure", json!(format!("{}: {e}", b.name)), json!("measurable"), false); + continue; + } + }; + bodies.push(json!({ + "volume_mm3": m.volume, "area_mm2": m.area, "center_of_mass": m.centroid, + "bbox": {"min": m.bbox.min, "max": m.bbox.max}, + "faces": m.merged.faces, "edges": m.merged.edges, "vertices": m.merged.vertices, + })); + if want_valid { + // One closed shell, and every validity check this build has. + c.push("shells", json!(format!("{}: {}", b.name, m.shells)), json!(1), m.shells == 1); + let v = solidcraft_engine::kernel::validity(&b.body); + c.push("record_valid", json!(v.record.clone().err().unwrap_or_else(|| "ok".into())), json!("ok"), v.record.is_ok()); + if let Some(sm) = &v.smlib { + c.push("smlib_check", json!(sm.clone().err().unwrap_or_else(|| "ok".into())), json!("ok"), sm.is_ok()); + } + } + } + let got = totals(&json!({"bodies": bodies})); + let (edges, vertices): (u64, u64) = + bodies.iter().fold((0, 0), |(e, v), b| (e + b["edges"].as_u64().unwrap_or(0), v + b["vertices"].as_u64().unwrap_or(0))); + let has = |k: &str| want.get(k).is_some_and(|v| !v.is_null()); + let rel = want["tolerance"]["rel"].as_f64().unwrap_or(REL_TOL); + let diag = |b: ([f64; 3], [f64; 3])| ((b.1[0] - b.0[0]).powi(2) + (b.1[1] - b.0[1]).powi(2) + (b.1[2] - b.0[2]).powi(2)).sqrt(); + let want_bbox = v3(&want["bbox"]["min"]).zip(v3(&want["bbox"]["max"])); + let abs = want["tolerance"]["abs_mm"].as_f64().unwrap_or_else(|| ABS_FRAC * want_bbox.map(diag).unwrap_or_else(|| diag(got.bbox)).max(1.0)); + let count = |c: &mut Checks, k: &str, g: u64| { + if has(k) { + c.push(k, json!(g), want[k].clone(), want[k].as_u64() == Some(g)); + } + }; + count(&mut c, "body_count", got.body_count as u64); + count(&mut c, "faces", got.faces); + count(&mut c, "edges", edges); + count(&mut c, "vertices", vertices); + if has("volume_mm3") { + c.rel("volume_mm3", got.volume, want["volume_mm3"].as_f64(), rel); + } + if has("area_mm2") { + c.rel("area_mm2", got.area, want["area_mm2"].as_f64(), rel); + } + if has("center_of_mass") { + c.vec("center_of_mass", got.com, v3(&want["center_of_mass"]), abs); + } + if has("bbox") { + c.vec("bbox.min", got.bbox.0, want_bbox.map(|b| b.0), abs); + c.vec("bbox.max", got.bbox.1, want_bbox.map(|b| b.1), abs); + } + out["status"] = json!(if c.pass { "pass" } else { "fail" }); + out["checks"] = json!(c.list); + out +} + +fn report(args: &[String], results: &[Value]) { + if args.iter().any(|a| a == "--json") { + println!("{}", serde_json::to_string_pretty(&Value::Array(results.to_vec())).unwrap_or_default()); + return; + } + for r in results { + let detail: Vec = r["checks"] + .as_array() + .into_iter() + .flatten() + .filter(|c| !c["ok"].as_bool().unwrap_or(false)) + .map(|c| format!("{} got {} want {}", c["check"].as_str().unwrap_or(""), c["got"], c["want"])) + .collect(); + let err = r["error"].as_str().map(|e| format!(" ({e})")).unwrap_or_default(); + println!( + "{} {}/{} [{}]{err}{}", + r["status"].as_str().unwrap_or("?").to_uppercase(), + r["area"].as_str().unwrap_or(""), + r["case"].as_str().unwrap_or(""), + r["kernel"].as_str().unwrap_or(""), + if detail.is_empty() { String::new() } else { format!(": {}", detail.join("; ")) } + ); + } +} + +pub fn run(args: &[String]) -> Result<(), String> { + let dirs: Vec<&String> = args.iter().filter(|a| !a.starts_with("--")).collect(); + if dirs.is_empty() { + return Err("kernel-cases: give one or more case directories (each with recipe.json and expected.json)".into()); + } + let results: Vec = dirs.iter().map(|d| check_case(d)).collect(); + report(args, &results); + let bad = results.iter().filter(|r| matches!(r["status"].as_str(), Some("fail" | "crash"))).count(); + if bad > 0 { Err(format!("{bad} kernel case(s) failed")) } else { Ok(()) } +} diff --git a/apps/solidcraft-cli/src/main.rs b/apps/solidcraft-cli/src/main.rs index 0558aeefe5..a73856567b 100644 --- a/apps/solidcraft-cli/src/main.rs +++ b/apps/solidcraft-cli/src/main.rs @@ -15,6 +15,7 @@ //! solidcraft-cli commands the command registry as JSON //! solidcraft-cli recipe translate an oracle recipe to a script //! solidcraft-cli oracle ... replay recipes and compare with expected.json +//! solidcraft-cli kernel-cases [--json] ... the kernel regression rig (tests/kernel-cases) //! solidcraft-cli mcp [--headless] [--in design|script] [--connect HOST:PORT [--token T]] [--allow-path DIR]... [--allow-any-path] //! [--param-check off|warn|strict] //! MCP server on stdio (docs/mcp.md) @@ -31,6 +32,7 @@ mod bench_agent; mod bench_assistant; mod crashes; mod fuzz; +mod kernel_cases; mod logger; mod oracle; mod preview; @@ -79,6 +81,8 @@ const USAGE_COMMANDS: &str = " solidcraft-cli commands\n"; const USAGE_RECIPE: &str = " solidcraft-cli recipe \n"; /// `oracle`'s lines in the usage. const USAGE_ORACLE: &str = " solidcraft-cli oracle ... [--json]\n"; +/// `kernel-cases`'s lines in the usage. +const USAGE_KERNEL_CASES: &str = " solidcraft-cli kernel-cases ... [--json]\n (the kernel regression rig, tests/kernel-cases; SOLIDCRAFT_BOOLEANS picks the kernel)\n"; /// `fuzz`'s lines in the usage. const USAGE_FUZZ: &str = " solidcraft-cli fuzz [--from N] [--count N] [--jobs J] [--timeout S] [--steps N] [--minimise] [--out DIR] [--report FILE]\n"; @@ -116,6 +120,7 @@ const COMMANDS: &[Command] = &[ Command { name: "commands", usage: USAGE_COMMANDS, listed: true }, Command { name: "recipe", usage: USAGE_RECIPE, listed: true }, Command { name: "oracle", usage: USAGE_ORACLE, listed: true }, + Command { name: "kernel-cases", usage: USAGE_KERNEL_CASES, listed: true }, Command { name: "fuzz", usage: USAGE_FUZZ, listed: true }, Command { name: "fuzz-one", usage: USAGE_FUZZ_ONE, listed: false }, Command { name: "mcp", usage: USAGE_MCP, listed: true }, @@ -164,6 +169,7 @@ fn main() -> ExitCode { recipe::to_script(&v).map(|s| println!("{}", pretty(&s))) }), Some("oracle") => oracle::run(&args[1..]), + Some("kernel-cases") => kernel_cases::run(&args[1..]), Some("fuzz") => fuzz::run(&args[1..]), Some("fuzz-one") => fuzz::one(&args[1..]), Some("mcp") => cmd_mcp(&args[1..]), diff --git a/apps/solidcraft-cli/src/oracle.rs b/apps/solidcraft-cli/src/oracle.rs index 37e2853e45..f7443c2e1e 100644 --- a/apps/solidcraft-cli/src/oracle.rs +++ b/apps/solidcraft-cli/src/oracle.rs @@ -13,28 +13,28 @@ use crate::{read_json, recipe}; /// Default relative tolerance for volume and area (our curved-face tessellation is within ~2e-4). pub const REL_TOL: f64 = 1e-3; /// Default tolerance for centre of mass and bounding box, as a fraction of the bbox diagonal. -const ABS_FRAC: f64 = 1e-3; +pub(crate) const ABS_FRAC: f64 = 1e-3; /// Chord tolerance used to triangulate our part for the surface-distance check (mm). const MESH_TOL: f64 = 0.01; /// Relative tolerance on the principal second moments of the two triangulations. const INERTIA_REL: f64 = 2e-3; /// What SolidCraft built: totals over every body. -struct Got { - body_count: usize, - volume: f64, - area: f64, - com: [f64; 3], - bbox: ([f64; 3], [f64; 3]), - faces: u64, +pub(crate) struct Got { + pub(crate) body_count: usize, + pub(crate) volume: f64, + pub(crate) area: f64, + pub(crate) com: [f64; 3], + pub(crate) bbox: ([f64; 3], [f64; 3]), + pub(crate) faces: u64, } -fn v3(v: &Value) -> Option<[f64; 3]> { +pub(crate) fn v3(v: &Value) -> Option<[f64; 3]> { let a = v.as_array()?; Some([a.first()?.as_f64()?, a.get(1)?.as_f64()?, a.get(2)?.as_f64()?]) } -fn totals(m: &Value) -> Got { +pub(crate) fn totals(m: &Value) -> Got { let bodies = m["bodies"].as_array().cloned().unwrap_or_default(); let (mut volume, mut area, mut faces) = (0.0, 0.0, 0); let mut moment = [0.0; 3]; @@ -57,21 +57,21 @@ fn totals(m: &Value) -> Got { Got { body_count: bodies.len(), volume, area, com, bbox: (lo, hi), faces } } -struct Checks { - list: Vec, - pass: bool, +pub(crate) struct Checks { + pub(crate) list: Vec, + pub(crate) pass: bool, } impl Checks { - fn push(&mut self, what: &str, got: Value, want: Value, ok: bool) { + pub(crate) fn push(&mut self, what: &str, got: Value, want: Value, ok: bool) { self.pass &= ok; self.list.push(json!({"check": what, "got": got, "want": want, "ok": ok})); } - fn rel(&mut self, what: &str, g: f64, w: Option, rel: f64) { + pub(crate) fn rel(&mut self, what: &str, g: f64, w: Option, rel: f64) { let ok = w.is_some_and(|w| g.is_finite() && (g - w).abs() <= rel * w.abs().max(1.0)); self.push(what, json!(g), json!(w), ok); } - fn vec(&mut self, what: &str, g: [f64; 3], w: Option<[f64; 3]>, tol: f64) { + pub(crate) fn vec(&mut self, what: &str, g: [f64; 3], w: Option<[f64; 3]>, tol: f64) { let ok = w.is_some_and(|w| g.iter().zip(w).all(|(g, w)| g.is_finite() && (g - w).abs() <= tol)); self.push(what, json!(g), json!(w), ok); } diff --git a/crates/kernel/src/lib.rs b/crates/kernel/src/lib.rs index 7fddd3dcae..c9423410d1 100644 --- a/crates/kernel/src/lib.rs +++ b/crates/kernel/src/lib.rs @@ -152,6 +152,7 @@ mod touching_walls; mod tspline; #[cfg(test)] mod untouched_faces_tests; +mod validity; pub mod watch; mod whole_face; @@ -223,6 +224,7 @@ pub use sweep_path::{RailFit, SweepControl, SweepOrientation, sweep_path, sweep_ pub use topo::{CylinderFace, cylinder_end_at, cylinder_face, cylinder_face_at}; pub use topo::{TopoCounts, merged_topology, seam_flags}; pub use tspline::tspline_sheet; +pub use validity::{Validity, smlib_built, validity}; pub use whole_face::{WholeFace, cut_takes_only_the_face, cut_tool, face_prism, matching_face}; #[derive(Debug, Clone, PartialEq, thiserror::Error)] diff --git a/crates/kernel/src/validity.rs b/crates/kernel/src/validity.rs new file mode 100644 index 0000000000..3ccd5f85bf --- /dev/null +++ b/crates/kernel/src/validity.rs @@ -0,0 +1,49 @@ +//! Is a body one valid solid? The kernel regression rig (`tests/kernel-cases`) asks this of every +//! result: the schema 4 record's structure and geometry (`Brep::validate`, `check_geometry`), +//! and, with the `smlib` feature, SMLib's own check of the same record. + +use crate::{Body, brep_record}; + +/// What the validity checks found for one body. `Err` holds why a check failed. +#[derive(Clone, Debug)] +pub struct Validity { + /// The record: structure, then geometry against topology within tolerances. + pub record: Result<(), String>, + /// SMLib's `check(All)` of the record; `None` in a build without SMLib. + pub smlib: Option>, +} + +impl Validity { + pub fn ok(&self) -> bool { + self.record.is_ok() && self.smlib.as_ref().is_none_or(Result::is_ok) + } +} + +/// Runs every validity check this build has on `body`. +pub fn validity(body: &Body) -> Validity { + let record = brep_record(body).map_err(|e| e.to_string()); + let checked = record.as_ref().map_err(Clone::clone).and_then(|r| { + r.validate().map_err(|e| format!("structure: {e}"))?; + r.check_geometry().map_err(|e| format!("geometry: {e}")) + }); + Validity { record: checked, smlib: smlib_validity(body) } +} + +#[cfg(feature = "smlib")] +fn smlib_validity(body: &Body) -> Option> { + let checked = crate::smlib_body(body).map_err(|e| e.to_string()).and_then(|b| { + let r = b.check(solidcraft_smlib::CheckLevel::All).map_err(|e| format!("SMLib check: {e}"))?; + if r.is_valid() { Ok(()) } else { Err(format!("SMLib check: {r}")) } + }); + Some(checked) +} + +#[cfg(not(feature = "smlib"))] +fn smlib_validity(_: &Body) -> Option> { + None +} + +/// Whether this build links SMLib (the `smlib` feature). +pub fn smlib_built() -> bool { + cfg!(feature = "smlib") +} diff --git a/tests/kernel-cases/README.md b/tests/kernel-cases/README.md new file mode 100644 index 0000000000..ebd03b82f4 --- /dev/null +++ b/tests/kernel-cases/README.md @@ -0,0 +1,93 @@ +# Kernel regression rig + +Every geometry case the old kernel (truck) got wrong, recorded in an issue, a pull request or a +crash report, is a use case here with its own right answer. The rig holds every kernel to all of +them. `cargo xtask kernel-cases` runs them and writes [`docs/kernel-cases.md`](../../docs/kernel-cases.md). + +```sh +cargo xtask kernel-cases # the default kernel +cargo xtask kernel-cases --kernel both --cli target/release/solidcraft-cli # and SMLib +cargo run -p solidcraft-cli -- kernel-cases tests/kernel-cases/booleans/ # one case +``` + +For the `smlib` column, build `solidcraft-cli` with `--features solidcraft-kernel/smlib` (after +`scripts/fetch-smlib.sh`). The rig sets `SOLIDCRAFT_BOOLEANS=smlib` for those runs. + +## Rules + +- **The answer never comes from a kernel.** Not from truck's output, nor any commercial + kernel's. It comes from analysis or a hand calculation, or else from the invariants below. +- **Kernels are graded against the case, never against each other.** +- **No user designs.** A crash report or a repro design becomes a small synthetic recipe that + shows the same geometry. Check for private content before committing anything. +- **No workaround on SolidCraft's side.** A case that fails on a kernel stays failing until that + kernel is fixed. SMLib defects go to tangency-labs/solidcore as issues (`priority: high`, + `bug`), and the fix goes into the fork. + +## Layout + +One directory per area, one per case: `tests/kernel-cases///`. + +| Area | Owner | +|---|---| +| `booleans` (combine, cuts, holes, patterns, split, mirror, draft) | booleans/shell/import/crash rig | +| `shell` (shell, offset, thicken) | booleans/shell/import/crash rig | +| `import` (STEP and IGES geometry) | booleans/shell/import/crash rig | +| `crashes` (crashes and hangs inside kernel operations) | booleans/shell/import/crash rig | +| `fillets`, `sweeps`, `threads`, `sheet-metal`, `tessellation` | history rig | + +`` is `-`, such as `1894-prism-cut-touching-walls`. + +Each case has: + +- `recipe.json`: an engine command script (`{"commands": [...]}`, as `solidcraft-cli run` + takes) or an oracle recipe (`tests/oracle/*/recipe.json`). `$CASE` in a string is the case + directory, for an input file such as `$CASE/input.step` (written by hand from the STEP or IGES + specification, never a user's file). +- `expected.json`: the right answer, graded by `solidcraft-cli kernel-cases`. + +Each area also has `skipped.json`: every issue and PR looked at that did not become a case, with +one line each saying why. + +```json +[{"source": "tangency-labs/solidcraft#123", "why": "UI only: the selection highlight, no geometry"}] +``` + +## expected.json + +```json +{ + "sources": ["tangency-labs/solidcraft#1894", "tangency-labs/solidcraft#1629"], + "what": "a prism cut whose walls only touch the target's walls", + "source": "analytic", + "method": "box 40x30x20 minus the prism 10x30x5 flush with the side: V = 24000 - 1500", + "body_count": 1, + "volume_mm3": 22500.0, + "area_mm2": 5200.0, + "faces": 8 +} +``` + +| Field | Meaning | +|---|---| +| `sources` | The issues, PRs or crash reports the case comes from (`owner/repo#N`). | +| `what` | One line: the geometry that went wrong. | +| `source` | `analytic` (numbers worked out by hand; `method` gives the formulas) or `invariants` (no number is known: only the outcome, validity and counts are graded). | +| `method` | How the numbers were derived, or which invariants hold and why. | +| `outcome` | `solid` (default: the script succeeds) or `refused` (the right answer is a clean error, never a panic or a hang). | +| `valid` | Default `true`: every body is one closed shell that passes the record's structure and geometry checks and, in builds with SMLib, SMLib's `check(All)`. `false` only where the result is legitimately not a solid. | +| `body_count`, `faces`, `edges`, `vertices` | Exact counts, where the geometry fixes them. Faces and edges count whole analytic surfaces and curves once (a full cylinder side is one face). | +| `volume_mm3`, `area_mm2`, `center_of_mass`, `bbox` | As in `tests/oracle`: totals over every body, relative tolerance 1e-3 for volume and area, 1e-3 of the box diagonal for positions. | +| `tolerance` | Optional overrides: `rel`, `abs_mm`. | + +## Results + +Per case and kernel: **pass**; **fail** with the numbers; **not supported yet** (the kernel says +so); **crash** (a panic caught by the guards, or the process died); **hang** (no result before +the timeout). + +## Exporting to solidcore + +solidcore#349 proposes a graded case format for the Rust port. The cases here are SolidCraft's +copy and the source; each one maps onto it field by field (`sources`, the derivation, the +graded quantities and invariants), so an exporter can write them out for solidcore unchanged. diff --git a/xtask/src/kernel_cases.rs b/xtask/src/kernel_cases.rs new file mode 100644 index 0000000000..bff77ea9be --- /dev/null +++ b/xtask/src/kernel_cases.rs @@ -0,0 +1,311 @@ +//! `cargo xtask kernel-cases`: the kernel regression rig. Runs every case in +//! `tests/kernel-cases///` through `solidcraft-cli kernel-cases`, one process per case +//! (so a crash or a hang is that case's result, not the run's), with each kernel asked for, and +//! writes `docs/kernel-cases.md`: results per case and kernel, coverage per area and per source, +//! and the issues and PRs looked at and skipped. Each kernel is graded against the cases' own +//! answers, never against the other kernel. + +use std::collections::BTreeMap; +use std::path::{Path, PathBuf}; +use std::process::{Command, Stdio}; +use std::sync::{Arc, Mutex}; +use std::time::{Duration, Instant}; + +use serde_json::{Value, json}; + +use crate::{cargo, root}; + +const DEFAULT_TIMEOUT_S: u64 = 180; + +/// Per kernel, how many cases ended in each status. +type StatusCounts = BTreeMap>; + +struct Options { + kernels: Vec, + cli: Option, + area: Option, + timeout: Duration, + jobs: usize, +} + +fn options(rest: &[&str]) -> Result { + let arg = |name: &str| rest.iter().position(|a| *a == name).and_then(|i| rest.get(i + 1)).map(|s| s.to_string()); + let kernels = match arg("--kernel").as_deref() { + None | Some("default") => vec!["default".to_string()], + Some("smlib") => vec!["smlib".to_string()], + Some("both") => vec!["default".to_string(), "smlib".to_string()], + Some(k) => return Err(format!("--kernel {k}: default, smlib or both")), + }; + let secs = arg("--timeout").map(|t| t.parse::().map_err(|e| format!("--timeout: {e}"))).transpose()?.unwrap_or(DEFAULT_TIMEOUT_S); + let jobs = arg("--jobs").map(|j| j.parse::().map_err(|e| format!("--jobs: {e}"))).transpose()?.unwrap_or(4).clamp(1, 32); + Ok(Options { kernels, cli: arg("--cli"), area: arg("--area"), timeout: Duration::from_secs(secs), jobs }) +} + +fn read(path: &Path) -> Value { + std::fs::read_to_string(path).ok().and_then(|t| serde_json::from_str(&t).ok()).unwrap_or(Value::Null) +} + +/// Every case directory, sorted by area then id. +fn cases(dir: &Path, only: Option<&str>) -> Result, String> { + let mut out = Vec::new(); + for area in std::fs::read_dir(dir).map_err(|e| format!("{}: {e}", dir.display()))?.flatten() { + let area = area.path(); + if !area.is_dir() || only.is_some_and(|a| area.file_name().is_none_or(|n| n != a)) { + continue; + } + for case in std::fs::read_dir(&area).map_err(|e| format!("{}: {e}", area.display()))?.flatten() { + let case = case.path(); + if case.join("recipe.json").is_file() && case.join("expected.json").is_file() { + out.push(case); + } + } + } + out.sort(); + Ok(out) +} + +/// One case on one kernel, in its own process, killed after the timeout. +fn run_one(o: &Options, kernel: &str, case: &Path) -> Value { + let rel = case.strip_prefix(root()).unwrap_or(case).to_path_buf(); + let mut cmd = match &o.cli { + Some(bin) => Command::new(bin), + None => { + let mut c = cargo(); + c.args(["run", "-q", "--release", "-p", "solidcraft-cli", "--"]); + c + } + }; + cmd.current_dir(root()).args(["kernel-cases", "--json"]).arg(&rel).stdout(Stdio::piped()).stderr(Stdio::piped()); + if kernel == "default" { + cmd.env_remove("SOLIDCRAFT_BOOLEANS"); + } else { + cmd.env("SOLIDCRAFT_BOOLEANS", kernel); + } + let base = |status: &str, error: String| { + let area = case.parent().and_then(|p| p.file_name()).map(|n| n.to_string_lossy().to_string()).unwrap_or_default(); + let id = case.file_name().map(|n| n.to_string_lossy().to_string()).unwrap_or_default(); + json!({"case": id, "area": area, "kernel": kernel, "status": status, "error": error}) + }; + let mut child = match cmd.spawn() { + Ok(c) => c, + Err(e) => return base("fail", format!("solidcraft-cli: {e}")), + }; + let start = Instant::now(); + loop { + match child.try_wait() { + Ok(Some(_)) => break, + Ok(None) if start.elapsed() > o.timeout => { + let _ = child.kill(); + let _ = child.wait(); + return base("hang", format!("no result after {} s", o.timeout.as_secs())); + } + Ok(None) => std::thread::sleep(Duration::from_millis(50)), + Err(e) => return base("fail", format!("wait: {e}")), + } + } + let out = match child.wait_with_output() { + Ok(o) => o, + Err(e) => return base("fail", format!("output: {e}")), + }; + match serde_json::from_slice::(&out.stdout).ok().and_then(|v| v.as_array().and_then(|a| a.first().cloned())) { + Some(mut r) => { + r["seconds"] = json!(start.elapsed().as_secs_f64()); + r + } + // No JSON: the process died (a native crash, an abort). + None => { + let err = String::from_utf8_lossy(&out.stderr); + let tail: String = err.lines().rev().take(3).collect::>().into_iter().rev().collect::>().join(" / "); + base("crash", format!("process exited ({}) without a result: {tail}", out.status)) + } + } +} + +fn run_all(o: &Options, list: &[PathBuf]) -> Vec { + let jobs: Vec<(String, PathBuf)> = o.kernels.iter().flat_map(|k| list.iter().map(move |c| (k.clone(), c.clone()))).collect(); + let next = Arc::new(Mutex::new(0usize)); + let results = Arc::new(Mutex::new(vec![Value::Null; jobs.len()])); + std::thread::scope(|s| { + for _ in 0..o.jobs.min(jobs.len().max(1)) { + let (next, results, jobs) = (Arc::clone(&next), Arc::clone(&results), &jobs); + s.spawn(move || { + loop { + let i = { + let Ok(mut n) = next.lock() else { return }; + let i = *n; + *n += 1; + i + }; + let Some((kernel, case)) = jobs.get(i) else { return }; + let r = run_one(o, kernel, case); + println!( + "{:<11} {} [{kernel}]", + r["status"].as_str().unwrap_or("?").to_uppercase(), + case.strip_prefix(root()).unwrap_or(case).display() + ); + if let Ok(mut all) = results.lock() + && let Some(slot) = all.get_mut(i) + { + *slot = r; + } + } + }); + } + }); + Arc::try_unwrap(results).ok().and_then(|m| m.into_inner().ok()).unwrap_or_default() +} + +fn cell(r: Option<&Value>) -> String { + let Some(r) = r else { return String::new() }; + let status = r["status"].as_str().unwrap_or("?"); + let mut notes: Vec = r["checks"] + .as_array() + .into_iter() + .flatten() + .filter(|c| !c["ok"].as_bool().unwrap_or(false)) + .map(|c| format!("{} {} (want {})", c["check"].as_str().unwrap_or(""), short(&c["got"]), short(&c["want"]))) + .collect(); + if let Some(e) = r["error"].as_str() { + notes.push(e.chars().take(160).collect()); + } + let s = match status { + "pass" => "pass".to_string(), + "unsupported" => "not supported yet".to_string(), + other => format!("**{}**", other.to_uppercase()), + }; + if notes.is_empty() { s } else { format!("{s}: {}", notes.join("; ")).replace('|', "/").replace('\n', " ") } +} + +fn short(v: &Value) -> String { + match v { + Value::Number(n) => n.as_f64().map(|x| format!("{x:.6}").trim_end_matches('0').trim_end_matches('.').to_string()).unwrap_or_default(), + Value::String(s) => s.chars().take(100).collect(), + Value::Array(a) => format!("[{}]", a.iter().map(short).collect::>().join(", ")), + other => other.to_string(), + } +} + +fn sources_of(want: &Value) -> Vec { + want["sources"].as_array().into_iter().flatten().filter_map(|s| s.as_str().map(str::to_string)).collect() +} + +/// `tangency-labs/solidcraft#123` as a link. +fn link(source: &str) -> String { + match source.split_once('#') { + Some((repo, n)) if repo.contains('/') => format!("[{source}](https://github.com/{repo}/issues/{n})"), + _ => source.to_string(), + } +} + +fn write_report(o: &Options, dir: &Path, list: &[PathBuf], results: &[Value]) -> Result { + let key = |r: &Value| (r["area"].as_str().unwrap_or("").to_string(), r["case"].as_str().unwrap_or("").to_string()); + let mut by_case: BTreeMap<(String, String), BTreeMap> = BTreeMap::new(); + for r in results { + by_case.entry(key(r)).or_default().insert(r["kernel"].as_str().unwrap_or("").to_string(), r.clone()); + } + // Coverage: per area, cases and their statuses per kernel; per source, its cases. + let mut areas: BTreeMap = BTreeMap::new(); + let mut per_source: BTreeMap> = BTreeMap::new(); + let mut kinds: BTreeMap = BTreeMap::new(); + let mut rows = String::new(); + for case in list { + let want = read(&case.join("expected.json")); + let area = case.parent().and_then(|p| p.file_name()).map(|n| n.to_string_lossy().to_string()).unwrap_or_default(); + let id = case.file_name().map(|n| n.to_string_lossy().to_string()).unwrap_or_default(); + let entry = areas.entry(area.clone()).or_default(); + entry.0 += 1; + let runs = by_case.get(&(area.clone(), id.clone())); + for k in &o.kernels { + let status = runs.and_then(|r| r.get(k)).and_then(|r| r["status"].as_str()).unwrap_or("not run").to_string(); + *entry.1.entry(k.clone()).or_default().entry(status).or_default() += 1; + } + let sources = sources_of(&want); + for s in &sources { + per_source.entry(s.clone()).or_default().push(format!("{area}/{id}")); + } + *kinds.entry(want["source"].as_str().unwrap_or("?").split_whitespace().next().unwrap_or("?").to_string()).or_default() += 1; + let cells: Vec = o.kernels.iter().map(|k| cell(runs.and_then(|r| r.get(k)))).collect(); + rows += &format!( + "| {area}/{id} | {} | {} | {} | {} |\n", + sources.iter().map(|s| link(s)).collect::>().join(" "), + want["what"].as_str().unwrap_or("").replace('|', "/"), + want["source"].as_str().unwrap_or("?"), + cells.join(" | ") + ); + } + let mut md = String::from( + "# Kernel regression rig\n\nGenerated by `cargo xtask kernel-cases`. Every geometry case the old kernel (truck) got wrong, as an issue, a PR or a crash report recorded it, is a use case in [`tests/kernel-cases/`](../tests/kernel-cases/) with its own right answer: numbers from analysis or a hand calculation (**analytic**) where the geometry has them, and otherwise the invariants (**invariants**): the operation succeeds, or refuses cleanly where that is the right answer; each body is one closed shell that passes the record's structure and geometry checks and, in builds with SMLib, SMLib's own check; the expected face, edge and vertex counts. Never a kernel's output, and never a commercial kernel's.\n\nEach kernel is graded against the case, never against the other kernel. **default** is the kernel SolidCraft ships today; **smlib** runs with `SOLIDCRAFT_BOOLEANS=smlib`. A failing case stays failing until its kernel is fixed; there is no workaround on SolidCraft's side. See [`tests/kernel-cases/README.md`](../tests/kernel-cases/README.md) for the format.\n\n", + ); + md += "## Coverage\n\n| Area | Cases |"; + for k in &o.kernels { + md += &format!(" {k} |"); + } + md += "\n|---|---|"; + md += &"---|".repeat(o.kernels.len()); + md += "\n"; + for (area, (n, statuses)) in &areas { + md += &format!("| {area} | {n} |"); + for k in &o.kernels { + let s = statuses.get(k).map(|m| m.iter().map(|(s, c)| format!("{c} {s}")).collect::>().join(", ")).unwrap_or_default(); + md += &format!(" {s} |"); + } + md += "\n"; + } + md += &format!( + "\n{} cases from {} sources; answers: {}.\n\n## Results\n\n| Case | Sources | What went wrong | Answer |", + list.len(), + per_source.len(), + kinds.iter().map(|(k, v)| format!("{v} {k}")).collect::>().join(", ") + ); + for k in &o.kernels { + md += &format!(" {k} |"); + } + md += "\n|---|---|---|---|"; + md += &"---|".repeat(o.kernels.len()); + md += "\n"; + md += &rows; + md += "\n## Sources\n\n| Source | Cases |\n|---|---|\n"; + for (s, cs) in &per_source { + md += &format!("| {} | {} |\n", link(s), cs.join(", ")); + } + // What was looked at and skipped, area by area (`/skipped.json`). + md += "\n## Looked at and skipped\n\n"; + let mut skipped = 0; + let mut area_dirs: Vec = std::fs::read_dir(dir).map_err(|e| e.to_string())?.flatten().map(|e| e.path()).filter(|p| p.is_dir()).collect(); + area_dirs.sort(); + for a in area_dirs { + let list = read(&a.join("skipped.json")); + let Some(items) = list.as_array().filter(|l| !l.is_empty()) else { continue }; + md += &format!("### {}\n\n| Source | Why skipped |\n|---|---|\n", a.file_name().map(|n| n.to_string_lossy().to_string()).unwrap_or_default()); + for it in items { + skipped += 1; + md += &format!("| {} | {} |\n", link(it["source"].as_str().unwrap_or("?")), it["why"].as_str().unwrap_or("").replace('|', "/")); + } + md += "\n"; + } + if skipped == 0 { + md += "None yet.\n"; + } + std::fs::write(root().join("docs/kernel-cases.md"), md).map_err(|e| format!("docs/kernel-cases.md: {e}"))?; + let summary: Vec = o + .kernels + .iter() + .map(|k| { + let n = results.iter().filter(|r| r["kernel"] == k.as_str() && r["status"] == "pass").count(); + format!("{k}: {n}/{} pass", list.len()) + }) + .collect(); + Ok(format!("kernel-cases: {} → docs/kernel-cases.md ({skipped} skipped sources listed)", summary.join(", "))) +} + +pub fn run(rest: &[&str]) -> Result<(), String> { + let o = options(rest)?; + let dir = root().join("tests/kernel-cases"); + let list = cases(&dir, o.area.as_deref())?; + if list.is_empty() { + return Err(format!("no cases (recipe.json + expected.json) under {}", dir.display())); + } + let results = run_all(&o, &list); + println!("{}", write_report(&o, &dir, &list, &results)?); + Ok(()) +} diff --git a/xtask/src/main.rs b/xtask/src/main.rs index 0aa7cf159e..25c963644e 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -10,6 +10,7 @@ mod external_corpus; mod features; mod i18n; mod icons; +mod kernel_cases; mod layers; mod licences; mod lists; @@ -45,6 +46,10 @@ commands: replay the reference tests (tests/oracle/*/recipe.json), compare with each case's expected.json and write docs/oracle.md (--cli: a solidcraft-cli already built, instead of cargo run --release) + kernel-cases [--kernel default|smlib|both] [--area AREA] [--cli BIN] [--timeout S] [--jobs N] + the kernel regression rig: run every case in tests/kernel-cases/// (one + process each, killed after --timeout, default 180 s) with each kernel asked for, + grade it against the case's own answer and write docs/kernel-cases.md action-matrix [--bless] [--results FILE] [--from DIR] try every ordinary action on every face, edge, vertex and body of the design corpus and compare with tests/action_matrix/baseline.json (--bless rewrites it; @@ -102,6 +107,7 @@ fn main() -> ExitCode { Some("app-icons") => app_icons::run(&root()), Some("features") => features::run(&root()), Some("i18n") => i18n::run(&root(), &rest), + Some("kernel-cases") => kernel_cases::run(&rest), Some("oracle") => cmd_oracle(rest.iter().position(|a| *a == "--cli").and_then(|i| rest.get(i + 1)).copied()), Some("action-matrix") => { let arg = |name: &str| rest.iter().position(|a| *a == name).and_then(|i| rest.get(i + 1)).copied();