diff --git a/xtask/src/kernel_cases.rs b/xtask/src/kernel_cases.rs index ff9499f1f1..7121ea2edd 100644 --- a/xtask/src/kernel_cases.rs +++ b/xtask/src/kernel_cases.rs @@ -276,7 +276,7 @@ fn write_report(dir: &Path, list: &[PathBuf], results: &[Value]) -> Result-/`). See [`tests/kernel-cases/README.md`](../tests/kernel-cases/README.md) for the format. The fillet cases are also a regression gate: CI's Fillet rig workflow runs them on every pull request that touches fillet code and fails on a case that fails and [`fillets/KNOWN_FAILING`](../tests/kernel-cases/fillets/KNOWN_FAILING) does not list, or a listed case that passes.\n\n", + "# Kernel regression rig\n\nGenerated by `cargo xtask kernel-cases`. Every geometry case the old kernel (truck) got wrong, as an issue, a PR or a crash report recorded it, is a use case in [`tests/kernel-cases/`](../tests/kernel-cases/) with its own right answer: numbers from analysis or a hand calculation (**analytic**) where the geometry has them, and otherwise the invariants (**invariants**): the operation succeeds, or refuses cleanly where that is the right answer; each body is one closed shell that passes the record's structure and geometry checks and SMLib's own check; the expected face, edge and vertex counts. Never a kernel's output, and never a commercial kernel's.\n\nThe cases run on SMLib, SolidCraft's kernel. A failing case stays failing until the kernel is fixed; there is no workaround on SolidCraft's side. A failed SMLib call leaves its inputs for replay in the run's output folder (`target/kernel-cases/dumps/-/`). See [`tests/kernel-cases/README.md`](../tests/kernel-cases/README.md) for the format. Every area is also a regression gate: CI's Kernel rig workflow runs them on every pull request that touches the kernel and fails on a case that fails and its area's `KNOWN_FAILING` (such as [`fillets/KNOWN_FAILING`](../tests/kernel-cases/fillets/KNOWN_FAILING)) does not list, or a listed case that passes.\n\n", ); md += "## Coverage\n\n| Area | Cases | Results |\n|---|---|---|\n"; for (area, (n, statuses)) in &areas { diff --git a/xtask/src/kernel_ladder.rs b/xtask/src/kernel_ladder.rs index 93efd826b9..5e384c7a54 100644 --- a/xtask/src/kernel_ladder.rs +++ b/xtask/src/kernel_ladder.rs @@ -1,4 +1,4 @@ -//! `cargo xtask kernel-cases --ladder [--record REASON]`: the kernel complexity ladder. +//! `cargo xtask kernel-cases --ladder [--record REASON | --gate]`: the kernel complexity ladder. //! //! `tests/kernel-cases/ladder/ladder.json` lists tiers in order of complexity and, per tier, its //! cases in order (paths under `tests/kernel-cases/`, so a tier may also list rig cases from other @@ -9,6 +9,8 @@ //! not shared with other cases. `--record REASON` re-measures the golden and re-sorts each tier //! by time, fastest first; both files are committed. The run's output folder (`--out`, default //! `target/kernel-ladder`) holds `results.json` and the SMLib dumps, as the rig's does. +//! `--gate` makes the run a regression gate instead of the report: it fails on any case whose +//! status or counts differ from the golden (times aside, which are the machine's). //! //! The report, `docs/kernel-ladder.md`, gives per tier the counts per status and the robustness //! frontier: the first case, in ladder order, that fails. @@ -103,17 +105,26 @@ fn golden_entry(r: &Value) -> Value { /// What changed against the golden: a slowdown, a status change, other counts. fn flags(r: &Value, g: Option<&Value>) -> Vec { - let Some(g) = g else { return vec!["no golden".into()] }; - let mut out = Vec::new(); - if g["status"] != r["status"] { - out.push(format!("status was {}", g["status"].as_str().unwrap_or("?"))); - } - if let (Some(now), Some(then)) = (r["seconds"].as_f64(), g["seconds"].as_f64()) + let mut out = changes(r, g); + if let Some(g) = g + && let (Some(now), Some(then)) = (r["seconds"].as_f64(), g["seconds"].as_f64()) && now > then * SLOWDOWN_FACTOR && now - then > SLOWDOWN_MIN_S { out.push(format!("slower: {now:.2} s (golden {then:.2} s)")); } + out +} + +/// What changed against the golden that the machine does not explain: no golden, a status +/// change, other counts. Times are left out (they depend on the machine the golden was recorded +/// on), so these are what the gate fails on. +fn changes(r: &Value, g: Option<&Value>) -> Vec { + let Some(g) = g else { return vec!["no golden".into()] }; + let mut out = Vec::new(); + if g["status"] != r["status"] { + out.push(format!("status was {}", g["status"].as_str().unwrap_or("?"))); + } for k in ["body_count", "faces", "edges", "vertices"] { let (now, then) = (&r["measured"][k], &g["measured"][k]); if !now.is_null() && !then.is_null() && now != then { @@ -123,6 +134,35 @@ fn flags(r: &Value, g: Option<&Value>) -> Vec { out } +/// The ladder's regression gate (`--gate`): every case's status and counts are the golden's. A +/// case that changed fails it, better or worse, as does a case with no golden: the change is +/// re-recorded with its reason (`--record`) in the same pull request. +fn gate(tiers: &[Tier], results: &BTreeMap, golden: &Value) -> Result { + let mut problems = Vec::new(); + let mut pass = 0; + for c in tiers.iter().flat_map(|t| &t.cases) { + let Some(r) = results.get(c) else { + problems.push(format!("{c}: no result")); + continue; + }; + pass += usize::from(r["status"] == "pass"); + let changed = changes(r, golden["cases"].get(c)); + if !changed.is_empty() { + problems.push(format!("CHANGED ({}): {c}: {}", r["status"].as_str().unwrap_or("?"), changed.join("; "))); + } + } + let n: usize = tiers.iter().map(|t| t.cases.len()).sum(); + if problems.is_empty() { + Ok(format!("kernel ladder gate: {pass}/{n} pass, every case as the golden records it")) + } else { + Err(format!( + "kernel ladder gate: {} case(s) differ from tests/kernel-cases/ladder/golden.json (re-record with --record \"why\" if the change is right):\n {}", + problems.len(), + problems.join("\n ") + )) + } +} + fn secs(v: &Value) -> String { v.as_f64().map(|s| format!("{s:.3}")).unwrap_or_else(|| "–".into()) } @@ -218,6 +258,10 @@ pub fn run(rest: &[&str]) -> Result<(), String> { Some(None) => return Err("--record needs a reason: --record \"why the golden changes\"".into()), Some(Some(r)) => Some(r.to_string()), }; + let gate_run = rest.contains(&"--gate"); + if gate_run && reason.is_some() { + return Err("--gate and --record: a gate run compares with the golden, a record run rewrites it".into()); + } let mut o = options(rest, "target/kernel-ladder")?; // Times are compared with the golden: cases run one at a time unless --jobs says otherwise, // so they don't share the machine with each other. @@ -236,7 +280,10 @@ pub fn run(rest: &[&str]) -> Result<(), String> { save_results(&o, &all)?; let results: BTreeMap = all.into_iter().map(|r| (rel_of(&r), r)).collect(); let golden = read(&golden_path()); - if let Some(reason) = &reason { + if gate_run { + // A gate run grades against the golden; the report stays the last full run's. + println!("{}", gate(&tiers, &results, &golden)?); + } else if let Some(reason) = &reason { record(reason, &manifest, &tiers, &results, &golden)?; let tiers = self::tiers(&read(&manifest_path()))?; println!("{}", write_report(&tiers, &results, &read(&golden_path()))?); @@ -264,6 +311,24 @@ mod tests { assert_eq!(flags(&same, None), ["no golden"]); } + #[test] + fn the_gate_fails_on_any_change_but_time() { + let t = vec![Tier { tier: "01-a".into(), what: String::new(), cases: vec!["ladder/01-a/001-x".into(), "ladder/01-a/002-y".into()] }]; + let golden = json!({"cases": { + "ladder/01-a/001-x": {"status": "pass", "seconds": 1.0, "measured": {"faces": 6}}, + "ladder/01-a/002-y": {"status": "fail", "seconds": 1.0} + }}); + let run = |x: Value, y: Value| gate(&t, &BTreeMap::from([("ladder/01-a/001-x".to_string(), x), ("ladder/01-a/002-y".to_string(), y)]), &golden); + // Ten times slower is the machine, not a change. + assert!(run(json!({"status": "pass", "seconds": 10.0, "measured": {"faces": 6}}), json!({"status": "fail"})).is_ok()); + let fixed = run(json!({"status": "pass", "measured": {"faces": 6}}), json!({"status": "pass"})).unwrap_err(); + assert!(fixed.contains("CHANGED (pass): ladder/01-a/002-y: status was fail"), "{fixed}"); + let counts = run(json!({"status": "pass", "measured": {"faces": 7}}), json!({"status": "fail"})).unwrap_err(); + assert!(counts.contains("faces 7 (golden 6)"), "{counts}"); + let unrecorded = gate(&t, &BTreeMap::new(), &golden).unwrap_err(); + assert!(unrecorded.contains("ladder/01-a/001-x: no result"), "{unrecorded}"); + } + #[test] fn tiers_need_paths() { let m = json!({"tiers": [{"tier": "01-a", "what": "w", "cases": ["ladder/01-a/001-x"]}]}); diff --git a/xtask/src/main.rs b/xtask/src/main.rs index 6916254c33..13be4f8d77 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -55,12 +55,13 @@ commands: each failing SMLib call (--cli: a solidcraft-cli built with --features smlib) --known-failing FILE: a regression gate instead of the report: fails on a case that fails and FILE does not list, and on a listed case that passes - (tests/kernel-cases/fillets/KNOWN_FAILING, CI's fillet rig job) - --ladder [--record REASON]: the complexity ladder instead + (tests/kernel-cases//KNOWN_FAILING, CI's Kernel rig workflow) + --ladder [--record REASON | --gate]: the complexity ladder instead (tests/kernel-cases/ladder/ladder.json, tiers in order), graded and compared with its timing golden (one case at a time unless --jobs), into docs/kernel-ladder.md (DIR default target/kernel-ladder); --record re-measures - the golden and re-sorts each tier by time + the golden and re-sorts each tier by time; --gate fails on any case whose status + or counts differ from the golden, instead of the report action-matrix [--bless] [--results FILE] [--from DIR] try every ordinary action on every face, edge, vertex and body of the design corpus and compare with tests/action_matrix/baseline.json (--bless rewrites it;