diff --git a/.github/workflows/fillet-rig.yml b/.github/workflows/fillet-rig.yml new file mode 100644 index 0000000000..57098f1134 --- /dev/null +++ b/.github/workflows/fillet-rig.yml @@ -0,0 +1,109 @@ +# The fillet rig as a regression gate: a pull request that touches fillet code (the paths below) +# runs every case in tests/kernel-cases/fillets/ on SMLib, through one release build of +# solidcraft-cli, and fails on a regression (a case that fails and +# tests/kernel-cases/fillets/KNOWN_FAILING does not list) or on a listed case that now passes +# (it is fixed: remove its line, so the list only shrinks). See tests/kernel-cases/README.md. +# +# GitHub filters a workflow by whole files, so the fillet parts of crates/doc/src/eval*.rs and +# crates/engine/src/cmd/features.rs select it whenever those files change. Pushes to main and +# develop that touch the same paths run it too and save the compiler cache (a Blacksmith sticky +# disk) that pull requests then mount read-only. +name: Fillet rig + +on: + pull_request: + types: [opened, synchronize, reopened, ready_for_review] + paths: + - "crates/kernel/src/blend*.rs" + - "crates/kernel/src/fillet*.rs" + - "crates/kernel/src/*corner*.rs" + - "crates/kernel/src/continuity*.rs" + - "crates/smlib/**" + - "crates/doc/src/eval*.rs" + - "crates/engine/src/cmd/features.rs" + - "tests/kernel-cases/fillets/**" + # The rig itself: the grading, the runner and this workflow. + - "apps/solidcraft-cli/src/kernel_cases.rs" + - "xtask/src/kernel_cases.rs" + - "scripts/fetch-smlib.sh" + - ".github/workflows/fillet-rig.yml" + push: + branches: [main, develop] + # The same paths as pull_request. + paths: + - "crates/kernel/src/blend*.rs" + - "crates/kernel/src/fillet*.rs" + - "crates/kernel/src/*corner*.rs" + - "crates/kernel/src/continuity*.rs" + - "crates/smlib/**" + - "crates/doc/src/eval*.rs" + - "crates/engine/src/cmd/features.rs" + - "tests/kernel-cases/fillets/**" + # The rig itself: the grading, the runner and this workflow. + - "apps/solidcraft-cli/src/kernel_cases.rs" + - "xtask/src/kernel_cases.rs" + - "scripts/fetch-smlib.sh" + - ".github/workflows/fillet-rig.yml" + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: fillet-rig-${{ github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +env: + CARGO_TERM_COLOR: always + +jobs: + fillets: + name: Fillet rig + # Draft pull requests run no CI, as in ci.yml. + if: github.event_name != 'pull_request' || !github.event.pull_request.draft + runs-on: blacksmith-8vcpu-ubuntu-2404 + timeout-minutes: 60 + env: + CARGO_INCREMENTAL: "0" + RUSTC_WRAPPER: sccache + SCCACHE_DIR: /home/runner/.cache/sccache + SCCACHE_CACHE_SIZE: 10G + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - name: Size parallel compiles to the job's memory + run: scripts/ci/build-jobs.sh + - uses: dtolnay/rust-toolchain@stable + # Only main and develop commit the cache; pull requests and forks mount a clone and never + # write back. + - name: Mount the compiler cache + uses: useblacksmith/stickydisk@3f6be1451e3cad893b778bc93495c80452a5da26 # v1.7.2 + continue-on-error: true + with: + key: ${{ github.repository }}-sccache-fillet-rig + path: /home/runner/.cache/sccache + commit: ${{ (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/develop') && 'on-change' || 'false' }} + - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 + with: + key: fillet-rig + save-if: ${{ github.ref == 'refs/heads/main' || github.ref == 'refs/heads/develop' }} + - name: Fetch SMLib (crates/smlib/SMLIB_REV) + shell: bash + run: scripts/fetch-smlib.sh + - name: Set up compiler cache + uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 + with: + version: v0.16.0 + - name: Build the CLI (release, SMLib) + run: cargo build --release --locked -p solidcraft-cli --features smlib + - name: Run the fillet cases against the known-failing list + run: >- + cargo run -q --locked -p xtask -- kernel-cases --area fillets + --cli target/release/solidcraft-cli --out "$RUNNER_TEMP/fillet-rig" + --known-failing tests/kernel-cases/fillets/KNOWN_FAILING + - name: Keep the results + if: always() + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: fillet-rig-results + path: ${{ runner.temp }}/fillet-rig/results.json + if-no-files-found: ignore diff --git a/tests/kernel-cases/README.md b/tests/kernel-cases/README.md index d07eabe33f..632291ac5b 100644 --- a/tests/kernel-cases/README.md +++ b/tests/kernel-cases/README.md @@ -97,6 +97,28 @@ the frozen truck-geometry PRs, a `kernel-paused` PR), for the report's coverage | `tolerance` | Optional overrides: `rel`, `abs_mm`, `mesh_mm`, `weld_mm`. | | `missing` | A capability SolidCraft has no command or parameter for yet (`"solid.thread has no start count"`): the case is reported **not supported yet** without running, so the gap stays visible. | +## The fillet gate + +The fillet cases are the fillet regression suite: each is one small case of one issue, with its +exact answer. `.github/workflows/fillet-rig.yml` runs them on every pull request (not a draft) +that touches fillet code: `crates/kernel/src/{blend*,fillet*,*corner*,continuity*}.rs`, +`crates/smlib/**` (`SMLIB_REV` and `SMLIB_PREBUILT` included), `crates/doc/src/eval*.rs`, +`crates/engine/src/cmd/features.rs`, `tests/kernel-cases/fillets/**` and the rig's own code +(GitHub filters whole files, so any change to `eval*.rs` or `features.rs` runs it). One release +build of `solidcraft-cli`, then: + +```sh +cargo xtask kernel-cases --area fillets --cli target/release/solidcraft-cli \ + --known-failing tests/kernel-cases/fillets/KNOWN_FAILING +``` + +`fillets/KNOWN_FAILING` lists the cases that fail today, sorted, one a line with its cause +(` `; `#` starts a comment). The gate fails on a **regression**, a case +that fails (or crashes, hangs, or is not supported yet) and is not listed, and on a **fixed** case, +one listed that passes: a fix removes its line in the same pull request, so the list only shrinks. +A new case that fails today goes in with its line. With `--known-failing` the run writes no +report; `docs/kernel-cases.md` stays the full run's. + ## The complexity ladder `ladder/` orders kernel operations by complexity instead of by where they went wrong: tiers from diff --git a/xtask/src/kernel_cases.rs b/xtask/src/kernel_cases.rs index 7e01c132e2..ff9499f1f1 100644 --- a/xtask/src/kernel_cases.rs +++ b/xtask/src/kernel_cases.rs @@ -30,6 +30,8 @@ pub(crate) struct Options { pub(crate) jobs: usize, /// The run's output folder: `results.json` and the SMLib dumps (`dumps/`). out: PathBuf, + /// `--known-failing FILE`: run as a regression gate against that baseline (see [`gate`]). + known_failing: Option, } /// The run's options; `out` is its output folder unless `--out` gives one. @@ -38,7 +40,11 @@ pub(crate) fn options(rest: &[&str], out: &str) -> Result { let secs = arg("--timeout").map(|t| t.parse::().map_err(|e| format!("--timeout: {e}"))).transpose()?.unwrap_or(DEFAULT_TIMEOUT_S); let jobs = arg("--jobs").map(|j| j.parse::().map_err(|e| format!("--jobs: {e}"))).transpose()?.unwrap_or(4).clamp(1, 32); let out = arg("--out").map_or_else(|| root().join(out), PathBuf::from); - Ok(Options { cli: arg("--cli"), area: arg("--area"), timeout: Duration::from_secs(secs), jobs, out }) + let known_failing = arg("--known-failing").map(|p| { + let p = PathBuf::from(p); + if p.is_absolute() { p } else { root().join(p) } + }); + Ok(Options { cli: arg("--cli"), area: arg("--area"), timeout: Duration::from_secs(secs), jobs, out, known_failing }) } /// Every case's full result as JSON, into the run's output folder. @@ -270,7 +276,7 @@ fn write_report(dir: &Path, list: &[PathBuf], results: &[Value]) -> Result-/`). See [`tests/kernel-cases/README.md`](../tests/kernel-cases/README.md) for the format.\n\n", + "# Kernel regression rig\n\nGenerated by `cargo xtask kernel-cases`. Every geometry case the old kernel (truck) got wrong, as an issue, a PR or a crash report recorded it, is a use case in [`tests/kernel-cases/`](../tests/kernel-cases/) with its own right answer: numbers from analysis or a hand calculation (**analytic**) where the geometry has them, and otherwise the invariants (**invariants**): the operation succeeds, or refuses cleanly where that is the right answer; each body is one closed shell that passes the record's structure and geometry checks and SMLib's own check; the expected face, edge and vertex counts. Never a kernel's output, and never a commercial kernel's.\n\nThe cases run on SMLib, SolidCraft's kernel. A failing case stays failing until the kernel is fixed; there is no workaround on SolidCraft's side. A failed SMLib call leaves its inputs for replay in the run's output folder (`target/kernel-cases/dumps/-/`). See [`tests/kernel-cases/README.md`](../tests/kernel-cases/README.md) for the format. The fillet cases are also a regression gate: CI's Fillet rig workflow runs them on every pull request that touches fillet code and fails on a case that fails and [`fillets/KNOWN_FAILING`](../tests/kernel-cases/fillets/KNOWN_FAILING) does not list, or a listed case that passes.\n\n", ); md += "## Coverage\n\n| Area | Cases | Results |\n|---|---|---|\n"; for (area, (n, statuses)) in &areas { @@ -335,6 +341,101 @@ pub fn run(rest: &[&str]) -> Result<(), String> { } let results = run_all(&o, &list); save_results(&o, &results)?; + if let Some(file) = &o.known_failing { + // A gate run grades one set against its baseline; the report stays the full run's. + let text = std::fs::read_to_string(file).map_err(|e| format!("{}: {e}", file.display()))?; + let base = file.parent().unwrap_or(&dir); + let paths: Vec = list.iter().map(|c| c.strip_prefix(base).unwrap_or(c).to_string_lossy().replace('\\', "/")).collect(); + let statuses: Vec<&str> = results.iter().map(|r| r["status"].as_str().unwrap_or("?")).collect(); + println!("{}", gate(&text, &paths, &statuses).map_err(|e| format!("{}: {e}", file.display()))?); + return Ok(()); + } println!("{}", write_report(&dir, &list, &results)?); Ok(()) } + +/// The cases a known-failing baseline lists, with their causes: one per line, ` `, +/// the case's directory relative to the baseline's own; `#` starts a comment line. The list is +/// sorted with no case twice, and every case has a cause. +fn known_failing(text: &str) -> Result, String> { + let mut out: Vec<(&str, &str)> = Vec::new(); + for (n, line) in text.lines().enumerate() { + let line = line.trim(); + if line.is_empty() || line.starts_with('#') { + continue; + } + let (case, cause) = line.split_once(char::is_whitespace).map_or((line, ""), |(c, w)| (c, w.trim())); + if cause.is_empty() { + return Err(format!("line {}: {case} gives no cause (` `)", n + 1)); + } + if let Some((last, _)) = out.last() + && *last >= case + { + return Err(format!( + "line {}: {case} is {} {last}: keep the list sorted, each case once", + n + 1, + if *last == case { "a repeat of" } else { "out of order after" } + )); + } + out.push((case, cause)); + } + Ok(out) +} + +/// The regression gate: `cases` (paths relative to the baseline's directory) ran with +/// `statuses`; every case passes except those the baseline `text` lists. A case that fails and +/// is not listed is a regression; a listed case that passes is fixed and must leave the list, so +/// the list only shrinks. Either fails the gate, as does a listed case that is not in the set. +fn gate(text: &str, cases: &[String], statuses: &[&str]) -> Result { + let listed = known_failing(text)?; + let mut problems = Vec::new(); + for (case, _) in &listed { + if !cases.iter().any(|c| c == case) { + problems.push(format!("listed but not a case of this set: {case}")); + } + } + let mut failing = 0; + for (case, status) in cases.iter().zip(statuses) { + match (listed.iter().find(|(c, _)| c == case), *status == "pass") { + (None, false) => problems.push(format!("REGRESSION ({status}): {case}")), + (Some(_), true) => problems.push(format!("FIXED, remove it from the known-failing list: {case}")), + (Some(_), false) => failing += 1, + (None, true) => {} + } + } + if problems.is_empty() { + Ok(format!("kernel-cases gate: {} pass, {failing} known failing, no regression", cases.len() - failing)) + } else { + Err(format!("kernel-cases gate:\n {}", problems.join("\n "))) + } +} + +#[cfg(test)] +mod tests { + use super::gate; + + fn run(text: &str, cases: &[(&str, &str)]) -> Result { + let names: Vec = cases.iter().map(|(c, _)| c.to_string()).collect(); + let statuses: Vec<&str> = cases.iter().map(|(_, s)| *s).collect(); + gate(text, &names, &statuses) + } + + #[test] + fn the_gate_fails_on_a_regression_and_on_a_fixed_case_still_listed() { + let list = "# today\na-case SMLib refuses the corner\nb-case G1 at the corner point\n"; + assert!(run(list, &[("a-case", "fail"), ("b-case", "crash"), ("c-case", "pass")]).is_ok()); + let regression = run(list, &[("a-case", "fail"), ("b-case", "fail"), ("c-case", "hang")]).unwrap_err(); + assert!(regression.contains("REGRESSION (hang): c-case"), "{regression}"); + let fixed = run(list, &[("a-case", "pass"), ("b-case", "fail"), ("c-case", "pass")]).unwrap_err(); + assert!(fixed.contains("FIXED, remove it from the known-failing list: a-case"), "{fixed}"); + let stale = run(list, &[("a-case", "fail"), ("c-case", "pass")]).unwrap_err(); + assert!(stale.contains("not a case of this set: b-case"), "{stale}"); + } + + #[test] + fn the_known_failing_list_is_sorted_with_a_cause_each() { + assert!(run("b x\na y\n", &[("a", "fail"), ("b", "fail")]).unwrap_err().contains("out of order")); + assert!(run("a x\na y\n", &[("a", "fail")]).unwrap_err().contains("a repeat of")); + assert!(run("a\n", &[("a", "fail")]).unwrap_err().contains("gives no cause")); + } +} diff --git a/xtask/src/main.rs b/xtask/src/main.rs index b1ae8f6f9c..6916254c33 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -47,12 +47,15 @@ commands: replay the reference tests (tests/oracle/*/recipe.json), compare with each case's expected.json and write docs/oracle.md (--cli: a solidcraft-cli already built, instead of cargo run --release) - kernel-cases [--area AREA] [--cli BIN] [--timeout S] [--jobs N] [--out DIR] + kernel-cases [--area AREA] [--cli BIN] [--timeout S] [--jobs N] [--out DIR] [--known-failing FILE] the kernel regression rig: run every case in tests/kernel-cases/// on SMLib (one process each, killed after --timeout, default 180 s), grade it against the case's own answer and write docs/kernel-cases.md; DIR (default target/kernel-cases) gets results.json and dumps/-/, the inputs of each failing SMLib call (--cli: a solidcraft-cli built with --features smlib) + --known-failing FILE: a regression gate instead of the report: fails on a case + that fails and FILE does not list, and on a listed case that passes + (tests/kernel-cases/fillets/KNOWN_FAILING, CI's fillet rig job) --ladder [--record REASON]: the complexity ladder instead (tests/kernel-cases/ladder/ladder.json, tiers in order), graded and compared with its timing golden (one case at a time unless --jobs), into