masi/solvecraft
1
0
Fork 0
mirror of https://github.com/bherbruck/solidcraft.git synced 2026-11-03 02:25:55 +00:00
Code Issues Packages Projects Releases 1 Wiki Activity

ci: fix local scheduling and Windows test baselines (#4941)

Browse Source 
* ci: repair local nextest scheduling and Windows fixtures

* test: refresh MSVC robustness references and isolate GNU policy

Integrate the Windows reference records from #3361 and the target-environment
selection and recording policy from #4159. Keep shared records and comparison
tolerances intact, and retain seed 36 as an existing failure tracked by #3583.

Co-authored-by: Ray <raykholo@users.noreply.github.com>
Co-authored-by: chris <christopher.nourse@gmail.com>

* test: make CI subprocess and revolve fixtures deterministic

---------

Co-authored-by: Ray <raykholo@users.noreply.github.com>
Co-authored-by: chris <christopher.nourse@gmail.com>
This commit is contained in:
cronos
2026-10-24 11:33:10 +00:00
committed by GitHub
parent 4612c5e6a9
commit 592f6a6fee
9 changed files with 625 additions and 90 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.config/nextest.toml
+47
View File
@@ -69,3 +69,50 @@ filter = """
))
"""
priority = 100
# Full local validation opts into its own scheduling policy. Keep GitHub CI's
# worker reservation and slow-test priorities above independent of this runner.
[profile.local-ci]
inherits = "ci"
[[profile.local-ci.overrides]]
filter = "binary(action_matrix) & test(=action_matrix_sample)"
threads-required = "num-cpus"
priority = 100
[[profile.local-ci.overrides]]
filter = "(package(solidcraft-engine) & test(=tests::robustness_goldens::seed_81)) | (package(solidcraft-kernel) & test(=tspline::tests::hostile_tsplines_never_panic))"
priority = 99
[[profile.local-ci.overrides]]
filter = """
(package(solidcraft-engine) & (
test(/^tests::robustness_goldens::seed_(81|45|20|24|95|02|57|40|18|48|32|46|85|61)$/)
| test(/^tests::robustness::/)
| test(=tests::hostile_params_never_panic)
| test(=cmd::clearance::tests::distances_are_exact_on_edges_and_curved_faces)
| test(=cmd::clipboard::tests::a_pasted_coil_joins_the_pasted_body)
| test(=tests::pleat_join::the_pleats_join_at_every_wall_from_2_4_to_2_6_mm)
| test(=tests::flush_join_loft::a_flange_flush_with_the_end_of_a_bent_tapered_tube_joins)
| test(=cmd::sheet::tests::perforated_sheet_takes_a_thousand_round_holes)
| test(=boolean_regression_tests::joined_loft_frames_plan_their_display_at_every_quality)
))
| (package(solidcraft-engine) & (binary(atomic_commands) | binary(step_export_regressions)))
| (package(solidcraft-kernel) & (
test(=curveblend::tests::rolling_ball_turns_about_a_rounded_corner_of_its_radius)
| test(=tests::fillet_tabbed_plate_outer_loop_and_partial_selection_after_hole)
| test(=tspline::tests::hostile_tsplines_never_panic)
| test(=fill::tests::regular_rims_of_three_to_eight_sides_fill_with_one_face_and_close_the_cup)
| test(/^issue_788_tests::equal_bores_crossing_/)
))
"""
priority = 98
# Native Windows shader/device creation must not overlap within the local run.
[test-groups.windows-gpu]
max-threads = 1
[[profile.local-ci.overrides]]
platform = "cfg(windows)"
filter = "package(solidcraft-gpu) | package(solidcraft) | (package(solidcraft-ui-egui) & test(/(offscreen|gpu)/))"
test-group = "windows-gpu"
crates/engine/src/tests/robustness_goldens.rs
+274 -79
View File
@@ -54,100 +54,295 @@ fn differs(want: &Value, got: &Value) -> Option<String> {
(!out.is_empty()).then(|| out.join("; "))
}
/// Platform records only override the seeds explicitly verified on that platform.
fn expected<'a>(file: &'a Value, seed: u64, platform: &str) -> &'a Value {
let key = seed.to_string();
file.get("platforms")
.and_then(|platforms| platforms.get(platform))
.and_then(|record| record.get("seeds"))
.and_then(|seeds| seeds.get(&key))
.unwrap_or(&file["seeds"][key])
/// Only Windows needs an environment split; other OS policies remain compatible.
#[derive(Clone, Copy, Debug)]
struct Target<'a> {
os: &'a str,
env: &'a str,
}
#[test]
fn platform_records_keep_shared_references_for_every_other_seed_and_platform() {
let file = json!({"seeds":{"1":{"volume":10},"2":{"volume":20}},"platforms":{"windows":{"seeds":{"1":{"volume":11}}}}});
assert_eq!(expected(&file, 1, "windows")["volume"], 11);
assert_eq!(expected(&file, 1, "linux")["volume"], 10);
assert_eq!(expected(&file, 2, "windows")["volume"], 20);
impl<'a> Target<'a> {
fn current() -> Self {
// cfg! describes the compilation target, including when cross-compiling.
let env = if cfg!(target_env = "msvc") {
"msvc"
} else if cfg!(target_env = "gnu") {
"gnu"
} else {
"other"
};
Self { os: std::env::consts::OS, env }
}
fn platform(self) -> Option<&'a str> {
match (self.os, self.env) {
("windows", "msvc") => Some("windows"), // Existing MSVC-verified records.
("windows", "gnu") => Some("windows-gnu"),
("windows", _) => None,
(os, _) => Some(os),
}
}
fn recording_platform(self, file: &Value) -> Result<Option<&'a str>, String> {
let platform = self.platform();
if self.os == "windows" {
return platform.map(Some).ok_or_else(|| format!("no recording policy for Windows target_env={}; define one before blessing", self.env));
}
Ok(platform.filter(|p| file["platforms"][*p]["seeds"].is_object()))
}
}
/// An override replaces the entire seed record, never individual fields.
/// The second return value identifies the selected reference in diagnostics.
fn expected<'a, 't>(file: &'a Value, seed: u64, target: Target<'t>) -> (&'a Value, &'t str) {
let key = seed.to_string();
if let Some(platform) = target.platform()
&& let Some(record) = file.get("platforms").and_then(|p| p.get(platform)).and_then(|p| p.get("seeds")).and_then(|s| s.get(&key))
{
return (record, platform);
}
(&file["seeds"][key], "shared")
}
fn difference(file: &Value, seed: u64, got: &Value, target: Target<'_>) -> Option<String> {
let (want, group) = expected(file, seed, target);
let change = if want.is_null() { Some("new".to_string()) } else { differs(want, got) };
change.map(|d| format!("seed {seed} [target_os={}, target_env={}, reference={group}]: {d}", target.os, target.env))
}
/// In-memory recording seam: regression tests never write the real reference file.
fn record(file: &mut Value, seed: u64, got: &Value, reason: &str, target: Target<'_>) -> Result<bool, String> {
if reason.trim().is_empty() {
return Err("ROBUSTNESS_GOLDENS_BLESS needs the reason for the change".to_string());
}
let platform = target.recording_platform(file)?;
if difference(file, seed, got, target).is_none() {
return Ok(false);
}
if file.get("about").is_none() {
file["about"] = json!(
"What each robustness seed builds (crates/engine/src/tests/robustness_goldens.rs, docs/robustness.md). Record changes with ROBUSTNESS_GOLDENS_BLESS=\"why\"."
);
}
if let Some(platform) = platform {
file["platforms"][platform]["seeds"][seed.to_string()] = got.clone();
} else {
file["seeds"][seed.to_string()] = got.clone();
}
let mut notes = file["notes"].as_array().cloned().unwrap_or_default();
// Coalesce only the same reason AND destination, including shared vs platform.
// Windows destinations encode the environment, so GNU and MSVC cannot coalesce.
let destination = platform.map(|p| json!(p));
match notes.last_mut() {
Some(last) if last["reason"] == json!(reason) && last.get("platform") == destination.as_ref() => {
let mut all: Vec<u64> = last["seeds"].as_array().into_iter().flatten().filter_map(Value::as_u64).chain([seed]).collect();
all.sort_unstable();
all.dedup();
last["seeds"] = json!(all);
}
_ => {
let mut note = json!({"reason": reason, "seeds": [seed]});
if let Some(platform) = platform {
note["platform"] = json!(platform);
}
notes.push(note);
}
}
file["notes"] = Value::Array(notes);
Ok(true)
}
fn serialize(file: &Value) -> String {
// Seeds in order, one per line, so a change is a small diff.
let seeds: BTreeMap<u64, Value> =
file["seeds"].as_object().into_iter().flatten().filter_map(|(k, v)| Some((k.parse::<u64>().ok()?, v.clone()))).collect();
let mut s = format!("{{\n \"about\": {},\n \"seeds\": {{\n", file["about"]);
let n = seeds.len();
for (i, (k, v)) in seeds.iter().enumerate() {
s += &format!(" \"{k}\": {}{}\n", v, if i + 1 < n { "," } else { "" });
}
let notes: Vec<String> = file["notes"].as_array().into_iter().flatten().map(|n| format!(" {n}")).collect();
s += " },\n";
if let Some(platforms) = file.get("platforms") {
s += &format!(" \"platforms\": {platforms},\n");
}
s += &format!(" \"notes\": [\n{}\n ]\n}}\n", notes.join(",\n"));
s
}
fn check(seed: u64) {
let got: BTreeMap<u64, Value> = BTreeMap::from([(seed, fuzz::golden(seed, MAX_STEPS))]);
let got = fuzz::golden(seed, MAX_STEPS);
let _lock = FILE.lock().unwrap_or_else(|e| e.into_inner());
let mut file = load();
let changed: Vec<(u64, String)> = got
.iter()
.filter_map(|(seed, g)| {
let w = expected(&file, *seed, std::env::consts::OS);
if w.is_null() { Some((*seed, "new".to_string())) } else { differs(w, g).map(|d| (*seed, d)) }
})
.collect();
let target = Target::current();
if let Ok(reason) = std::env::var("ROBUSTNESS_GOLDENS_BLESS") {
assert!(!reason.trim().is_empty(), "ROBUSTNESS_GOLDENS_BLESS needs the reason for the change");
if changed.is_empty() {
return;
if record(&mut file, seed, &got, &reason, target).unwrap_or_else(|e| panic!("{e}")) {
std::fs::write(path(), serialize(&file)).unwrap_or_else(|e| panic!("{}: {e}", path().display()));
}
if file.get("about").is_none() {
file["about"] = json!(
"What each robustness seed builds (crates/engine/src/tests/robustness_goldens.rs, docs/robustness.md). Record changes with ROBUSTNESS_GOLDENS_BLESS=\"why\"."
);
}
let platform = std::env::consts::OS;
let platform_record = file["platforms"][platform]["seeds"].is_object();
for (seed, g) in &got {
if platform_record {
if changed.iter().any(|(changed, _)| changed == seed) {
file["platforms"][platform]["seeds"][seed.to_string()] = g.clone();
}
} else {
file["seeds"][seed.to_string()] = g.clone();
}
}
let mut notes = file["notes"].as_array().cloned().unwrap_or_default();
let seeds: Vec<u64> = changed.iter().map(|(s, _)| *s).collect();
// The parts of one record (the tests run side by side) share one note.
match notes.last_mut() {
Some(last) if last["reason"] == json!(reason) && (!platform_record || last["platform"] == json!(platform)) => {
let mut all: Vec<u64> = last["seeds"].as_array().into_iter().flatten().filter_map(Value::as_u64).chain(seeds).collect();
all.sort_unstable();
all.dedup();
last["seeds"] = json!(all);
}
_ => {
let mut note = json!({"reason": reason, "seeds": seeds});
if platform_record {
note["platform"] = json!(platform);
}
notes.push(note);
}
}
file["notes"] = Value::Array(notes);
// Seeds in order, one per line, so a change is a small diff.
let seeds: BTreeMap<u64, Value> =
file["seeds"].as_object().into_iter().flatten().filter_map(|(k, v)| Some((k.parse::<u64>().ok()?, v.clone()))).collect();
let mut s = format!("{{\n \"about\": {},\n \"seeds\": {{\n", file["about"]);
let n = seeds.len();
for (i, (k, v)) in seeds.iter().enumerate() {
s += &format!(" \"{k}\": {}{}\n", v, if i + 1 < n { "," } else { "" });
}
let notes: Vec<String> = file["notes"].as_array().into_iter().flatten().map(|n| format!(" {n}")).collect();
s += " },\n";
if let Some(platforms) = file.get("platforms") {
s += &format!(" \"platforms\": {platforms},\n");
}
s += &format!(" \"notes\": [\n{}\n ]\n}}\n", notes.join(",\n"));
std::fs::write(path(), s).unwrap_or_else(|e| panic!("{}: {e}", path().display()));
return;
}
let change = difference(&file, seed, &got, target);
assert!(
changed.is_empty(),
"{} seed(s) build differently from tests/robustness/goldens.json (if that is intended, record it with ROBUSTNESS_GOLDENS_BLESS=\"why\" and say why in the pull request):\n{}",
changed.len(),
changed.iter().map(|(s, d)| format!("seed {s}: {d}")).collect::<Vec<_>>().join("\n")
change.is_none(),
"build differs from tests/robustness/goldens.json (if that is intended, record it with ROBUSTNESS_GOLDENS_BLESS=\"why\" and say why in the pull request):\n{}",
change.unwrap_or_default()
);
}
const MSVC: Target<'_> = Target { os: "windows", env: "msvc" };
const GNU: Target<'_> = Target { os: "windows", env: "gnu" };
const LINUX: Target<'_> = Target { os: "linux", env: "gnu" };
fn policy_fixture() -> Value {
json!({
"seeds": {"1": {"volume": 10, "faces": 2, "outcome": "ok"}, "2": {"volume": 20}},
"platforms": {
"windows": {"seeds": {"1": {"volume": 11, "outcome": "failed"}}, "verification": "MSVC"},
"macos": {"seeds": {"1": {"volume": 12}}}
},
"notes": [{"platform": "windows", "reason": "same reason", "seeds": [1]}]
})
}
#[test]
fn policy_selects_msvc_and_gnu_overrides_with_shared_fallback() {
let mut file = policy_fixture();
assert_eq!(expected(&file, 1, MSVC), (&file["platforms"]["windows"]["seeds"]["1"], "windows"));
assert_eq!(expected(&file, 1, GNU), (&file["seeds"]["1"], "shared"));
file["platforms"]["windows-gnu"]["seeds"]["1"] = json!({"volume": 13});
assert_eq!(expected(&file, 1, GNU), (&file["platforms"]["windows-gnu"]["seeds"]["1"], "windows-gnu"));
for target in [MSVC, GNU] {
assert_eq!(expected(&file, 2, target), (&file["seeds"]["2"], "shared"));
}
// A platform record is whole, including failure outcomes and missing fields.
assert!(expected(&file, 1, MSVC).0.get("faces").is_none());
assert_eq!(expected(&file, 1, MSVC).0["outcome"], "failed");
assert!(differs(expected(&file, 1, MSVC).0, &file["seeds"]["1"]).is_some());
assert!(expected(&file, 1, GNU).0.get("faces").is_none());
assert!(expected(&file, 99, GNU).0.is_null());
}
#[test]
fn policy_preserves_non_windows_and_refuses_unknown_windows_recording() {
let mut file = policy_fixture();
let macos = Target { os: "macos", env: "" };
assert_eq!(expected(&file, 1, LINUX), (&file["seeds"]["1"], "shared"));
assert_eq!(expected(&file, 1, macos), (&file["platforms"]["macos"]["seeds"]["1"], "macos"));
assert_eq!(LINUX.recording_platform(&file), Ok(None));
assert_eq!(macos.recording_platform(&file), Ok(Some("macos")));
for env in ["", "other", "new-env"] {
let unknown = Target { os: "windows", env };
let before = file.clone();
assert_eq!(expected(&file, 1, unknown), (&file["seeds"]["1"], "shared"));
assert!(record(&mut file, 1, &json!({"volume": 30}), "why", unknown).is_err());
assert_eq!(file, before);
}
assert_eq!(expected(&file, 1, Target { os: "future-os", env: "gnu" }).1, "shared");
}
#[test]
fn policy_records_only_changed_gnu_seeds_and_preserves_other_destinations_and_notes() {
let mut file = policy_fixture();
let before = file.clone();
let got = json!({"volume": 30, "outcome": "failed", "error": "known failure"});
assert_eq!(record(&mut file, 2, &got, "same reason", GNU), Ok(true));
assert_eq!(file["seeds"], before["seeds"]);
assert_eq!(file["platforms"]["windows"], before["platforms"]["windows"]);
assert_eq!(file["platforms"]["macos"], before["platforms"]["macos"]);
assert_eq!(file["notes"][0], before["notes"][0]);
assert_eq!(file["notes"][1], json!({"platform": "windows-gnu", "reason": "same reason", "seeds": [2]}));
assert_eq!(expected(&file, 2, GNU), (&got, "windows-gnu"));
assert!(file["platforms"]["windows-gnu"]["seeds"].get("1").is_none());
assert_eq!(record(&mut file, 1, &got, "same reason", GNU), Ok(true));
assert_eq!(file["notes"][1]["seeds"], json!([1, 2]));
let unchanged = file.clone();
assert_eq!(record(&mut file, 1, &got, "same reason", GNU), Ok(false));
assert_eq!(file, unchanged);
assert_eq!(record(&mut file, 1, &json!({"volume": 31}), "same reason", GNU), Ok(true));
assert_eq!(file["notes"][1]["seeds"], json!([1, 2]));
assert!(expected(&file, 1, GNU).0.get("error").is_none());
assert_eq!(serde_json::from_str::<Value>(&serialize(&file)).ok().as_ref(), Some(&file));
}
#[test]
fn policy_isolates_msvc_recording_even_without_a_namespace() {
for target in [MSVC, GNU] {
let mut file = json!({"seeds": {"1": {"volume": 10}}, "notes": []});
let before = file.clone();
assert_eq!(record(&mut file, 1, &json!({"volume": 10}), "why", target), Ok(false));
assert_eq!(file, before); // No spurious namespace or note for a matching fallback.
assert_eq!(record(&mut file, 1, &json!({"volume": 11}), "why", target), Ok(true));
assert_eq!(file["seeds"], before["seeds"]);
assert_eq!(expected(&file, 1, target).0["volume"], 11);
assert_eq!(file["notes"][0]["platform"], json!(target.platform()));
}
}
#[test]
fn policy_notes_never_coalesce_across_destinations_in_either_direction() {
let mut file = policy_fixture();
for target in [GNU, MSVC, LINUX, GNU, LINUX, MSVC] {
let previous_notes = file["notes"].as_array().cloned().unwrap_or_default();
let got = json!({"volume": 100 + previous_notes.len()});
assert_eq!(record(&mut file, 1, &got, "same reason", target), Ok(true));
let notes = file["notes"].as_array().cloned().unwrap_or_default();
assert_eq!(notes.len(), previous_notes.len() + 1);
assert_eq!(&notes[..previous_notes.len()], previous_notes);
}
let before = file["notes"].clone();
assert_eq!(record(&mut file, 2, &json!({"volume": 50}), "different reason", MSVC), Ok(true));
assert_eq!(file["notes"].as_array().map(Vec::len), before.as_array().map(|n| n.len() + 1));
let before = file.clone();
assert!(record(&mut file, 1, &json!({"volume": 99}), " ", GNU).is_err());
assert_eq!(file, before);
}
#[test]
fn policy_comparator_stays_strict_and_diagnostics_name_the_selected_reference() {
let want = json!({"volume": 100.0, "area": 10.0, "faces": 2, "outcome": "ok", "steps": ["box"], "error": null, "rejected": 0, "unsupported": 0});
let mut got = want.clone();
got["volume"] = json!(100.00005);
got["area"] = json!(10.000005);
assert_eq!(differs(&want, &got), None);
for (field, value) in [
("volume", json!(100.001)),
("area", json!(10.001)),
("faces", json!(3)),
("outcome", json!("failed")),
("steps", json!(["hole"])),
("error", json!("refused")),
("rejected", json!(1)),
("unsupported", json!(1)),
] {
let mut got = want.clone();
got[field] = value;
assert!(differs(&want, &got).is_some(), "{field}");
}
let file = policy_fixture();
let diagnostic = difference(&file, 1, &json!({"volume": 100}), GNU).unwrap_or_default();
assert!(diagnostic.contains("target_os=windows, target_env=gnu, reference=shared"), "{diagnostic}");
let diagnostic = difference(&file, 1, &json!({"volume": 100}), MSVC).unwrap_or_default();
assert!(diagnostic.contains("target_env=msvc, reference=windows"), "{diagnostic}");
}
#[test]
fn policy_non_windows_recording_keeps_existing_os_destination_and_shared_notes() {
let mut file = policy_fixture();
let before = file.clone();
let got = json!({"volume": 40});
let macos = Target { os: "macos", env: "" };
assert_eq!(record(&mut file, 1, &got, "same reason", macos), Ok(true));
assert_eq!(file["seeds"], before["seeds"]);
assert_eq!(file["platforms"]["windows"], before["platforms"]["windows"]);
assert_eq!(expected(&file, 1, macos), (&got, "macos"));
let platforms = file["platforms"].clone();
assert_eq!(record(&mut file, 1, &got, "same reason", LINUX), Ok(true));
assert_eq!(file["platforms"], platforms);
assert_eq!(file["notes"][2], json!({"reason": "same reason", "seeds": [1]}));
assert_eq!(record(&mut file, 2, &got, "same reason", LINUX), Ok(true));
assert_eq!(file["notes"][2]["seeds"], json!([1, 2]));
}
/// One test per seed, and the list of them.
macro_rules! seeds {
($($name:ident = $seed:literal),* $(,)?) => {
crates/ui-egui/src/dialogs_all_tests.rs
+17 -1
View File
@@ -60,7 +60,9 @@ fn representative(h: &Harness, command: &str) -> Dialog {
} else if input.accept & CURVES != 0 || command == "sheet.fold" && index == 0 {
vec![Sel::SketchCurve { id: path.clone() }]
} else if input.accept & AXES != 0 {
vec![Sel::Axis { name: "Z".into() }]
// The Revolve fixture's profile is in XY; its preview needs an axis in that plane.
let axis = if command == "solid.revolve" { "Y" } else { "Z" };
vec![Sel::Axis { name: axis.into() }]
} else if input.accept & (FACES | PLANAR_FACES) != 0 {
vec![face_sel(s, point).unwrap()]
} else if input.accept & EDGES != 0 {
@@ -293,6 +295,20 @@ fn registry_dialog_inventory_has_no_unclassified_feature_forms() {
}
}
#[test]
fn revolve_layout_fixture_builds_a_valid_solid() {
let mut h = fixture();
let d = representative(&h, "solid.revolve");
let before = h.app.session.model.results.len();
let commands = crate::dialogs::preview_commands(&h.app, &d).unwrap();
assert!(!commands.is_empty());
for (command, params) in commands {
h.app.run(&command, params).unwrap();
}
assert!(h.app.session.model.results.len() > before);
assert!(h.app.session.model.results.iter().all(|result| result.error.is_none()));
}
#[test]
fn new_cards_settle_before_the_first_click_in_both_layouts() {
for mode in [crate::shell::LayoutMode::Floating, crate::shell::LayoutMode::Docked] {
docs/book/src/dev/testing.md
+29
View File
@@ -68,6 +68,35 @@ with doc tests alongside. Arguments after `--` go to each test binary, for examp
`cargo xtask test -- shell`. To run the test binaries sequentially instead, use
`cargo test --workspace --profile ci`.
## Test-level scheduling for full local CI
Install [cargo-nextest](https://nexte.st/docs/installation/pre-built-binaries/), then set
`SOLIDCRAFT_TEST_RUNNER=nextest` when running `cargo xtask ci`. For example, on PowerShell:
```powershell
$env:SOLIDCRAFT_TEST_RUNNER = 'nextest'
cargo xtask ci
```
This optional backend builds the workspace tests once, then schedules individual tests using
the `local-ci` nextest profile, which inherits `ci`. The action matrix runs first, then the longest geometry
cases start before the other costly tests, so the ordinary queue does not stall behind an
all-slot reservation. It **ignores the default filter**,
so the heavy tests below still run, does not stop after a test failure, and does not retry failures.
Tests marked `#[ignore]` remain ignored, as with the ordinary full local CI.
The default concurrency is at most four tests; `NEXTEST_TEST_THREADS` overrides it. The action
matrix's child workers default to the smaller of that budget and the host CPU count;
`ACTION_MATRIX_JOBS` explicitly overrides this worker count. The local profile reserves CPU
slots for the matrix and keeps its scheduling changes separate from GitHub CI. On Windows,
the `solidcraft-gpu` and app packages, plus UI offscreen/GPU tests, use a serial group. The Wasm check and doc tests still run alongside,
and the web-offline gate still follows a successful native/Wasm/doc run.
The backend keeps Cargo and binary metadata under `target/xtask-nextest/` and reuses it for the
native run, freeing Cargo to run the side gates. Nextest prints each test's result and duration
as it finishes. `SOLIDCRAFT_TEST_RUNNER=libtest` (or leaving it unset) retains the original runner;
`cargo xtask test` and its Cargo/libtest argument forwarding are unchanged. Unknown runner names
fail explicitly.
## CI sharding and heavy tests
On GitHub, one job builds the tests into a [cargo-nextest](https://nexte.st/) archive and four
docs/robustness.md
+28 -6
View File
@@ -48,12 +48,30 @@ kernel, engine, doc, sketch and geom crates) run every seed but the slowest few
and the nightly run them all. Each seed is its own test
(`seed_00` … `seed_95`), so the CI shards and their cores share them out.
Explicit `platforms.<os>.seeds` records override only the listed seeds on that
operating system; all other seeds keep the shared records and strict tolerances.
The first four Windows records were independently reproduced on unmodified main
`1e984c8e`; seed 94 was independently reproduced on unmodified main `c5992386`
after the extrusion rebase. Both match the extrusion branch. Recording on a platform
with such entries updates its changed seeds and preserves shared references.
Reference selection uses the compilation target's operating system and environment.
On Windows, `platforms.windows.seeds` contains the MSVC-verified records;
`platforms.windows-gnu.seeds`, when present, contains GNU records. An override
replaces the whole seed record. Seeds missing from the selected group use the shared
record, never another Windows group's record. Non-Windows targets keep their existing
OS-group selection and shared fallback. Unknown Windows environments compare with the
shared records and refuse recording until an explicit policy is defined.
All 96 seeds and the strict measurement and topology tolerances remain unchanged.
The Windows records for seeds 1, 23 and 34 were independently reproduced on unmodified
main `1e984c8e`; seed 94 was independently reproduced on unmodified main `c5992386`
after the extrusion rebase. Both match the extrusion branch. Seed 13's was recorded on
unmodified main `65e6d9da`; seed 23's was re-recorded after #1653. Seed 36's, recorded
on main `79a843c9`, holds a failure: on Windows (and macOS) its last step leaves a body that fails the
validity check, where Linux builds it ok (#3583). The record only makes a further change
show; it does not accept the failure. Recording on a platform with such entries updates its
changed seeds and preserves shared references. A platform record replaces the shared
one whole, so a change that moves a seed with a platform record needs that seed checked
on that platform too: seed 67's Windows record went stale this way after #2064 and
#2247 and was removed once Windows built the shared record again.
The Windows records were made with the MSVC target (`x86_64-pc-windows-msvc`) and all
six were checked again on main `e9490093`. GNU builds (`x86_64-pc-windows-gnu`)
use the shared records unless an explicit GNU override exists, so MSVC-only outcomes
do not leak into GNU validation (#4104).
To record an intended change, give the reason:
@@ -62,6 +80,10 @@ ROBUSTNESS_GOLDENS_BLESS="why it changed" cargo test --profile ci -p solidcraft-
```
The changed seeds and the reason are added to the file's `notes`; say why in the pull request too.
Recording on Windows always writes to the current environment's own group, even when
that group does not exist yet. It preserves shared records and the other Windows
environment's records. Non-Windows recording keeps its existing OS-group policy.
Notes coalesce only when both the reason and destination match.
The generator feeds the current geometry's bounds and face/edge picks back into later
commands. The same RNG seed can therefore generate different command coordinates or picks
tests/robustness/goldens.json
+6 -2
View File
@@ -98,7 +98,7 @@
"94": {"area":3480.70333,"bodies":1,"edges":63,"error":null,"faces":27,"outcome":"ok","rejected":1,"shells":2,"steps":["revolved section","cut cylinder","fillet","hole","rectangular pattern","cut box","cut box"],"unsupported":0,"volume":5456.446856},
"95": {"area":2375.340299,"bodies":1,"edges":33,"error":null,"faces":12,"outcome":"ok","rejected":2,"shells":1,"steps":["revolved section","hole","cut cylinder","cut sphere","circular pattern","cut sphere"],"unsupported":1,"volume":6704.220611}
},
"platforms": {"windows":{"seeds":{"1":{"area":2642.450837,"bodies":1,"edges":32,"error":null,"faces":13,"outcome":"ok","rejected":0,"shells":1,"steps":["revolved section","cut cylinder","join cylinder"],"unsupported":0,"volume":6991.569339},"23":{"area":1795.696636,"bodies":1,"edges":15,"error":null,"faces":7,"outcome":"ok","rejected":0,"shells":1,"steps":["revolved section","cut sphere"],"unsupported":0,"volume":3198.694618},"34":{"area":1086.303608,"bodies":1,"edges":30,"error":null,"faces":14,"outcome":"ok","rejected":0,"shells":1,"steps":["revolved section","cut sphere","hole"],"unsupported":0,"volume":2134.3653},"67":{"area":7943.46222,"bodies":2,"edges":86,"error":null,"faces":36,"outcome":"ok","rejected":0,"shells":2,"steps":["extruded profile","cut cylinder","hole","join cylinder","join box","hole","chamfer","rectangular pattern"],"unsupported":2,"volume":27607.962186},"94":{"area":3470.837042,"bodies":1,"edges":63,"error":null,"faces":27,"outcome":"ok","rejected":1,"shells":2,"steps":["revolved section","cut cylinder","fillet","hole","rectangular pattern","cut box","cut box"],"unsupported":0,"volume":5456.438772}}}},
"platforms": {"windows":{"seeds":{"1":{"area":2642.450837,"bodies":1,"edges":32,"error":null,"faces":13,"outcome":"ok","rejected":0,"shells":1,"steps":["revolved section","cut cylinder","join cylinder"],"unsupported":0,"volume":6991.569339},"13":{"area":2101.521015,"bodies":1,"edges":103,"error":null,"faces":41,"outcome":"ok","rejected":1,"shells":1,"steps":["revolved section","cut cylinder","join box","rectangular pattern","cut cylinder","cut box","fillet","hole"],"unsupported":0,"volume":2396.939546},"23":{"area":1795.693105,"bodies":1,"edges":15,"error":null,"faces":7,"outcome":"ok","rejected":0,"shells":1,"steps":["revolved section","cut sphere"],"unsupported":0,"volume":3198.694089},"34":{"area":1086.303608,"bodies":1,"edges":30,"error":null,"faces":14,"outcome":"ok","rejected":0,"shells":1,"steps":["revolved section","cut sphere","hole"],"unsupported":0,"volume":2134.3653},"36":{"area":9964.083285,"bodies":1,"edges":45,"error":"lumps: shell boundaries intersect or touch; cavity ownership","faces":21,"outcome":"failed","rejected":0,"shells":2,"steps":["revolved section","cut cylinder","join box","fillet","join cylinder"],"unsupported":0,"volume":54852.503099},"94":{"area":3470.837042,"bodies":1,"edges":63,"error":null,"faces":27,"outcome":"ok","rejected":1,"shells":2,"steps":["revolved section","cut cylinder","fillet","hole","rectangular pattern","cut box","cut box"],"unsupported":0,"volume":5456.438772}}}},
"notes": [
{"reason":"First record: what each seed builds on main at 1a0100cd","seeds":[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95]},
{"reason":"seed 0 meshes differently since display tessellation plans (#805), which landed alongside these goldens (#826)","seeds":[0]},
@@ -148,6 +148,10 @@
{"reason":"A cut no longer comes out empty where a shell of one body meets nothing of the other (a sealed void, a body clear of the tool): truck placed such faces by their surfaces alone, which a turned body's reversed faces and a void's inward shell got wrong. Seeds 0, 45 and 77 each had a cut refused as empty that now takes what it should, and the generator builds different steps after it: seed 0 a sphere clear of the body (removes nothing), seed 45 a hole on the axis of a shelled turned body (widens its bore from r 2.5 to r 4 over 17.5 mm: 536.0 mm3), seed 77 a hole inside a turned body's bore (removes nothing; a later pattern copy drills r 1.5 through 27.5 mm: 194.4 mm3). Every step passes the validity and volume-direction checks.","seeds":[0,45,77]},
{"reason":"Trimmed faces of display meshes stay on their surfaces (#1653): seeds 1, 23, 55, 64 and 86 end with bodies whose trimmed faces were meshed off their surfaces; measured on the corrected meshes their area and volume move slightly (seed 86 most: area +9.6 mm2, volume -41.4 mm3, 0.2 %). Steps, outcome and counts are unchanged; recorded from Linux CI on main at the time of the rebase. The slow seeds 14, 20 and 40 (run only by the merge queue) move too: 14 area +0.92 mm2 / volume +2.64 mm3, 20 area +2.60 mm2 / volume +14.8 mm3, 40 area +0.61 mm2 / volume -2.25 mm3, from merge-group run 39450833424.","seeds":[1,14,20,23,40,55,64,86]},
{"reason":"measure (#2069): a body whose meshes leave a crack is summed about its box's centre instead of the origin, so the crack is closed by a cone to a point inside the body. Seeds 14, 20, 40, 55, 61 and 86 end with such a body; their volumes move from main's by -0.52 %, +0.11 %, -0.017 %, -0.001 %, -0.0015 % and +0.026 %. Steps, outcomes, counts and areas (to 3e-7 relative) are unchanged. Recorded on Linux.","seeds":[14,20,40,55,61,86]},
{"reason":"Chamfer cutters that reach into a nearby wall are not used (#4539): a chamfer in seed 9 was refused, as its result changed 33 mm3 of the part, more than the chamfer can (20). With its cutter refused it builds locally, so the design takes 8 steps without the final cut box. Area 2427.18 -> 2403.70 mm2, volume 6417.16 -> 6524.64 mm3, unsupported steps 2 -> 0; counts unchanged. Recorded on Linux.","seeds":[9]}
{"reason":"Chamfer cutters that reach into a nearby wall are not used (#4539): a chamfer in seed 9 was refused, as its result changed 33 mm3 of the part, more than the chamfer can (20). With its cutter refused it builds locally, so the design takes 8 steps without the final cut box. Area 2427.18 -> 2403.70 mm2, volume 6417.16 -> 6524.64 mm3, unsupported steps 2 -> 0; counts unchanged. Recorded on Linux.","seeds":[9]},
{"platform":"windows","reason":"Seed 67's Windows record (from main 1e984c8e) predates #2064 and #2247, which changed the shared record. On main 65e6d9da Windows builds the shared record exactly (one body, 35 faces/84 edges, 27606.450855 mm3, 7940.064 mm2): its generated coordinates differ from Linux's only in the last digits. The stale Windows record is removed, so Windows checks seed 67 against the shared record again.","seeds":[67]},
{"platform":"windows","reason":"Seed 13 on Windows (main 65e6d9da) builds what Linux built before #2064. Its first four steps match Linux to nine decimals. The second cylinder cut takes its base from the body bounds: y = -1.78e-14 on Windows, -1.95e-14 on Linux (math-library rounding, #924). At the same volume (2694.4476 mm3 on both) Windows leaves split the two coplanar faces that #2064 merges on Linux (37 faces/88 edges, not 35/84). The fillet picks its edge by index, so it rounds a different edge (2675.18 mm3, not 2688.99), and the hole after it removes the same 278.2 mm3. Every step passes the validity and volume-direction checks.","seeds":[13]},
{"platform":"windows","reason":"Seed 23's Windows record predates #1653 (trimmed faces of display meshes stay on their surfaces), which moved its shared record's area; on Windows the final body's area moves the same way (1795.696636 to 1795.693105 mm2). Steps, outcome and counts are unchanged.","seeds":[23]},
{"platform":"windows","reason":"Seed 36 on Windows (main 79a843c9) ends differently from Linux: the body after its last step (join cylinder) fails the validity check ('lumps: shell boundaries intersect or touch; cavity ownership'), so the run ends failed with 21 faces/45 edges, 54852.503099 mm3. macOS fails the same way; only Linux x86_64 builds it ok. Recorded so Windows notices any further change; the failure itself is tracked in #3583.","seeds":[36]}
]
}
xtask/src/main.rs
+2 -1
View File
@@ -13,6 +13,7 @@ mod layers;
mod licences;
mod lists;
mod test;
mod test_nextest;
mod ui_containers;
mod web_offline;
mod web_offline_test;
@@ -382,7 +383,7 @@ fn cmd_ci() -> Result<(), String> {
c.args(["clippy", "--workspace", "--all-targets", "--profile", "ci", "--", "-D", "warnings"]);
run(c, "cargo clippy -D warnings")?;
let (wasm, n) = wasm_check()?;
test::run(&[], &[], vec![(format!("wasm32-unknown-unknown check ({n} crates)"), wasm), test::doc_tests(&[], &[])])?;
test_nextest::run_selected(vec![(format!("wasm32-unknown-unknown check ({n} crates)"), wasm), test::doc_tests(&[], &[])])?;
web_offline_test::run(&root(), None, false)?;
eprintln!("ci: all gates passed");
Ok(())
xtask/src/test_nextest.rs
+221
View File
@@ -0,0 +1,221 @@
//! Optional test-level scheduling for full local CI. Reuse the built binaries so the Wasm and
//! doc-test jobs can use Cargo alongside the native run, without rebuilding native tests.
use std::ffi::OsString;
use std::path::PathBuf;
use std::process::{Command, Stdio};
use std::sync::mpsc;
use std::time::Instant;
use crate::cargo;
/// Select the full-CI runner without changing `xtask test`'s argument forwarding.
pub fn run_selected(side: Vec<(String, Command)>) -> Result<(), String> {
match std::env::var("SOLIDCRAFT_TEST_RUNNER") {
Err(std::env::VarError::NotPresent) => crate::test::run(&[], &[], side),
Ok(value) if value == "libtest" => crate::test::run(&[], &[], side),
Ok(value) if value == "nextest" => run(side),
_ => Err("SOLIDCRAFT_TEST_RUNNER must be libtest or nextest".into()),
}
}
pub struct Prepared {
pub cargo_metadata: PathBuf,
pub binaries_metadata: PathBuf,
}
impl Prepared {
pub fn command(&self) -> Command {
self.command_with_options(
std::env::var_os("NEXTEST_TEST_THREADS"),
std::env::var_os("ACTION_MATRIX_JOBS"),
std::thread::available_parallelism().map_or(1, |n| n.get()),
)
}
fn command_with_options(&self, threads: Option<OsString>, matrix_jobs: Option<OsString>, cpus: usize) -> Command {
let mut c = cargo();
c.args(["nextest", "run", "--cargo-metadata"]).arg(&self.cargo_metadata).arg("--binaries-metadata").arg(&self.binaries_metadata).args([
"--profile",
"local-ci",
"--ignore-default-filter",
"--no-fail-fast",
"--retries",
"0",
"--no-tests",
"fail",
"--test-threads",
]);
// One geometry test can also use Rayon. Bound the default; callers can explicitly
// choose another nextest concurrency without changing libtest's per-binary setting.
let threads = threads.unwrap_or_else(|| cpus.clamp(1, 4).to_string().into());
// The matrix has its own child-process pool. A nextest slot reservation alone
// does not bound it: without this it still spawns one worker per host CPU.
// Leave invalid thread values to nextest's own diagnostic before tests start.
let slots = match threads.to_str() {
Some("num-cpus") => cpus,
Some(value) => value.parse::<isize>().ok().map(|n| if n < 0 { cpus.saturating_sub(n.unsigned_abs()) } else { n as usize }).unwrap_or(1),
None => 1,
}
.clamp(1, cpus.max(1));
c.arg(threads).env("ACTION_MATRIX_JOBS", matrix_jobs.unwrap_or_else(|| slots.to_string().into()));
c
}
}
pub fn run(side: Vec<(String, Command)>) -> Result<(), String> {
let prepared = prepare()?;
execute(("nextest full workspace".into(), prepared.command()), side)
}
pub fn prepare() -> Result<Prepared, String> {
let mut version = cargo();
version.args(["nextest", "--version"]);
capture(version, "cargo nextest --version")
.map_err(|e| format!("{e}; install cargo-nextest first (https://nexte.st/docs/installation/pre-built-binaries/)"))?;
let mut metadata = cargo();
metadata.args(["metadata", "--format-version", "1", "--locked"]);
let cargo_json = capture(metadata, "cargo metadata")?;
let parsed: serde_json::Value = serde_json::from_slice(&cargo_json).map_err(|e| format!("Cargo metadata: {e}"))?;
let target = parsed.get("target_directory").and_then(|v| v.as_str()).ok_or("Cargo metadata has no target_directory")?;
let dir = PathBuf::from(target).join("xtask-nextest").join(std::process::id().to_string());
std::fs::create_dir_all(&dir).map_err(|e| format!("create {}: {e}", dir.display()))?;
let cargo_metadata = dir.join("cargo-metadata.json");
std::fs::write(&cargo_metadata, cargo_json).map_err(|e| format!("write {}: {e}", cargo_metadata.display()))?;
let mut build = cargo();
build.args(["nextest", "list", "--workspace", "--cargo-profile", "ci", "--locked", "--list-type", "binaries-only", "--message-format", "json"]);
let binaries_json = capture(build, "cargo nextest list (build all workspace tests)")?;
let binaries_metadata = dir.join("binaries-metadata.json");
// Retain both metadata files under target for diagnostics and reproducible listing.
std::fs::write(&binaries_metadata, binaries_json).map_err(|e| format!("write {}: {e}", binaries_metadata.display()))?;
Ok(Prepared { cargo_metadata, binaries_metadata })
}
fn capture(mut c: Command, label: &str) -> Result<Vec<u8>, String> {
eprintln!("$ {label}");
let out = c.stdin(Stdio::null()).stderr(Stdio::inherit()).output().map_err(|e| format!("{label}: {e}"))?;
if out.status.success() { Ok(out.stdout) } else { Err(format!("{label}: {}", out.status)) }
}
/// Keep every gate running after failures, and stream native test names/times as they finish.
pub fn execute(native: (String, Command), side: Vec<(String, Command)>) -> Result<(), String> {
let (tx, rx) = mpsc::channel();
let mut failures = Vec::new();
std::thread::scope(|scope| {
let tx_native = tx.clone();
scope.spawn(move || {
let _ = tx_native.send(run_job(native));
});
scope.spawn(move || {
for job in side {
let _ = tx.send(run_job(job));
}
});
for (label, result, seconds) in rx {
match result {
Ok(()) => eprintln!("ok {label:<44} {seconds:6.1}s"),
Err(e) => {
eprintln!("FAIL {label:<44} {seconds:6.1}s {e}");
failures.push(label);
}
}
}
});
if failures.is_empty() { Ok(()) } else { Err(format!("failed: {}", failures.join(", "))) }
}
fn run_job((label, mut c): (String, Command)) -> (String, Result<(), String>, f64) {
let started = Instant::now();
let result = c.stdin(Stdio::null()).status().map_err(|e| e.to_string()).and_then(|s| if s.success() { Ok(()) } else { Err(s.to_string()) });
(label, result, started.elapsed().as_secs_f64())
}
#[cfg(test)]
mod tests {
use super::*;
use std::sync::atomic::{AtomicU64, Ordering};
#[test]
fn full_ci_does_not_drop_heavy_tests_stop_on_failure_or_retry_failures() {
let p = Prepared { cargo_metadata: "cargo.json".into(), binaries_metadata: "binaries.json".into() };
let c = p.command_with_options(Some("3".into()), None, 16);
let args: Vec<_> = c.get_args().map(|s| s.to_string_lossy().into_owned()).collect();
assert!(args.windows(2).any(|s| s == ["--profile", "local-ci"]));
assert!(args.iter().any(|s| s == "--ignore-default-filter"));
assert!(args.iter().any(|s| s == "--no-fail-fast"));
assert!(args.windows(2).any(|s| s == ["--retries", "0"]));
assert!(args.windows(2).any(|s| s == ["--no-tests", "fail"]));
assert!(args.windows(2).any(|s| s == ["--test-threads", "3"]));
assert!(!args.iter().any(|s| s == "--workspace" || s == "--cargo-profile"));
}
#[test]
fn matrix_workers_follow_the_effective_nextest_budget() {
let p = Prepared { cargo_metadata: "cargo.json".into(), binaries_metadata: "binaries.json".into() };
for (cpus, threads, expected) in [
(16, None, "4"),
(2, None, "2"),
(1, None, "1"),
(16, Some("1"), "1"),
(16, Some("6"), "6"),
(16, Some("num-cpus"), "16"),
(16, Some("-2"), "14"),
(2, Some("-8"), "1"),
(2, Some("8"), "2"),
] {
let c = p.command_with_options(threads.map(Into::into), None, cpus);
assert_eq!(c.get_envs().find(|(k, _)| *k == "ACTION_MATRIX_JOBS").and_then(|(_, v)| v), Some(std::ffi::OsStr::new(expected)));
}
let c = p.command_with_options(Some("3".into()), Some("2".into()), 16);
assert_eq!(c.get_envs().find(|(k, _)| *k == "ACTION_MATRIX_JOBS").and_then(|(_, v)| v), Some(std::ffi::OsStr::new("2")));
}
#[test]
fn subprocess_fixture() {
let Some(path) = std::env::var_os("SOLIDCRAFT_XTASK_TEST_MARKER") else { return };
std::fs::write(path, b"ran").unwrap();
assert_ne!(std::env::var("SOLIDCRAFT_XTASK_TEST_FAIL").ok().as_deref(), Some("1"), "intentional child-process failure");
}
fn fixture(path: &std::path::Path, fail: bool) -> Command {
let mut c = Command::new(std::env::current_exe().unwrap());
c.args(["--exact", "test_nextest::tests::subprocess_fixture", "--test-threads=1"])
.env("SOLIDCRAFT_XTASK_TEST_MARKER", path)
.env("SOLIDCRAFT_XTASK_TEST_FAIL", if fail { "1" } else { "0" })
// Keep nested libtest reports out of the parent harness transcript.
.stdout(Stdio::null())
.stderr(Stdio::null());
c
}
#[test]
fn subprocess_regressions_leave_one_unambiguous_test_report() {
let out = Command::new(std::env::current_exe().unwrap())
.args(["--exact", "test_nextest::tests::failures_propagate_without_skipping_later_side_gates", "--test-threads=1"])
.output()
.unwrap();
assert!(out.status.success(), "{out:?}");
let stdout = String::from_utf8(out.stdout).unwrap();
assert_eq!(stdout.lines().filter(|line| line.starts_with("test result:")).count(), 1, "{stdout:?}");
assert!(!stdout.contains("test test_nextest::tests::subprocess_fixture"), "{stdout:?}");
}
#[test]
fn failures_propagate_without_skipping_later_side_gates() {
static NEXT: AtomicU64 = AtomicU64::new(0);
for (native_fails, side_fails, expected) in [(true, false, Some("native")), (false, true, Some("first")), (false, false, None)] {
let dir = std::env::temp_dir().join(format!("solidcraft-xtask-nextest-{}-{}", std::process::id(), NEXT.fetch_add(1, Ordering::Relaxed)));
std::fs::create_dir_all(&dir).unwrap();
let native = dir.join("native");
let first = dir.join("first");
let last = dir.join("last");
let result = execute(
("native".into(), fixture(&native, native_fails)),
vec![("first".into(), fixture(&first, side_fails)), ("last".into(), fixture(&last, false))],
);
assert_eq!(result, expected.map_or(Ok(()), |label| Err(format!("failed: {label}"))));
assert!(native.is_file() && first.is_file() && last.is_file());
std::fs::remove_dir_all(dir).unwrap();
}
}
}
xtask/src/ui_containers.rs
+1 -1
View File
@@ -467,7 +467,7 @@ mod tests {
std::fs::write(&source, "fn show() { egui::Area::new(id); }").unwrap();
std::fs::write(&example, "fn main() { egui::Window::new(title); }").unwrap();
std::fs::write(&implementation, "fn factory() { egui::Window::new(title); }").unwrap();
let rejected = run(&root).unwrap_err();
let rejected = run(&root).unwrap_err().replace('\\', "/");
assert!(rejected.contains("future-ui/src/nested/panel.rs"));
assert!(rejected.contains("future-platform/examples/viewer.rs"));
assert!(!rejected.contains("ui_kit/container.rs"), "only the implementation is exempt");