From 592f6a6feee257e2abfc94450ab7d7dc7bd334c3 Mon Sep 17 00:00:00 2001 From: cronos Date: Sat, 24 Oct 2026 11:33:10 +0000 Subject: [PATCH] ci: fix local scheduling and Windows test baselines (#4941) * ci: repair local nextest scheduling and Windows fixtures * test: refresh MSVC robustness references and isolate GNU policy Integrate the Windows reference records from #3361 and the target-environment selection and recording policy from #4159. Keep shared records and comparison tolerances intact, and retain seed 36 as an existing failure tracked by #3583. Co-authored-by: Ray Co-authored-by: chris * test: make CI subprocess and revolve fixtures deterministic --------- Co-authored-by: Ray Co-authored-by: chris --- .config/nextest.toml | 47 +++ crates/engine/src/tests/robustness_goldens.rs | 353 ++++++++++++++---- crates/ui-egui/src/dialogs_all_tests.rs | 18 +- docs/book/src/dev/testing.md | 29 ++ docs/robustness.md | 34 +- tests/robustness/goldens.json | 8 +- xtask/src/main.rs | 3 +- xtask/src/test_nextest.rs | 221 +++++++++++ xtask/src/ui_containers.rs | 2 +- 9 files changed, 625 insertions(+), 90 deletions(-) create mode 100644 xtask/src/test_nextest.rs diff --git a/.config/nextest.toml b/.config/nextest.toml index 0dc8aa8732..619a0b6be2 100644 --- a/.config/nextest.toml +++ b/.config/nextest.toml @@ -69,3 +69,50 @@ filter = """ )) """ priority = 100 + +# Full local validation opts into its own scheduling policy. Keep GitHub CI's +# worker reservation and slow-test priorities above independent of this runner. +[profile.local-ci] +inherits = "ci" + +[[profile.local-ci.overrides]] +filter = "binary(action_matrix) & test(=action_matrix_sample)" +threads-required = "num-cpus" +priority = 100 + +[[profile.local-ci.overrides]] +filter = "(package(solidcraft-engine) & test(=tests::robustness_goldens::seed_81)) | (package(solidcraft-kernel) & test(=tspline::tests::hostile_tsplines_never_panic))" +priority = 99 + +[[profile.local-ci.overrides]] +filter = """ +(package(solidcraft-engine) & ( + test(/^tests::robustness_goldens::seed_(81|45|20|24|95|02|57|40|18|48|32|46|85|61)$/) + | test(/^tests::robustness::/) + | test(=tests::hostile_params_never_panic) + | test(=cmd::clearance::tests::distances_are_exact_on_edges_and_curved_faces) + | test(=cmd::clipboard::tests::a_pasted_coil_joins_the_pasted_body) + | test(=tests::pleat_join::the_pleats_join_at_every_wall_from_2_4_to_2_6_mm) + | test(=tests::flush_join_loft::a_flange_flush_with_the_end_of_a_bent_tapered_tube_joins) + | test(=cmd::sheet::tests::perforated_sheet_takes_a_thousand_round_holes) + | test(=boolean_regression_tests::joined_loft_frames_plan_their_display_at_every_quality) +)) +| (package(solidcraft-engine) & (binary(atomic_commands) | binary(step_export_regressions))) +| (package(solidcraft-kernel) & ( + test(=curveblend::tests::rolling_ball_turns_about_a_rounded_corner_of_its_radius) + | test(=tests::fillet_tabbed_plate_outer_loop_and_partial_selection_after_hole) + | test(=tspline::tests::hostile_tsplines_never_panic) + | test(=fill::tests::regular_rims_of_three_to_eight_sides_fill_with_one_face_and_close_the_cup) + | test(/^issue_788_tests::equal_bores_crossing_/) +)) +""" +priority = 98 + +# Native Windows shader/device creation must not overlap within the local run. +[test-groups.windows-gpu] +max-threads = 1 + +[[profile.local-ci.overrides]] +platform = "cfg(windows)" +filter = "package(solidcraft-gpu) | package(solidcraft) | (package(solidcraft-ui-egui) & test(/(offscreen|gpu)/))" +test-group = "windows-gpu" diff --git a/crates/engine/src/tests/robustness_goldens.rs b/crates/engine/src/tests/robustness_goldens.rs index e0d2776be2..ad717c3693 100644 --- a/crates/engine/src/tests/robustness_goldens.rs +++ b/crates/engine/src/tests/robustness_goldens.rs @@ -54,100 +54,295 @@ fn differs(want: &Value, got: &Value) -> Option { (!out.is_empty()).then(|| out.join("; ")) } -/// Platform records only override the seeds explicitly verified on that platform. -fn expected<'a>(file: &'a Value, seed: u64, platform: &str) -> &'a Value { - let key = seed.to_string(); - file.get("platforms") - .and_then(|platforms| platforms.get(platform)) - .and_then(|record| record.get("seeds")) - .and_then(|seeds| seeds.get(&key)) - .unwrap_or(&file["seeds"][key]) +/// Only Windows needs an environment split; other OS policies remain compatible. +#[derive(Clone, Copy, Debug)] +struct Target<'a> { + os: &'a str, + env: &'a str, } -#[test] -fn platform_records_keep_shared_references_for_every_other_seed_and_platform() { - let file = json!({"seeds":{"1":{"volume":10},"2":{"volume":20}},"platforms":{"windows":{"seeds":{"1":{"volume":11}}}}}); - assert_eq!(expected(&file, 1, "windows")["volume"], 11); - assert_eq!(expected(&file, 1, "linux")["volume"], 10); - assert_eq!(expected(&file, 2, "windows")["volume"], 20); +impl<'a> Target<'a> { + fn current() -> Self { + // cfg! describes the compilation target, including when cross-compiling. + let env = if cfg!(target_env = "msvc") { + "msvc" + } else if cfg!(target_env = "gnu") { + "gnu" + } else { + "other" + }; + Self { os: std::env::consts::OS, env } + } + + fn platform(self) -> Option<&'a str> { + match (self.os, self.env) { + ("windows", "msvc") => Some("windows"), // Existing MSVC-verified records. + ("windows", "gnu") => Some("windows-gnu"), + ("windows", _) => None, + (os, _) => Some(os), + } + } + + fn recording_platform(self, file: &Value) -> Result, String> { + let platform = self.platform(); + if self.os == "windows" { + return platform.map(Some).ok_or_else(|| format!("no recording policy for Windows target_env={}; define one before blessing", self.env)); + } + Ok(platform.filter(|p| file["platforms"][*p]["seeds"].is_object())) + } +} + +/// An override replaces the entire seed record, never individual fields. +/// The second return value identifies the selected reference in diagnostics. +fn expected<'a, 't>(file: &'a Value, seed: u64, target: Target<'t>) -> (&'a Value, &'t str) { + let key = seed.to_string(); + if let Some(platform) = target.platform() + && let Some(record) = file.get("platforms").and_then(|p| p.get(platform)).and_then(|p| p.get("seeds")).and_then(|s| s.get(&key)) + { + return (record, platform); + } + (&file["seeds"][key], "shared") +} + +fn difference(file: &Value, seed: u64, got: &Value, target: Target<'_>) -> Option { + let (want, group) = expected(file, seed, target); + let change = if want.is_null() { Some("new".to_string()) } else { differs(want, got) }; + change.map(|d| format!("seed {seed} [target_os={}, target_env={}, reference={group}]: {d}", target.os, target.env)) +} + +/// In-memory recording seam: regression tests never write the real reference file. +fn record(file: &mut Value, seed: u64, got: &Value, reason: &str, target: Target<'_>) -> Result { + if reason.trim().is_empty() { + return Err("ROBUSTNESS_GOLDENS_BLESS needs the reason for the change".to_string()); + } + let platform = target.recording_platform(file)?; + if difference(file, seed, got, target).is_none() { + return Ok(false); + } + if file.get("about").is_none() { + file["about"] = json!( + "What each robustness seed builds (crates/engine/src/tests/robustness_goldens.rs, docs/robustness.md). Record changes with ROBUSTNESS_GOLDENS_BLESS=\"why\"." + ); + } + if let Some(platform) = platform { + file["platforms"][platform]["seeds"][seed.to_string()] = got.clone(); + } else { + file["seeds"][seed.to_string()] = got.clone(); + } + let mut notes = file["notes"].as_array().cloned().unwrap_or_default(); + // Coalesce only the same reason AND destination, including shared vs platform. + // Windows destinations encode the environment, so GNU and MSVC cannot coalesce. + let destination = platform.map(|p| json!(p)); + match notes.last_mut() { + Some(last) if last["reason"] == json!(reason) && last.get("platform") == destination.as_ref() => { + let mut all: Vec = last["seeds"].as_array().into_iter().flatten().filter_map(Value::as_u64).chain([seed]).collect(); + all.sort_unstable(); + all.dedup(); + last["seeds"] = json!(all); + } + _ => { + let mut note = json!({"reason": reason, "seeds": [seed]}); + if let Some(platform) = platform { + note["platform"] = json!(platform); + } + notes.push(note); + } + } + file["notes"] = Value::Array(notes); + Ok(true) +} + +fn serialize(file: &Value) -> String { + // Seeds in order, one per line, so a change is a small diff. + let seeds: BTreeMap = + file["seeds"].as_object().into_iter().flatten().filter_map(|(k, v)| Some((k.parse::().ok()?, v.clone()))).collect(); + let mut s = format!("{{\n \"about\": {},\n \"seeds\": {{\n", file["about"]); + let n = seeds.len(); + for (i, (k, v)) in seeds.iter().enumerate() { + s += &format!(" \"{k}\": {}{}\n", v, if i + 1 < n { "," } else { "" }); + } + let notes: Vec = file["notes"].as_array().into_iter().flatten().map(|n| format!(" {n}")).collect(); + s += " },\n"; + if let Some(platforms) = file.get("platforms") { + s += &format!(" \"platforms\": {platforms},\n"); + } + s += &format!(" \"notes\": [\n{}\n ]\n}}\n", notes.join(",\n")); + s } fn check(seed: u64) { - let got: BTreeMap = BTreeMap::from([(seed, fuzz::golden(seed, MAX_STEPS))]); + let got = fuzz::golden(seed, MAX_STEPS); let _lock = FILE.lock().unwrap_or_else(|e| e.into_inner()); let mut file = load(); - let changed: Vec<(u64, String)> = got - .iter() - .filter_map(|(seed, g)| { - let w = expected(&file, *seed, std::env::consts::OS); - if w.is_null() { Some((*seed, "new".to_string())) } else { differs(w, g).map(|d| (*seed, d)) } - }) - .collect(); + let target = Target::current(); if let Ok(reason) = std::env::var("ROBUSTNESS_GOLDENS_BLESS") { - assert!(!reason.trim().is_empty(), "ROBUSTNESS_GOLDENS_BLESS needs the reason for the change"); - if changed.is_empty() { - return; + if record(&mut file, seed, &got, &reason, target).unwrap_or_else(|e| panic!("{e}")) { + std::fs::write(path(), serialize(&file)).unwrap_or_else(|e| panic!("{}: {e}", path().display())); } - if file.get("about").is_none() { - file["about"] = json!( - "What each robustness seed builds (crates/engine/src/tests/robustness_goldens.rs, docs/robustness.md). Record changes with ROBUSTNESS_GOLDENS_BLESS=\"why\"." - ); - } - let platform = std::env::consts::OS; - let platform_record = file["platforms"][platform]["seeds"].is_object(); - for (seed, g) in &got { - if platform_record { - if changed.iter().any(|(changed, _)| changed == seed) { - file["platforms"][platform]["seeds"][seed.to_string()] = g.clone(); - } - } else { - file["seeds"][seed.to_string()] = g.clone(); - } - } - let mut notes = file["notes"].as_array().cloned().unwrap_or_default(); - let seeds: Vec = changed.iter().map(|(s, _)| *s).collect(); - // The parts of one record (the tests run side by side) share one note. - match notes.last_mut() { - Some(last) if last["reason"] == json!(reason) && (!platform_record || last["platform"] == json!(platform)) => { - let mut all: Vec = last["seeds"].as_array().into_iter().flatten().filter_map(Value::as_u64).chain(seeds).collect(); - all.sort_unstable(); - all.dedup(); - last["seeds"] = json!(all); - } - _ => { - let mut note = json!({"reason": reason, "seeds": seeds}); - if platform_record { - note["platform"] = json!(platform); - } - notes.push(note); - } - } - file["notes"] = Value::Array(notes); - // Seeds in order, one per line, so a change is a small diff. - let seeds: BTreeMap = - file["seeds"].as_object().into_iter().flatten().filter_map(|(k, v)| Some((k.parse::().ok()?, v.clone()))).collect(); - let mut s = format!("{{\n \"about\": {},\n \"seeds\": {{\n", file["about"]); - let n = seeds.len(); - for (i, (k, v)) in seeds.iter().enumerate() { - s += &format!(" \"{k}\": {}{}\n", v, if i + 1 < n { "," } else { "" }); - } - let notes: Vec = file["notes"].as_array().into_iter().flatten().map(|n| format!(" {n}")).collect(); - s += " },\n"; - if let Some(platforms) = file.get("platforms") { - s += &format!(" \"platforms\": {platforms},\n"); - } - s += &format!(" \"notes\": [\n{}\n ]\n}}\n", notes.join(",\n")); - std::fs::write(path(), s).unwrap_or_else(|e| panic!("{}: {e}", path().display())); return; } + let change = difference(&file, seed, &got, target); assert!( - changed.is_empty(), - "{} seed(s) build differently from tests/robustness/goldens.json (if that is intended, record it with ROBUSTNESS_GOLDENS_BLESS=\"why\" and say why in the pull request):\n{}", - changed.len(), - changed.iter().map(|(s, d)| format!("seed {s}: {d}")).collect::>().join("\n") + change.is_none(), + "build differs from tests/robustness/goldens.json (if that is intended, record it with ROBUSTNESS_GOLDENS_BLESS=\"why\" and say why in the pull request):\n{}", + change.unwrap_or_default() ); } +const MSVC: Target<'_> = Target { os: "windows", env: "msvc" }; +const GNU: Target<'_> = Target { os: "windows", env: "gnu" }; +const LINUX: Target<'_> = Target { os: "linux", env: "gnu" }; + +fn policy_fixture() -> Value { + json!({ + "seeds": {"1": {"volume": 10, "faces": 2, "outcome": "ok"}, "2": {"volume": 20}}, + "platforms": { + "windows": {"seeds": {"1": {"volume": 11, "outcome": "failed"}}, "verification": "MSVC"}, + "macos": {"seeds": {"1": {"volume": 12}}} + }, + "notes": [{"platform": "windows", "reason": "same reason", "seeds": [1]}] + }) +} + +#[test] +fn policy_selects_msvc_and_gnu_overrides_with_shared_fallback() { + let mut file = policy_fixture(); + assert_eq!(expected(&file, 1, MSVC), (&file["platforms"]["windows"]["seeds"]["1"], "windows")); + assert_eq!(expected(&file, 1, GNU), (&file["seeds"]["1"], "shared")); + file["platforms"]["windows-gnu"]["seeds"]["1"] = json!({"volume": 13}); + assert_eq!(expected(&file, 1, GNU), (&file["platforms"]["windows-gnu"]["seeds"]["1"], "windows-gnu")); + for target in [MSVC, GNU] { + assert_eq!(expected(&file, 2, target), (&file["seeds"]["2"], "shared")); + } + // A platform record is whole, including failure outcomes and missing fields. + assert!(expected(&file, 1, MSVC).0.get("faces").is_none()); + assert_eq!(expected(&file, 1, MSVC).0["outcome"], "failed"); + assert!(differs(expected(&file, 1, MSVC).0, &file["seeds"]["1"]).is_some()); + assert!(expected(&file, 1, GNU).0.get("faces").is_none()); + assert!(expected(&file, 99, GNU).0.is_null()); +} + +#[test] +fn policy_preserves_non_windows_and_refuses_unknown_windows_recording() { + let mut file = policy_fixture(); + let macos = Target { os: "macos", env: "" }; + assert_eq!(expected(&file, 1, LINUX), (&file["seeds"]["1"], "shared")); + assert_eq!(expected(&file, 1, macos), (&file["platforms"]["macos"]["seeds"]["1"], "macos")); + assert_eq!(LINUX.recording_platform(&file), Ok(None)); + assert_eq!(macos.recording_platform(&file), Ok(Some("macos"))); + for env in ["", "other", "new-env"] { + let unknown = Target { os: "windows", env }; + let before = file.clone(); + assert_eq!(expected(&file, 1, unknown), (&file["seeds"]["1"], "shared")); + assert!(record(&mut file, 1, &json!({"volume": 30}), "why", unknown).is_err()); + assert_eq!(file, before); + } + assert_eq!(expected(&file, 1, Target { os: "future-os", env: "gnu" }).1, "shared"); +} + +#[test] +fn policy_records_only_changed_gnu_seeds_and_preserves_other_destinations_and_notes() { + let mut file = policy_fixture(); + let before = file.clone(); + let got = json!({"volume": 30, "outcome": "failed", "error": "known failure"}); + assert_eq!(record(&mut file, 2, &got, "same reason", GNU), Ok(true)); + assert_eq!(file["seeds"], before["seeds"]); + assert_eq!(file["platforms"]["windows"], before["platforms"]["windows"]); + assert_eq!(file["platforms"]["macos"], before["platforms"]["macos"]); + assert_eq!(file["notes"][0], before["notes"][0]); + assert_eq!(file["notes"][1], json!({"platform": "windows-gnu", "reason": "same reason", "seeds": [2]})); + assert_eq!(expected(&file, 2, GNU), (&got, "windows-gnu")); + assert!(file["platforms"]["windows-gnu"]["seeds"].get("1").is_none()); + assert_eq!(record(&mut file, 1, &got, "same reason", GNU), Ok(true)); + assert_eq!(file["notes"][1]["seeds"], json!([1, 2])); + let unchanged = file.clone(); + assert_eq!(record(&mut file, 1, &got, "same reason", GNU), Ok(false)); + assert_eq!(file, unchanged); + assert_eq!(record(&mut file, 1, &json!({"volume": 31}), "same reason", GNU), Ok(true)); + assert_eq!(file["notes"][1]["seeds"], json!([1, 2])); + assert!(expected(&file, 1, GNU).0.get("error").is_none()); + assert_eq!(serde_json::from_str::(&serialize(&file)).ok().as_ref(), Some(&file)); +} + +#[test] +fn policy_isolates_msvc_recording_even_without_a_namespace() { + for target in [MSVC, GNU] { + let mut file = json!({"seeds": {"1": {"volume": 10}}, "notes": []}); + let before = file.clone(); + assert_eq!(record(&mut file, 1, &json!({"volume": 10}), "why", target), Ok(false)); + assert_eq!(file, before); // No spurious namespace or note for a matching fallback. + assert_eq!(record(&mut file, 1, &json!({"volume": 11}), "why", target), Ok(true)); + assert_eq!(file["seeds"], before["seeds"]); + assert_eq!(expected(&file, 1, target).0["volume"], 11); + assert_eq!(file["notes"][0]["platform"], json!(target.platform())); + } +} + +#[test] +fn policy_notes_never_coalesce_across_destinations_in_either_direction() { + let mut file = policy_fixture(); + for target in [GNU, MSVC, LINUX, GNU, LINUX, MSVC] { + let previous_notes = file["notes"].as_array().cloned().unwrap_or_default(); + let got = json!({"volume": 100 + previous_notes.len()}); + assert_eq!(record(&mut file, 1, &got, "same reason", target), Ok(true)); + let notes = file["notes"].as_array().cloned().unwrap_or_default(); + assert_eq!(notes.len(), previous_notes.len() + 1); + assert_eq!(¬es[..previous_notes.len()], previous_notes); + } + let before = file["notes"].clone(); + assert_eq!(record(&mut file, 2, &json!({"volume": 50}), "different reason", MSVC), Ok(true)); + assert_eq!(file["notes"].as_array().map(Vec::len), before.as_array().map(|n| n.len() + 1)); + let before = file.clone(); + assert!(record(&mut file, 1, &json!({"volume": 99}), " ", GNU).is_err()); + assert_eq!(file, before); +} + +#[test] +fn policy_comparator_stays_strict_and_diagnostics_name_the_selected_reference() { + let want = json!({"volume": 100.0, "area": 10.0, "faces": 2, "outcome": "ok", "steps": ["box"], "error": null, "rejected": 0, "unsupported": 0}); + let mut got = want.clone(); + got["volume"] = json!(100.00005); + got["area"] = json!(10.000005); + assert_eq!(differs(&want, &got), None); + for (field, value) in [ + ("volume", json!(100.001)), + ("area", json!(10.001)), + ("faces", json!(3)), + ("outcome", json!("failed")), + ("steps", json!(["hole"])), + ("error", json!("refused")), + ("rejected", json!(1)), + ("unsupported", json!(1)), + ] { + let mut got = want.clone(); + got[field] = value; + assert!(differs(&want, &got).is_some(), "{field}"); + } + let file = policy_fixture(); + let diagnostic = difference(&file, 1, &json!({"volume": 100}), GNU).unwrap_or_default(); + assert!(diagnostic.contains("target_os=windows, target_env=gnu, reference=shared"), "{diagnostic}"); + let diagnostic = difference(&file, 1, &json!({"volume": 100}), MSVC).unwrap_or_default(); + assert!(diagnostic.contains("target_env=msvc, reference=windows"), "{diagnostic}"); +} + +#[test] +fn policy_non_windows_recording_keeps_existing_os_destination_and_shared_notes() { + let mut file = policy_fixture(); + let before = file.clone(); + let got = json!({"volume": 40}); + let macos = Target { os: "macos", env: "" }; + assert_eq!(record(&mut file, 1, &got, "same reason", macos), Ok(true)); + assert_eq!(file["seeds"], before["seeds"]); + assert_eq!(file["platforms"]["windows"], before["platforms"]["windows"]); + assert_eq!(expected(&file, 1, macos), (&got, "macos")); + let platforms = file["platforms"].clone(); + assert_eq!(record(&mut file, 1, &got, "same reason", LINUX), Ok(true)); + assert_eq!(file["platforms"], platforms); + assert_eq!(file["notes"][2], json!({"reason": "same reason", "seeds": [1]})); + assert_eq!(record(&mut file, 2, &got, "same reason", LINUX), Ok(true)); + assert_eq!(file["notes"][2]["seeds"], json!([1, 2])); +} + /// One test per seed, and the list of them. macro_rules! seeds { ($($name:ident = $seed:literal),* $(,)?) => { diff --git a/crates/ui-egui/src/dialogs_all_tests.rs b/crates/ui-egui/src/dialogs_all_tests.rs index 272dc3e68f..8e2b6ab866 100644 --- a/crates/ui-egui/src/dialogs_all_tests.rs +++ b/crates/ui-egui/src/dialogs_all_tests.rs @@ -60,7 +60,9 @@ fn representative(h: &Harness, command: &str) -> Dialog { } else if input.accept & CURVES != 0 || command == "sheet.fold" && index == 0 { vec![Sel::SketchCurve { id: path.clone() }] } else if input.accept & AXES != 0 { - vec![Sel::Axis { name: "Z".into() }] + // The Revolve fixture's profile is in XY; its preview needs an axis in that plane. + let axis = if command == "solid.revolve" { "Y" } else { "Z" }; + vec![Sel::Axis { name: axis.into() }] } else if input.accept & (FACES | PLANAR_FACES) != 0 { vec![face_sel(s, point).unwrap()] } else if input.accept & EDGES != 0 { @@ -293,6 +295,20 @@ fn registry_dialog_inventory_has_no_unclassified_feature_forms() { } } +#[test] +fn revolve_layout_fixture_builds_a_valid_solid() { + let mut h = fixture(); + let d = representative(&h, "solid.revolve"); + let before = h.app.session.model.results.len(); + let commands = crate::dialogs::preview_commands(&h.app, &d).unwrap(); + assert!(!commands.is_empty()); + for (command, params) in commands { + h.app.run(&command, params).unwrap(); + } + assert!(h.app.session.model.results.len() > before); + assert!(h.app.session.model.results.iter().all(|result| result.error.is_none())); +} + #[test] fn new_cards_settle_before_the_first_click_in_both_layouts() { for mode in [crate::shell::LayoutMode::Floating, crate::shell::LayoutMode::Docked] { diff --git a/docs/book/src/dev/testing.md b/docs/book/src/dev/testing.md index 70ff566fbd..0f6a343f51 100644 --- a/docs/book/src/dev/testing.md +++ b/docs/book/src/dev/testing.md @@ -68,6 +68,35 @@ with doc tests alongside. Arguments after `--` go to each test binary, for examp `cargo xtask test -- shell`. To run the test binaries sequentially instead, use `cargo test --workspace --profile ci`. +## Test-level scheduling for full local CI + +Install [cargo-nextest](https://nexte.st/docs/installation/pre-built-binaries/), then set +`SOLIDCRAFT_TEST_RUNNER=nextest` when running `cargo xtask ci`. For example, on PowerShell: + +```powershell +$env:SOLIDCRAFT_TEST_RUNNER = 'nextest' +cargo xtask ci +``` + +This optional backend builds the workspace tests once, then schedules individual tests using +the `local-ci` nextest profile, which inherits `ci`. The action matrix runs first, then the longest geometry +cases start before the other costly tests, so the ordinary queue does not stall behind an +all-slot reservation. It **ignores the default filter**, +so the heavy tests below still run, does not stop after a test failure, and does not retry failures. +Tests marked `#[ignore]` remain ignored, as with the ordinary full local CI. +The default concurrency is at most four tests; `NEXTEST_TEST_THREADS` overrides it. The action +matrix's child workers default to the smaller of that budget and the host CPU count; +`ACTION_MATRIX_JOBS` explicitly overrides this worker count. The local profile reserves CPU +slots for the matrix and keeps its scheduling changes separate from GitHub CI. On Windows, +the `solidcraft-gpu` and app packages, plus UI offscreen/GPU tests, use a serial group. The Wasm check and doc tests still run alongside, +and the web-offline gate still follows a successful native/Wasm/doc run. + +The backend keeps Cargo and binary metadata under `target/xtask-nextest/` and reuses it for the +native run, freeing Cargo to run the side gates. Nextest prints each test's result and duration +as it finishes. `SOLIDCRAFT_TEST_RUNNER=libtest` (or leaving it unset) retains the original runner; +`cargo xtask test` and its Cargo/libtest argument forwarding are unchanged. Unknown runner names +fail explicitly. + ## CI sharding and heavy tests On GitHub, one job builds the tests into a [cargo-nextest](https://nexte.st/) archive and four diff --git a/docs/robustness.md b/docs/robustness.md index ebb88d8882..2cec478960 100644 --- a/docs/robustness.md +++ b/docs/robustness.md @@ -48,12 +48,30 @@ kernel, engine, doc, sketch and geom crates) run every seed but the slowest few and the nightly run them all. Each seed is its own test (`seed_00` … `seed_95`), so the CI shards and their cores share them out. -Explicit `platforms..seeds` records override only the listed seeds on that -operating system; all other seeds keep the shared records and strict tolerances. -The first four Windows records were independently reproduced on unmodified main -`1e984c8e`; seed 94 was independently reproduced on unmodified main `c5992386` -after the extrusion rebase. Both match the extrusion branch. Recording on a platform -with such entries updates its changed seeds and preserves shared references. +Reference selection uses the compilation target's operating system and environment. +On Windows, `platforms.windows.seeds` contains the MSVC-verified records; +`platforms.windows-gnu.seeds`, when present, contains GNU records. An override +replaces the whole seed record. Seeds missing from the selected group use the shared +record, never another Windows group's record. Non-Windows targets keep their existing +OS-group selection and shared fallback. Unknown Windows environments compare with the +shared records and refuse recording until an explicit policy is defined. +All 96 seeds and the strict measurement and topology tolerances remain unchanged. + +The Windows records for seeds 1, 23 and 34 were independently reproduced on unmodified +main `1e984c8e`; seed 94 was independently reproduced on unmodified main `c5992386` +after the extrusion rebase. Both match the extrusion branch. Seed 13's was recorded on +unmodified main `65e6d9da`; seed 23's was re-recorded after #1653. Seed 36's, recorded +on main `79a843c9`, holds a failure: on Windows (and macOS) its last step leaves a body that fails the +validity check, where Linux builds it ok (#3583). The record only makes a further change +show; it does not accept the failure. Recording on a platform with such entries updates its +changed seeds and preserves shared references. A platform record replaces the shared +one whole, so a change that moves a seed with a platform record needs that seed checked +on that platform too: seed 67's Windows record went stale this way after #2064 and +#2247 and was removed once Windows built the shared record again. +The Windows records were made with the MSVC target (`x86_64-pc-windows-msvc`) and all +six were checked again on main `e9490093`. GNU builds (`x86_64-pc-windows-gnu`) +use the shared records unless an explicit GNU override exists, so MSVC-only outcomes +do not leak into GNU validation (#4104). To record an intended change, give the reason: @@ -62,6 +80,10 @@ ROBUSTNESS_GOLDENS_BLESS="why it changed" cargo test --profile ci -p solidcraft- ``` The changed seeds and the reason are added to the file's `notes`; say why in the pull request too. +Recording on Windows always writes to the current environment's own group, even when +that group does not exist yet. It preserves shared records and the other Windows +environment's records. Non-Windows recording keeps its existing OS-group policy. +Notes coalesce only when both the reason and destination match. The generator feeds the current geometry's bounds and face/edge picks back into later commands. The same RNG seed can therefore generate different command coordinates or picks diff --git a/tests/robustness/goldens.json b/tests/robustness/goldens.json index 378044afe9..65f687a081 100644 --- a/tests/robustness/goldens.json +++ b/tests/robustness/goldens.json @@ -98,7 +98,7 @@ "94": {"area":3480.70333,"bodies":1,"edges":63,"error":null,"faces":27,"outcome":"ok","rejected":1,"shells":2,"steps":["revolved section","cut cylinder","fillet","hole","rectangular pattern","cut box","cut box"],"unsupported":0,"volume":5456.446856}, "95": {"area":2375.340299,"bodies":1,"edges":33,"error":null,"faces":12,"outcome":"ok","rejected":2,"shells":1,"steps":["revolved section","hole","cut cylinder","cut sphere","circular pattern","cut sphere"],"unsupported":1,"volume":6704.220611} }, - "platforms": {"windows":{"seeds":{"1":{"area":2642.450837,"bodies":1,"edges":32,"error":null,"faces":13,"outcome":"ok","rejected":0,"shells":1,"steps":["revolved section","cut cylinder","join cylinder"],"unsupported":0,"volume":6991.569339},"23":{"area":1795.696636,"bodies":1,"edges":15,"error":null,"faces":7,"outcome":"ok","rejected":0,"shells":1,"steps":["revolved section","cut sphere"],"unsupported":0,"volume":3198.694618},"34":{"area":1086.303608,"bodies":1,"edges":30,"error":null,"faces":14,"outcome":"ok","rejected":0,"shells":1,"steps":["revolved section","cut sphere","hole"],"unsupported":0,"volume":2134.3653},"67":{"area":7943.46222,"bodies":2,"edges":86,"error":null,"faces":36,"outcome":"ok","rejected":0,"shells":2,"steps":["extruded profile","cut cylinder","hole","join cylinder","join box","hole","chamfer","rectangular pattern"],"unsupported":2,"volume":27607.962186},"94":{"area":3470.837042,"bodies":1,"edges":63,"error":null,"faces":27,"outcome":"ok","rejected":1,"shells":2,"steps":["revolved section","cut cylinder","fillet","hole","rectangular pattern","cut box","cut box"],"unsupported":0,"volume":5456.438772}}}}, + "platforms": {"windows":{"seeds":{"1":{"area":2642.450837,"bodies":1,"edges":32,"error":null,"faces":13,"outcome":"ok","rejected":0,"shells":1,"steps":["revolved section","cut cylinder","join cylinder"],"unsupported":0,"volume":6991.569339},"13":{"area":2101.521015,"bodies":1,"edges":103,"error":null,"faces":41,"outcome":"ok","rejected":1,"shells":1,"steps":["revolved section","cut cylinder","join box","rectangular pattern","cut cylinder","cut box","fillet","hole"],"unsupported":0,"volume":2396.939546},"23":{"area":1795.693105,"bodies":1,"edges":15,"error":null,"faces":7,"outcome":"ok","rejected":0,"shells":1,"steps":["revolved section","cut sphere"],"unsupported":0,"volume":3198.694089},"34":{"area":1086.303608,"bodies":1,"edges":30,"error":null,"faces":14,"outcome":"ok","rejected":0,"shells":1,"steps":["revolved section","cut sphere","hole"],"unsupported":0,"volume":2134.3653},"36":{"area":9964.083285,"bodies":1,"edges":45,"error":"lumps: shell boundaries intersect or touch; cavity ownership","faces":21,"outcome":"failed","rejected":0,"shells":2,"steps":["revolved section","cut cylinder","join box","fillet","join cylinder"],"unsupported":0,"volume":54852.503099},"94":{"area":3470.837042,"bodies":1,"edges":63,"error":null,"faces":27,"outcome":"ok","rejected":1,"shells":2,"steps":["revolved section","cut cylinder","fillet","hole","rectangular pattern","cut box","cut box"],"unsupported":0,"volume":5456.438772}}}}, "notes": [ {"reason":"First record: what each seed builds on main at 1a0100cd","seeds":[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95]}, {"reason":"seed 0 meshes differently since display tessellation plans (#805), which landed alongside these goldens (#826)","seeds":[0]}, @@ -148,6 +148,10 @@ {"reason":"A cut no longer comes out empty where a shell of one body meets nothing of the other (a sealed void, a body clear of the tool): truck placed such faces by their surfaces alone, which a turned body's reversed faces and a void's inward shell got wrong. Seeds 0, 45 and 77 each had a cut refused as empty that now takes what it should, and the generator builds different steps after it: seed 0 a sphere clear of the body (removes nothing), seed 45 a hole on the axis of a shelled turned body (widens its bore from r 2.5 to r 4 over 17.5 mm: 536.0 mm3), seed 77 a hole inside a turned body's bore (removes nothing; a later pattern copy drills r 1.5 through 27.5 mm: 194.4 mm3). Every step passes the validity and volume-direction checks.","seeds":[0,45,77]}, {"reason":"Trimmed faces of display meshes stay on their surfaces (#1653): seeds 1, 23, 55, 64 and 86 end with bodies whose trimmed faces were meshed off their surfaces; measured on the corrected meshes their area and volume move slightly (seed 86 most: area +9.6 mm2, volume -41.4 mm3, 0.2 %). Steps, outcome and counts are unchanged; recorded from Linux CI on main at the time of the rebase. The slow seeds 14, 20 and 40 (run only by the merge queue) move too: 14 area +0.92 mm2 / volume +2.64 mm3, 20 area +2.60 mm2 / volume +14.8 mm3, 40 area +0.61 mm2 / volume -2.25 mm3, from merge-group run 39450833424.","seeds":[1,14,20,23,40,55,64,86]}, {"reason":"measure (#2069): a body whose meshes leave a crack is summed about its box's centre instead of the origin, so the crack is closed by a cone to a point inside the body. Seeds 14, 20, 40, 55, 61 and 86 end with such a body; their volumes move from main's by -0.52 %, +0.11 %, -0.017 %, -0.001 %, -0.0015 % and +0.026 %. Steps, outcomes, counts and areas (to 3e-7 relative) are unchanged. Recorded on Linux.","seeds":[14,20,40,55,61,86]}, - {"reason":"Chamfer cutters that reach into a nearby wall are not used (#4539): a chamfer in seed 9 was refused, as its result changed 33 mm3 of the part, more than the chamfer can (20). With its cutter refused it builds locally, so the design takes 8 steps without the final cut box. Area 2427.18 -> 2403.70 mm2, volume 6417.16 -> 6524.64 mm3, unsupported steps 2 -> 0; counts unchanged. Recorded on Linux.","seeds":[9]} + {"reason":"Chamfer cutters that reach into a nearby wall are not used (#4539): a chamfer in seed 9 was refused, as its result changed 33 mm3 of the part, more than the chamfer can (20). With its cutter refused it builds locally, so the design takes 8 steps without the final cut box. Area 2427.18 -> 2403.70 mm2, volume 6417.16 -> 6524.64 mm3, unsupported steps 2 -> 0; counts unchanged. Recorded on Linux.","seeds":[9]}, + {"platform":"windows","reason":"Seed 67's Windows record (from main 1e984c8e) predates #2064 and #2247, which changed the shared record. On main 65e6d9da Windows builds the shared record exactly (one body, 35 faces/84 edges, 27606.450855 mm3, 7940.064 mm2): its generated coordinates differ from Linux's only in the last digits. The stale Windows record is removed, so Windows checks seed 67 against the shared record again.","seeds":[67]}, + {"platform":"windows","reason":"Seed 13 on Windows (main 65e6d9da) builds what Linux built before #2064. Its first four steps match Linux to nine decimals. The second cylinder cut takes its base from the body bounds: y = -1.78e-14 on Windows, -1.95e-14 on Linux (math-library rounding, #924). At the same volume (2694.4476 mm3 on both) Windows leaves split the two coplanar faces that #2064 merges on Linux (37 faces/88 edges, not 35/84). The fillet picks its edge by index, so it rounds a different edge (2675.18 mm3, not 2688.99), and the hole after it removes the same 278.2 mm3. Every step passes the validity and volume-direction checks.","seeds":[13]}, + {"platform":"windows","reason":"Seed 23's Windows record predates #1653 (trimmed faces of display meshes stay on their surfaces), which moved its shared record's area; on Windows the final body's area moves the same way (1795.696636 to 1795.693105 mm2). Steps, outcome and counts are unchanged.","seeds":[23]}, + {"platform":"windows","reason":"Seed 36 on Windows (main 79a843c9) ends differently from Linux: the body after its last step (join cylinder) fails the validity check ('lumps: shell boundaries intersect or touch; cavity ownership'), so the run ends failed with 21 faces/45 edges, 54852.503099 mm3. macOS fails the same way; only Linux x86_64 builds it ok. Recorded so Windows notices any further change; the failure itself is tracked in #3583.","seeds":[36]} ] } diff --git a/xtask/src/main.rs b/xtask/src/main.rs index f193d215b0..7d648abd25 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -13,6 +13,7 @@ mod layers; mod licences; mod lists; mod test; +mod test_nextest; mod ui_containers; mod web_offline; mod web_offline_test; @@ -382,7 +383,7 @@ fn cmd_ci() -> Result<(), String> { c.args(["clippy", "--workspace", "--all-targets", "--profile", "ci", "--", "-D", "warnings"]); run(c, "cargo clippy -D warnings")?; let (wasm, n) = wasm_check()?; - test::run(&[], &[], vec![(format!("wasm32-unknown-unknown check ({n} crates)"), wasm), test::doc_tests(&[], &[])])?; + test_nextest::run_selected(vec![(format!("wasm32-unknown-unknown check ({n} crates)"), wasm), test::doc_tests(&[], &[])])?; web_offline_test::run(&root(), None, false)?; eprintln!("ci: all gates passed"); Ok(()) diff --git a/xtask/src/test_nextest.rs b/xtask/src/test_nextest.rs new file mode 100644 index 0000000000..c8858f4781 --- /dev/null +++ b/xtask/src/test_nextest.rs @@ -0,0 +1,221 @@ +//! Optional test-level scheduling for full local CI. Reuse the built binaries so the Wasm and +//! doc-test jobs can use Cargo alongside the native run, without rebuilding native tests. + +use std::ffi::OsString; +use std::path::PathBuf; +use std::process::{Command, Stdio}; +use std::sync::mpsc; +use std::time::Instant; + +use crate::cargo; + +/// Select the full-CI runner without changing `xtask test`'s argument forwarding. +pub fn run_selected(side: Vec<(String, Command)>) -> Result<(), String> { + match std::env::var("SOLIDCRAFT_TEST_RUNNER") { + Err(std::env::VarError::NotPresent) => crate::test::run(&[], &[], side), + Ok(value) if value == "libtest" => crate::test::run(&[], &[], side), + Ok(value) if value == "nextest" => run(side), + _ => Err("SOLIDCRAFT_TEST_RUNNER must be libtest or nextest".into()), + } +} + +pub struct Prepared { + pub cargo_metadata: PathBuf, + pub binaries_metadata: PathBuf, +} + +impl Prepared { + pub fn command(&self) -> Command { + self.command_with_options( + std::env::var_os("NEXTEST_TEST_THREADS"), + std::env::var_os("ACTION_MATRIX_JOBS"), + std::thread::available_parallelism().map_or(1, |n| n.get()), + ) + } + + fn command_with_options(&self, threads: Option, matrix_jobs: Option, cpus: usize) -> Command { + let mut c = cargo(); + c.args(["nextest", "run", "--cargo-metadata"]).arg(&self.cargo_metadata).arg("--binaries-metadata").arg(&self.binaries_metadata).args([ + "--profile", + "local-ci", + "--ignore-default-filter", + "--no-fail-fast", + "--retries", + "0", + "--no-tests", + "fail", + "--test-threads", + ]); + // One geometry test can also use Rayon. Bound the default; callers can explicitly + // choose another nextest concurrency without changing libtest's per-binary setting. + let threads = threads.unwrap_or_else(|| cpus.clamp(1, 4).to_string().into()); + // The matrix has its own child-process pool. A nextest slot reservation alone + // does not bound it: without this it still spawns one worker per host CPU. + // Leave invalid thread values to nextest's own diagnostic before tests start. + let slots = match threads.to_str() { + Some("num-cpus") => cpus, + Some(value) => value.parse::().ok().map(|n| if n < 0 { cpus.saturating_sub(n.unsigned_abs()) } else { n as usize }).unwrap_or(1), + None => 1, + } + .clamp(1, cpus.max(1)); + c.arg(threads).env("ACTION_MATRIX_JOBS", matrix_jobs.unwrap_or_else(|| slots.to_string().into())); + c + } +} + +pub fn run(side: Vec<(String, Command)>) -> Result<(), String> { + let prepared = prepare()?; + execute(("nextest full workspace".into(), prepared.command()), side) +} + +pub fn prepare() -> Result { + let mut version = cargo(); + version.args(["nextest", "--version"]); + capture(version, "cargo nextest --version") + .map_err(|e| format!("{e}; install cargo-nextest first (https://nexte.st/docs/installation/pre-built-binaries/)"))?; + let mut metadata = cargo(); + metadata.args(["metadata", "--format-version", "1", "--locked"]); + let cargo_json = capture(metadata, "cargo metadata")?; + let parsed: serde_json::Value = serde_json::from_slice(&cargo_json).map_err(|e| format!("Cargo metadata: {e}"))?; + let target = parsed.get("target_directory").and_then(|v| v.as_str()).ok_or("Cargo metadata has no target_directory")?; + let dir = PathBuf::from(target).join("xtask-nextest").join(std::process::id().to_string()); + std::fs::create_dir_all(&dir).map_err(|e| format!("create {}: {e}", dir.display()))?; + let cargo_metadata = dir.join("cargo-metadata.json"); + std::fs::write(&cargo_metadata, cargo_json).map_err(|e| format!("write {}: {e}", cargo_metadata.display()))?; + let mut build = cargo(); + build.args(["nextest", "list", "--workspace", "--cargo-profile", "ci", "--locked", "--list-type", "binaries-only", "--message-format", "json"]); + let binaries_json = capture(build, "cargo nextest list (build all workspace tests)")?; + let binaries_metadata = dir.join("binaries-metadata.json"); + // Retain both metadata files under target for diagnostics and reproducible listing. + std::fs::write(&binaries_metadata, binaries_json).map_err(|e| format!("write {}: {e}", binaries_metadata.display()))?; + Ok(Prepared { cargo_metadata, binaries_metadata }) +} + +fn capture(mut c: Command, label: &str) -> Result, String> { + eprintln!("$ {label}"); + let out = c.stdin(Stdio::null()).stderr(Stdio::inherit()).output().map_err(|e| format!("{label}: {e}"))?; + if out.status.success() { Ok(out.stdout) } else { Err(format!("{label}: {}", out.status)) } +} + +/// Keep every gate running after failures, and stream native test names/times as they finish. +pub fn execute(native: (String, Command), side: Vec<(String, Command)>) -> Result<(), String> { + let (tx, rx) = mpsc::channel(); + let mut failures = Vec::new(); + std::thread::scope(|scope| { + let tx_native = tx.clone(); + scope.spawn(move || { + let _ = tx_native.send(run_job(native)); + }); + scope.spawn(move || { + for job in side { + let _ = tx.send(run_job(job)); + } + }); + for (label, result, seconds) in rx { + match result { + Ok(()) => eprintln!("ok {label:<44} {seconds:6.1}s"), + Err(e) => { + eprintln!("FAIL {label:<44} {seconds:6.1}s {e}"); + failures.push(label); + } + } + } + }); + if failures.is_empty() { Ok(()) } else { Err(format!("failed: {}", failures.join(", "))) } +} + +fn run_job((label, mut c): (String, Command)) -> (String, Result<(), String>, f64) { + let started = Instant::now(); + let result = c.stdin(Stdio::null()).status().map_err(|e| e.to_string()).and_then(|s| if s.success() { Ok(()) } else { Err(s.to_string()) }); + (label, result, started.elapsed().as_secs_f64()) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::atomic::{AtomicU64, Ordering}; + + #[test] + fn full_ci_does_not_drop_heavy_tests_stop_on_failure_or_retry_failures() { + let p = Prepared { cargo_metadata: "cargo.json".into(), binaries_metadata: "binaries.json".into() }; + let c = p.command_with_options(Some("3".into()), None, 16); + let args: Vec<_> = c.get_args().map(|s| s.to_string_lossy().into_owned()).collect(); + assert!(args.windows(2).any(|s| s == ["--profile", "local-ci"])); + assert!(args.iter().any(|s| s == "--ignore-default-filter")); + assert!(args.iter().any(|s| s == "--no-fail-fast")); + assert!(args.windows(2).any(|s| s == ["--retries", "0"])); + assert!(args.windows(2).any(|s| s == ["--no-tests", "fail"])); + assert!(args.windows(2).any(|s| s == ["--test-threads", "3"])); + assert!(!args.iter().any(|s| s == "--workspace" || s == "--cargo-profile")); + } + + #[test] + fn matrix_workers_follow_the_effective_nextest_budget() { + let p = Prepared { cargo_metadata: "cargo.json".into(), binaries_metadata: "binaries.json".into() }; + for (cpus, threads, expected) in [ + (16, None, "4"), + (2, None, "2"), + (1, None, "1"), + (16, Some("1"), "1"), + (16, Some("6"), "6"), + (16, Some("num-cpus"), "16"), + (16, Some("-2"), "14"), + (2, Some("-8"), "1"), + (2, Some("8"), "2"), + ] { + let c = p.command_with_options(threads.map(Into::into), None, cpus); + assert_eq!(c.get_envs().find(|(k, _)| *k == "ACTION_MATRIX_JOBS").and_then(|(_, v)| v), Some(std::ffi::OsStr::new(expected))); + } + let c = p.command_with_options(Some("3".into()), Some("2".into()), 16); + assert_eq!(c.get_envs().find(|(k, _)| *k == "ACTION_MATRIX_JOBS").and_then(|(_, v)| v), Some(std::ffi::OsStr::new("2"))); + } + + #[test] + fn subprocess_fixture() { + let Some(path) = std::env::var_os("SOLIDCRAFT_XTASK_TEST_MARKER") else { return }; + std::fs::write(path, b"ran").unwrap(); + assert_ne!(std::env::var("SOLIDCRAFT_XTASK_TEST_FAIL").ok().as_deref(), Some("1"), "intentional child-process failure"); + } + + fn fixture(path: &std::path::Path, fail: bool) -> Command { + let mut c = Command::new(std::env::current_exe().unwrap()); + c.args(["--exact", "test_nextest::tests::subprocess_fixture", "--test-threads=1"]) + .env("SOLIDCRAFT_XTASK_TEST_MARKER", path) + .env("SOLIDCRAFT_XTASK_TEST_FAIL", if fail { "1" } else { "0" }) + // Keep nested libtest reports out of the parent harness transcript. + .stdout(Stdio::null()) + .stderr(Stdio::null()); + c + } + + #[test] + fn subprocess_regressions_leave_one_unambiguous_test_report() { + let out = Command::new(std::env::current_exe().unwrap()) + .args(["--exact", "test_nextest::tests::failures_propagate_without_skipping_later_side_gates", "--test-threads=1"]) + .output() + .unwrap(); + assert!(out.status.success(), "{out:?}"); + let stdout = String::from_utf8(out.stdout).unwrap(); + assert_eq!(stdout.lines().filter(|line| line.starts_with("test result:")).count(), 1, "{stdout:?}"); + assert!(!stdout.contains("test test_nextest::tests::subprocess_fixture"), "{stdout:?}"); + } + + #[test] + fn failures_propagate_without_skipping_later_side_gates() { + static NEXT: AtomicU64 = AtomicU64::new(0); + for (native_fails, side_fails, expected) in [(true, false, Some("native")), (false, true, Some("first")), (false, false, None)] { + let dir = std::env::temp_dir().join(format!("solidcraft-xtask-nextest-{}-{}", std::process::id(), NEXT.fetch_add(1, Ordering::Relaxed))); + std::fs::create_dir_all(&dir).unwrap(); + let native = dir.join("native"); + let first = dir.join("first"); + let last = dir.join("last"); + let result = execute( + ("native".into(), fixture(&native, native_fails)), + vec![("first".into(), fixture(&first, side_fails)), ("last".into(), fixture(&last, false))], + ); + assert_eq!(result, expected.map_or(Ok(()), |label| Err(format!("failed: {label}")))); + assert!(native.is_file() && first.is_file() && last.is_file()); + std::fs::remove_dir_all(dir).unwrap(); + } + } +} diff --git a/xtask/src/ui_containers.rs b/xtask/src/ui_containers.rs index 5cddfa9467..475f79851a 100644 --- a/xtask/src/ui_containers.rs +++ b/xtask/src/ui_containers.rs @@ -467,7 +467,7 @@ mod tests { std::fs::write(&source, "fn show() { egui::Area::new(id); }").unwrap(); std::fs::write(&example, "fn main() { egui::Window::new(title); }").unwrap(); std::fs::write(&implementation, "fn factory() { egui::Window::new(title); }").unwrap(); - let rejected = run(&root).unwrap_err(); + let rejected = run(&root).unwrap_err().replace('\\', "/"); assert!(rejected.contains("future-ui/src/nested/panel.rs")); assert!(rejected.contains("future-platform/examples/viewer.rs")); assert!(!rejected.contains("ui_kit/container.rs"), "only the implementation is exempt");