diff --git a/tests/kernel-cases/ladder/golden.json b/tests/kernel-cases/ladder/golden.json index 381124689d..7d0435ca0c 100644 --- a/tests/kernel-cases/ladder/golden.json +++ b/tests/kernel-cases/ladder/golden.json @@ -1054,7 +1054,8 @@ "faces": 204, "vertices": 398, "volume_mm3": 1804.924132147576 - } + }, + "unstable": "SMLib v1.1.0-alpha: the union's post-process keeps pointers to edges it has deleted, so the same input makes 597 or 594 edges; fixed in v1.1.0-alpha.1" }, "ladder/06-threads/019-external-tr40x14-p7-two-start": { "measured": { diff --git a/xtask/src/kernel_ladder.rs b/xtask/src/kernel_ladder.rs index 7ea61f219e..978b904871 100644 --- a/xtask/src/kernel_ladder.rs +++ b/xtask/src/kernel_ladder.rs @@ -94,12 +94,16 @@ fn rel_of(r: &Value) -> String { r["path"].as_str().unwrap_or("").to_string() } -/// The golden entry for one result: status, the commands' time, what was made. -fn golden_entry(r: &Value) -> Value { +/// The golden entry for one result: status, the commands' time, what was made, and the old +/// entry's `unstable` reason, which only an edit of the golden removes. +fn golden_entry(r: &Value, old: Option<&Value>) -> Value { let mut g = json!({"status": r["status"], "seconds": r["seconds"]}); if let Some(m) = r.get("measured").filter(|m| m.is_object()) { g["measured"] = m.clone(); } + if let Some(why) = old.and_then(|o| o.get("unstable")).filter(|w| w.is_string()) { + g["unstable"] = why.clone(); + } g } @@ -118,13 +122,17 @@ fn flags(r: &Value, g: Option<&Value>) -> Vec { /// What changed against the golden that the machine does not explain: no golden, a status /// change, other counts. Times are left out (they depend on the machine the golden was recorded -/// on), so these are what the gate fails on. +/// on), so these are what the gate fails on. A case the golden marks `unstable` (its counts differ +/// from run to run, with the reason) is compared on its status only. fn changes(r: &Value, g: Option<&Value>) -> Vec { let Some(g) = g else { return vec!["no golden".into()] }; let mut out = Vec::new(); if g["status"] != r["status"] { out.push(format!("status was {}", g["status"].as_str().unwrap_or("?"))); } + if g["unstable"].is_string() { + return out; + } for k in ["body_count", "faces", "edges", "vertices"] { let (now, then) = (&r["measured"][k], &g["measured"][k]); if !now.is_null() && !then.is_null() && now != then { @@ -139,8 +147,12 @@ fn changes(r: &Value, g: Option<&Value>) -> Vec { /// re-recorded with its reason (`--record`) in the same pull request. fn gate(tiers: &[Tier], results: &BTreeMap, golden: &Value) -> Result { let mut problems = Vec::new(); + let mut unstable = Vec::new(); let mut pass = 0; for c in tiers.iter().flat_map(|t| &t.cases) { + if let Some(why) = golden["cases"][c.as_str()]["unstable"].as_str() { + unstable.push(format!("{c} ({why})")); + } let Some(r) = results.get(c) else { problems.push(format!("{c}: no result")); continue; @@ -153,7 +165,11 @@ fn gate(tiers: &[Tier], results: &BTreeMap, golden: &Value) -> Re } let n: usize = tiers.iter().map(|t| t.cases.len()).sum(); if problems.is_empty() { - Ok(format!("kernel ladder gate: {pass}/{n} pass, every case as the golden records it")) + let mut ok = format!("kernel ladder gate: {pass}/{n} pass, every case as the golden records it"); + for u in &unstable { + ok += &format!("\n unstable, counts not compared: {u}"); + } + Ok(ok) } else { Err(format!( "kernel ladder gate: {} case(s) differ from tests/kernel-cases/ladder/golden.json (re-record with --record \"why\" if the change is right):\n {}", @@ -225,7 +241,8 @@ fn record(reason: &str, manifest: &Value, tiers: &[Tier], results: &BTreeMap