smlib, kernel: a refused setback corner is named by its point and why

SMLib's out_failed_corner reaches Rust as Error::CornerRefused { corner }, and the
kernel names the corner by its point: a G2 fillet at a corner of mixed convexity
says where it is and that SMLib sets back only corners whose rounds are all
convex or all concave. Engine test checks the message through solid.fillet.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01LUXeKMoxpM7Vuv8Rb6E3K2

crates/engine/src/cmd/features.rs

@@ -3248,6 +3248,16 @@ mod corner_tests {
         assert!(s.execute("solid.fillet", &json!({"edges": [[10, 0, 0]], "radius": 4, "corner": "round"})).is_err());
         assert!(s.execute("solid.fillet", &json!({"edges": [[10, 0, 0]], "radius": 4, "corner": "g2", "type": "chord"})).is_err());
         assert!(s.execute("solid.fillet", &json!({"edges": [[10, 0, 0]], "radius": 4, "corner": "g2", "fit": "reduce"})).is_err());
+        // A corner where convex and concave rounds meet (the top of a notch's inner edge, two
+        // convex rounds and the concave one) is refused, the message naming it and why.
+        let mut s = Session::default();
+        run(&mut s, "solid.box", &json!({"length": 60, "width": 40, "height": 20, "body_name": "B"}));
+        run(&mut s, "solid.box", &json!({"corner": [40, 20, 10], "length": 30, "width": 30, "height": 20, "body_name": "N"}));
+        run(&mut s, "solid.combine", &json!({"target": "B", "tools": ["N"], "operation": "cut"}));
+        let mixed = json!({"edges": [[40, 20, 15], [50, 20, 20], [40, 30, 20]], "radius": 3, "corner": "g2"});
+        let err = s.execute("solid.fillet", &mixed).unwrap_err().to_string();
+        eprintln!("mixed corner g2: {err}");
+        assert!(err.contains("not supported yet") && err.contains("(40.000, 20.000, 20.000)") && err.contains("all convex or all concave"), "{err}");
     }
 }
 #[cfg(test)]

crates/kernel/src/blend_smlib.rs

@@ -217,7 +217,9 @@ mod imp {
         let corners = corner_vertices(body, setbacks.iter().map(|(p, _)| *p))?;
         let distances: Vec<f64> = setbacks.iter().map(|(_, d)| *d).collect();
         let tol = pick_tolerance(body);
-        run(body, what, &picked.edges, |b| b.blend_setback(Blend::Round, &picked.points, sizes, &corners, &distances, tol))
+        let at: Vec<Vec3> = setbacks.iter().map(|(p, _)| *p).collect();
+        run_with(body, what, &picked.edges, &at, |b| b.blend_setback(Blend::Round, &picked.points, sizes, &corners, &distances, tol).map(|r| (r, ())))
+            .map(|(out, (), _)| out)
     }
 
     /// [`blend_setback`] with the set-back corners left open (`super::OpenCorners`).
@@ -234,9 +236,10 @@ mod imp {
         let picked = resolve(body, edges)?;
         let corners = corner_vertices(body, setbacks.iter().map(|(p, _)| *p))?;
         let distances: Vec<f64> = setbacks.iter().map(|(_, d)| *d).collect();
+        let at: Vec<Vec3> = setbacks.iter().map(|(p, _)| *p).collect();
         let tol = pick_tolerance(body);
         let (sheet, holes, order) =
-            run_with(body, what, &picked.edges, |b| b.blend_setback_open(Blend::Round, &picked.points, sizes, &corners, &distances, tol))?;
+            run_with(body, what, &picked.edges, &at, |b| b.blend_setback_open(Blend::Round, &picked.points, sizes, &corners, &distances, tol))?;
         // SMLib's indices in the sheet's own orders.
         let place = |list: &[usize], i: usize| if list.is_empty() { Some(i) } else { list.get(i).copied() };
         let renumber = |hole: &solidcraft_smlib::Hole| -> Result<Vec<(usize, bool)>> {
@@ -302,15 +305,18 @@ mod imp {
 
     /// `op` on `body` in SMLib, tagged as operand 0, checked, decoded, with its history.
     fn run(body: &Body, what: &str, edges: &[usize], op: impl FnOnce(&Brep) -> solidcraft_smlib::Result<Brep>) -> Result<Body> {
-        run_with(body, what, edges, |b| op(b).map(|r| (r, ()))).map(|(out, (), _)| out)
+        run_with(body, what, edges, &[], |b| op(b).map(|r| (r, ()))).map(|(out, (), _)| out)
     }
 
     /// [`run`] for an `op` that also gives something of its own, with where SMLib's faces and
     /// edges of the result are in the body's orders.
+    /// `corners`: the corner points of a blend with setbacks, in the order `op` gives them to
+    /// SMLib, so a refused corner is named by its place.
     fn run_with<T>(
         body: &Body,
         what: &str,
         edges: &[usize],
+        corners: &[Vec3],
         op: impl FnOnce(&Brep) -> solidcraft_smlib::Result<(Brep, T)>,
     ) -> Result<(Body, T, Option<solidcraft_smlib::history::Order>)> {
         guard(what, || {
@@ -318,6 +324,15 @@ mod imp {
             let tagged = sb.tagged(0).map_err(|e| failed(what, "tag", e))?;
             let (r, extra) = op(&tagged).map_err(|e| match e {
                 SmError::Unsupported => super::unsupported(&format!("this {what}, which SMLib cannot build exactly")),
+                SmError::CornerRefused { corner } => match corners.get(corner) {
+                    // The same words as `super::unsupported`, the reason after them.
+                    Some(p) => KernelError::Invalid(format!(
+                        "not supported yet: setting back the corner at ({:.3}, {:.3}, {:.3}) with the SMLib kernel (a G2 corner is set \
+                         back): SMLib sets back a corner only where every edge meeting it is rounded and they are all convex or all concave",
+                        p.x, p.y, p.z
+                    )),
+                    None => failed(what, "operation", SmError::CornerRefused { corner }),
+                },
                 SmError::NoEffect => KernelError::Failed(format!("SMLib made no change: the {what} reported success but built no face")),
                 SmError::NoRoom => KernelError::TooLarge(format!("the {what} is larger than the faces beside its edges allow")),
                 e => failed(what, "operation", e),

crates/kernel/src/corner_g2_tests.rs

@@ -189,7 +189,10 @@ fn a_concave_corner_takes_the_g2_patch_too() {
     let (m, mb) = (measure(&g2).unwrap(), measure(&ball).unwrap());
     let (_, clear, n) = corner_seams(&g2, 0.15);
     let (_, ball_clear, _) = corner_seams(&ball, 0.15);
-    eprintln!("concave G2 corner: {} faces (ball {}), volume {:.4} (ball {:.4}), jump clear of the corners {clear:.3e} ({n} stations), ball's {ball_clear:.3e}", m.faces, mb.faces, m.volume, mb.volume);
+    eprintln!(
+        "concave G2 corner: {} faces (ball {}), volume {:.4} (ball {:.4}), jump clear of the corners {clear:.3e} ({n} stations), ball's {ball_clear:.3e}",
+        m.faces, mb.faces, m.volume, mb.volume
+    );
     assert_eq!(m.faces, mb.faces);
     assert!(clear <= CURVATURE_SHARE / 5.0 && ball_clear > 0.19, "{clear} {ball_clear}");
 }
@@ -203,5 +206,6 @@ fn a_mixed_convexity_corner_is_refused() {
     assert!(fillet(&body, &edges, 5.0).is_ok());
     let err = fillet_g2_corners(&body, &edges, 5.0, &[]).unwrap_err().to_string();
     eprintln!("mixed corner: {err}");
-    assert!(err.contains("not supported yet"), "{err}");
+    // The refusal names the corner and why: its rounds are not all of one convexity.
+    assert!(err.contains("not supported yet") && err.contains("(60.000, 20.000, 10.000)") && err.contains("all convex or all concave"), "{err}");
 }

crates/smlib/src/ffi/ops.rs

@@ -43,6 +43,19 @@ pub enum Pick {
 /// or several things (only for `ERR_PICK_*`).
 pub type PickError = (i32, Option<usize>);
 
+/// A failed blend with corner setbacks: the status, the index of the edge point that selected
+/// nothing or several edges (only for `ERR_PICK_*`), and the index of the corner point whose
+/// corner SMLib refused to set back (only for `ERR_UNSUPPORTED`).
+pub type SetbackError = (i32, Option<usize>, Option<usize>);
+
+/// The [`SetbackError`] of status `s`, given what the C call wrote to its failed indices.
+#[cfg(feature = "native")]
+fn setback_error(s: i32, failed: i64, failed_corner: i64) -> SetbackError {
+    let at = (s == ERR_PICK_NONE || s == ERR_PICK_AMBIGUOUS).then(|| usize::try_from(failed).ok()).flatten();
+    let corner = (s == ERR_UNSUPPORTED).then(|| usize::try_from(failed_corner).ok()).flatten();
+    (s, at, corner)
+}
+
 #[cfg(feature = "native")]
 type PickFn = unsafe extern "C" fn(*const RawBrep, *const f64, i64, f64, f64, *mut *mut RawBrep, *mut i64) -> i32;
 
@@ -308,11 +321,11 @@ pub fn blend_setback(
     corners: &[[f64; 3]],
     setbacks: &[f64],
     tolerance: f64,
-) -> Result<Handle, PickError> {
-    let n = i64::try_from(points.len()).map_err(|_| (ERR_ARG, None))?;
-    let nc = i64::try_from(corners.len()).map_err(|_| (ERR_ARG, None))?;
+) -> Result<Handle, SetbackError> {
+    let n = i64::try_from(points.len()).map_err(|_| (ERR_ARG, None, None))?;
+    let nc = i64::try_from(corners.len()).map_err(|_| (ERR_ARG, None, None))?;
     if sizes.len() != points.len() || setbacks.len() != corners.len() {
-        return Err((ERR_ARG, None));
+        return Err((ERR_ARG, None, None));
     }
     let flat = points.as_ptr().cast::<f64>();
     let flat_corners = corners.as_ptr().cast::<f64>();
@@ -333,15 +346,12 @@ pub fn blend_setback(
             &mut failed_corner,
         )
     })
-    .map_err(|s| {
-        let at = (s == ERR_PICK_NONE || s == ERR_PICK_AMBIGUOUS).then(|| usize::try_from(failed).ok()).flatten();
-        (s, at)
-    })
+    .map_err(|s| setback_error(s, failed, failed_corner))
 }
 
 #[cfg(not(feature = "native"))]
-pub fn blend_setback(_: i32, _: &Handle, _: &[[f64; 3]], _: &[f64], _: &[[f64; 3]], _: &[f64], _: f64) -> Result<Handle, PickError> {
-    Err((NOT_BUILT, None))
+pub fn blend_setback(_: i32, _: &Handle, _: &[[f64; 3]], _: &[f64], _: &[[f64; 3]], _: &[f64], _: f64) -> Result<Handle, SetbackError> {
+    Err((NOT_BUILT, None, None))
 }
 
 /// `smlib_blend_setback_open`: the sheet and the holes' values as the C function writes them
@@ -356,17 +366,17 @@ pub fn blend_setback_open(
     corners: &[[f64; 3]],
     setbacks: &[f64],
     tolerance: f64,
-) -> Result<(Handle, Vec<i64>), PickError> {
-    let n = i64::try_from(points.len()).map_err(|_| (ERR_ARG, None))?;
-    let nc = i64::try_from(corners.len()).map_err(|_| (ERR_ARG, None))?;
+) -> Result<(Handle, Vec<i64>), SetbackError> {
+    let n = i64::try_from(points.len()).map_err(|_| (ERR_ARG, None, None))?;
+    let nc = i64::try_from(corners.len()).map_err(|_| (ERR_ARG, None, None))?;
     if sizes.len() != points.len() || setbacks.len() != corners.len() {
-        return Err((ERR_ARG, None));
+        return Err((ERR_ARG, None, None));
     }
     let flat = points.as_ptr().cast::<f64>();
     let flat_corners = corners.as_ptr().cast::<f64>();
     let mut holes = vec![0i64; corners.len().saturating_mul(1 + 3 * 64)];
     for _ in 0..2 {
-        let cap = i64::try_from(holes.len()).map_err(|_| (ERR_ARG, None))?;
+        let cap = i64::try_from(holes.len()).map_err(|_| (ERR_ARG, None, None))?;
         let (mut failed, mut failed_corner, mut len) = (-1i64, -1i64, -1i64);
         let made = new_brep(|out| unsafe {
             smlib_blend_setback_open(
@@ -389,22 +399,27 @@ pub fn blend_setback_open(
         });
         match made {
             Ok(h) => {
-                holes.truncate(usize::try_from(len).map_err(|_| (ERR_BUFFER, None))?);
+                holes.truncate(usize::try_from(len).map_err(|_| (ERR_BUFFER, None, None))?);
                 return Ok((h, holes));
             }
-            Err(ERR_BUFFER) if len > cap => holes.resize(usize::try_from(len).map_err(|_| (ERR_BUFFER, None))?, 0),
-            Err(s) => {
-                let at = (s == ERR_PICK_NONE || s == ERR_PICK_AMBIGUOUS).then(|| usize::try_from(failed).ok()).flatten();
-                return Err((s, at));
-            }
+            Err(ERR_BUFFER) if len > cap => holes.resize(usize::try_from(len).map_err(|_| (ERR_BUFFER, None, None))?, 0),
+            Err(s) => return Err(setback_error(s, failed, failed_corner)),
         }
     }
-    Err((ERR_BUFFER, None))
+    Err((ERR_BUFFER, None, None))
 }
 
 #[cfg(not(feature = "native"))]
-pub fn blend_setback_open(_: i32, _: &Handle, _: &[[f64; 3]], _: &[f64], _: &[[f64; 3]], _: &[f64], _: f64) -> Result<(Handle, Vec<i64>), PickError> {
-    Err((NOT_BUILT, None))
+pub fn blend_setback_open(
+    _: i32,
+    _: &Handle,
+    _: &[[f64; 3]],
+    _: &[f64],
+    _: &[[f64; 3]],
+    _: &[f64],
+    _: f64,
+) -> Result<(Handle, Vec<i64>), SetbackError> {
+    Err((NOT_BUILT, None, None))
 }
 
 /// `smlib_fillet_variable`.

crates/smlib/src/lib.rs

@@ -58,6 +58,9 @@ pub enum Error {
     WallsTouch,
     /// SMLib cannot build this exactly yet (a variable radius whose law has a corner).
     Unsupported,
+    /// SMLib does not set back the corner of corner point `corner`: SMLib sets back a corner
+    /// only where every edge meeting it is blended and they are all convex or all concave.
+    CornerRefused { corner: usize },
     /// SMLib reported success but the operation did nothing: a blend that made no face, a face
     /// move or draft that moved no face.
     NoEffect,
@@ -89,6 +92,11 @@ impl std::fmt::Display for Error {
             Error::NoRoom => write!(f, "there is no room for it: an offset leaves none inside the body, or a blend is larger than its faces allow"),
             Error::WallsTouch => write!(f, "the offset's walls would meet along a line, which leaves no solid there"),
             Error::Unsupported => write!(f, "not supported yet: SMLib cannot build this exactly"),
+            Error::CornerRefused { corner } => write!(
+                f,
+                "not supported yet: SMLib does not set back corner {}: it sets back a corner only where every edge meeting it is blended, all convex or all concave",
+                corner + 1
+            ),
             Error::NoEffect => write!(f, "SMLib made no change: it reported success, but the operation did nothing"),
             Error::NotExact => write!(f, "SMLib cannot build this exactly in the input's geometry"),
             Error::Corner { why, at } => {

crates/smlib/src/ops.rs

@@ -55,6 +55,13 @@ fn pick_error((s, at): ffi::ops::PickError) -> Error {
     }
 }
 
+fn setback_error((s, at, corner): ffi::ops::SetbackError) -> Error {
+    match corner {
+        Some(corner) => Error::CornerRefused { corner },
+        None => pick_error((s, at)),
+    }
+}
+
 /// One edge round an opened corner's hole ([`Brep::blend_setback_open`]): its index in the
 /// sheet's edge order, the face beside it in its face order, and whether that face, run
 /// counter-clockwise about its outward normal, runs the edge from its start vertex to its end.
@@ -214,7 +221,7 @@ impl Brep {
             Blend::Chord => ffi::ops::BLEND_CHORD,
             Blend::Chamfer => ffi::ops::BLEND_CHAMFER,
         };
-        ffi::ops::blend_setback(kind, &self.0, edges, sizes, corners, setbacks, tolerance).map(Brep).map_err(pick_error)
+        ffi::ops::blend_setback(kind, &self.0, edges, sizes, corners, setbacks, tolerance).map(Brep).map_err(setback_error)
     }
 
     /// [`Brep::blend_setback`] with each set-back corner left open: SMLib's patch between the
@@ -247,7 +254,7 @@ impl Brep {
             Blend::Chord => ffi::ops::BLEND_CHORD,
             Blend::Chamfer => ffi::ops::BLEND_CHAMFER,
         };
-        let (h, values) = ffi::ops::blend_setback_open(kind, &self.0, edges, sizes, corners, setbacks, tolerance).map_err(pick_error)?;
+        let (h, values) = ffi::ops::blend_setback_open(kind, &self.0, edges, sizes, corners, setbacks, tolerance).map_err(setback_error)?;
         // A buffer that does not read as the C API documents it is a buffer error.
         let holes = holes(&values, corners.len()).ok_or_else(|| error(ffi::ERR_BUFFER))?;
         Ok((Brep(h), holes))

crates/smlib/src/ops_tests.rs

@@ -403,6 +403,9 @@ fn a_setback_corner_left_open_returns_its_hole() {
         assert_eq!(end(h, true), end(next, false), "edge {k} ends where the next starts");
         assert!(h.face < 9);
     }
-    // A corner where an edge is not rounded builds no setback.
-    assert!(matches!(b.blend_setback_open(Blend::Round, &edges[..2], &[5.0; 2], &[[60.0, 40.0, 20.0]], &[8.0], 0.1), Err(Error::Unsupported)));
+    // A corner where an edge is not rounded builds no setback, and the refusal names it.
+    let refused = b.blend_setback_open(Blend::Round, &edges[..2], &[5.0; 2], &[[60.0, 40.0, 20.0]], &[8.0], 0.1);
+    assert!(matches!(refused, Err(Error::CornerRefused { corner: 0 })), "{refused:?}");
+    let refused = b.blend_setback(Blend::Round, &edges[..2], &[5.0; 2], &[[60.0, 40.0, 20.0]], &[8.0], 0.1);
+    assert!(matches!(refused, Err(Error::CornerRefused { corner: 0 })), "{refused:?}");
 }