Skip to content

SolidCraft: an open-source parametric 3D CAD application built in pure Rust. Desktop, web, and automation through MCP.

MCP: compare the model with a reference picture (#1486) (#6416)

Dahak Builds251573a5a477
diff --git a/Cargo.lock b/Cargo.lock
index c98f5fa9ba..2d6fe07de1 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -4383,6 +4383,7 @@ dependencies = [
name = "solidcraft-mcp"
version = "0.29.0"
dependencies = [
+ "image",
"serde_json",
"solidcraft-engine",
"solidcraft-gpu",
diff --git a/assets/locales/pending/compare-picture.json b/assets/locales/pending/compare-picture.json
new file mode 100644
index 0000000000..d6966b2d4f
--- /dev/null
+++ b/assets/locales/pending/compare-picture.json
@@ -0,0 +1,20 @@
+{
+ "en": {
+ "cmd.inspect_compare_picture": "Compare Picture"
+ },
+ "ar": {
+ "cmd.inspect_compare_picture": "مقارنة بصورة مرجعية"
+ },
+ "he": {
+ "cmd.inspect_compare_picture": "השוואה לתמונת ייחוס"
+ },
+ "ja": {
+ "cmd.inspect_compare_picture": "参考画像と比較"
+ },
+ "vi": {
+ "cmd.inspect_compare_picture": "So sánh với ảnh tham chiếu"
+ },
+ "zh": {
+ "cmd.inspect_compare_picture": "比较参考图"
+ }
+}
diff --git a/crates/engine/Cargo.toml b/crates/engine/Cargo.toml
index 4239cc0ae8..075c37fd30 100644
--- a/crates/engine/Cargo.toml
+++ b/crates/engine/Cargo.toml
@@ -37,6 +37,7 @@ solidcraft-kernel = { workspace = true }
solidcraft-doc = { workspace = true }
solidcraft-io = { workspace = true }
solidcraft-render = { workspace = true }
+# reference pictures for inspect.compare_picture (PNG and JPEG decode, PNG encode)
image = { workspace = true, features = ["jpeg"] }
serde = { workspace = true }
serde_json = { workspace = true }
diff --git a/crates/engine/src/cmd/compare_picture.rs b/crates/engine/src/cmd/compare_picture.rs
new file mode 100644
index 0000000000..6dce661b0f
--- /dev/null
+++ b/crates/engine/src/cmd/compare_picture.rs
@@ -0,0 +1,692 @@
+//! Compare the model with a reference picture (#1486): agents model to a picture, and an
+//! overlap score plus per-edge mm offsets tell the agent which edge to move and by how much.
+
+use std::collections::{HashMap, VecDeque};
+use std::io::Cursor;
+
+use serde_json::{Value, json};
+use solidcraft_doc::canvas::base64_encode;
+
+use super::CommandSpec;
+use super::silhouette::{self, Silhouette};
+use crate::params::bad;
+use crate::{Result, Session};
+
+const CMD: &str = "inspect.compare_picture";
+/// A reference picture is read through the vfs with a byte cap, like `screenshot` output.
+const MAX_REFERENCE_BYTES: usize = 32 << 20;
+/// Decoded pixels are capped as well, so a small file cannot unfold into a huge raster.
+const MAX_DECODED_PIXELS: u64 = 16 << 20;
+/// The comparison grid each mask is fitted into at most — the picture's and the fitted
+/// model's alike; keeps the slide search and the memory it needs bounded.
+const MAX_GRID: usize = 1024;
+/// Background grows from the border only through steps of at most this per channel, so a
+/// backdrop's gradual shading belongs to it and the step at the part's outline stops it.
+const BG_STEP: u16 = 12;
+/// When the first pass finds no part, the flood retries at this stricter step: a white part
+/// on a (245, 245, 245) studio backdrop is a step of 10, which BG_STEP lets through.
+const BG_STEP_STRICT: u16 = 4;
+/// A picture whose border is mostly transparent: a pixel below this alpha is see-through.
+const ALPHA_BG: u8 = 128;
+/// The global retry's backdrop: a pixel within this of the border's main colour is
+/// background. A light JPEG (white on (245, 245, 245) at quality 75) blurs the step at the
+/// part's outline below any local limit, so that pass compares to the backdrop's colour.
+const BG_NEAR: u16 = 8;
+/// Stations per side for the edge offsets.
+const STATIONS: usize = 9;
+
+// For agents modelling to a picture; nothing to show a person, so no toolbar button.
+pub static COMMANDS: &[CommandSpec] = &[CommandSpec::new(CMD, "Compare Picture", run).noundo().icon("compare").params(
+ "reference: path to a PNG or JPEG the model should match; view?/camera?/bodies?/rows? as inspect.silhouette; \
+ crop_px?: [x, y, w, h] in pixels, or crop_fraction?: [x, y, w, h] as fractions 0..1 of the picture \
+ (to take one view off a board); part_fills_crop?: false — with true the crop box is the part itself \
+ and no background search runs, for a crop that tightly frames the part; \
+ fit?: \"width\" | \"height\" | \"both\" (default \"width\"): one uniform scale, taken from the named \
+ dimension (\"both\" claims both and scales by width, reporting aspect_mismatch when the shapes differ); \
+ overlay?: false leaves the overlay PNG out → overlap (0..1), edges: top|bottom|left|right in mm at nine stations \
+ (positive where the model reaches past the picture, null where a station misses one of the two), the grid and its \
+ mm_per_px, and overlay_png (red model, blue picture) as base64 PNG. The compare is outline only: \
+ holes are not compared — both the picture's and the model's holes are filled",
+)];
+
+fn run(s: &mut Session, p: &Value) -> Result<Value> {
+ let reference = p
+ .get("reference")
+ .and_then(Value::as_str)
+ .filter(|v| !v.trim().is_empty())
+ .ok_or_else(|| bad(CMD, "`reference` is required: a path to a PNG or JPEG"))?;
+ let fit = match p.get("fit") {
+ None => Fit::Width,
+ Some(Value::String(v)) => match v.as_str() {
+ "width" => Fit::Width,
+ "height" => Fit::Height,
+ "both" => Fit::Both,
+ other => return Err(bad(CMD, format!("`fit` must be \"width\", \"height\" or \"both\", not `{other}`"))),
+ },
+ Some(_) => return Err(bad(CMD, "`fit` must be a string")),
+ };
+ let overlay = p.get("overlay").map(|v| v.as_bool().unwrap_or(false)).unwrap_or(true);
+ let part_fills_crop = p.get("part_fills_crop").and_then(Value::as_bool).unwrap_or(false);
+
+ // The model side: the same outline inspect.silhouette returns, as filled rows.
+ let camera = silhouette::camera(p)?;
+ let rows = silhouette::rows(p)?;
+ let wanted = silhouette::body_names(p)?;
+ let model = silhouette::outline(s, &camera, wanted.as_deref(), rows)?;
+
+ // The picture side: decode, crop, and find the part.
+ let bytes = solidcraft_io::vfs::read(reference, MAX_REFERENCE_BYTES).map_err(|e| bad(CMD, format!("{reference}: {e}")))?;
+ let img = decode(&bytes)?;
+ let img = apply_crop(img, p.get("crop_px"), p.get("crop_fraction"))?;
+ let picture = if part_fills_crop {
+ part_from_box(&img).map_err(|reason| bad(CMD, format!("the crop cannot be the part: {reason}")))?
+ } else {
+ find_part(&img).map_err(|reason| bad(CMD, format!("no part found in the picture: {reason}")))?
+ };
+
+ // Fit the model outline to the picture's scale, then slide it to the best overlap.
+ let (mw, mh, mmpp_x, mmpp_y) = fit_model(&model, picture.width, picture.height, fit);
+ let mut model_scaled = resample(&model.mask, model.width, model.height, mw, mh);
+ // The same hole rule the picture side has: a through-hole in the model counts as its
+ // outline, not as a hole the picture has to match.
+ fill_holes(&mut model_scaled);
+ let best = best_overlap(&model_scaled, &picture);
+ let placed = Placed { dx: best.dx, dy: best.dy, mask: &model_scaled };
+ let aspect_mismatch = (picture.width as f64 * mh as f64) / (picture.height as f64 * mw as f64);
+
+ let mut out = json!({
+ "overlap": round4(best.overlap),
+ "stations": STATIONS,
+ "edges": {
+ "top": edge_offsets(&placed, &picture, Edge::Top, mmpp_y),
+ "bottom": edge_offsets(&placed, &picture, Edge::Bottom, mmpp_y),
+ "left": edge_offsets(&placed, &picture, Edge::Left, mmpp_x),
+ "right": edge_offsets(&placed, &picture, Edge::Right, mmpp_x),
+ },
+ "fit": match fit { Fit::Width => "width", Fit::Height => "height", Fit::Both => "both" },
+ "aspect_mismatch": round4(aspect_mismatch),
+ "mm_per_px": {"x": round4(mmpp_x), "y": round4(mmpp_y)},
+ "model": {"width": model.width, "height": model.height, "box_mm": [[round4(model.lo[0]), round4(model.lo[1])], [round4(model.hi[0]), round4(model.hi[1])]], "bodies": model.bodies},
+ "picture": {"width": picture.width, "height": picture.height, "source": reference},
+ });
+ if overlay {
+ out["overlay_png"] = json!(base64_encode(&overlay_png(&picture, &placed)));
+ }
+ Ok(out)
+}
+
+#[derive(Clone, Copy)]
+enum Fit {
+ Width,
+ Height,
+ Both,
+}
+
+#[derive(Clone, Copy)]
+enum Edge {
+ Top,
+ Bottom,
+ Left,
+ Right,
+}
+
+/// A picture mask, tightly framed to the found part.
+struct Part {
+ width: usize,
+ height: usize,
+ mask: Vec<Vec<bool>>,
+}
+
+/// A model mask placed relative to the picture: its top-left sits at (dx, dy) in the
+/// picture's own pixel coordinates, which may be negative.
+struct Placed<'a> {
+ dx: i64,
+ dy: i64,
+ mask: &'a [Vec<bool>],
+}
+
+/// Decode PNG or JPEG to RGBA8, refusing anything else and anything that unfolds too large.
+fn decode(bytes: &[u8]) -> Result<image::RgbaImage> {
+ let guess = image::ImageReader::new(Cursor::new(bytes)).with_guessed_format().map_err(|e| bad(CMD, format!("cannot read the picture: {e}")))?;
+ if !matches!(guess.format(), Some(image::ImageFormat::Png) | Some(image::ImageFormat::Jpeg)) {
+ return Err(bad(CMD, "the reference must be a PNG or a JPEG"));
+ }
+ let (w, h) = guess.into_dimensions().map_err(|e| bad(CMD, format!("cannot read the picture: {e}")))?;
+ if w as u64 * h as u64 > MAX_DECODED_PIXELS {
+ return Err(bad(CMD, format!("the picture is {w}x{h}: over the {} megapixel limit", MAX_DECODED_PIXELS >> 20)));
+ }
+ let reader = image::ImageReader::new(Cursor::new(bytes)).with_guessed_format().map_err(|e| bad(CMD, format!("cannot read the picture: {e}")))?;
+ let img = reader.decode().map_err(|e| bad(CMD, format!("cannot decode the picture: {e}")))?;
+ Ok(img.to_rgba8())
+}
+
+/// Apply `crop_px: [x, y, w, h]` (pixels) or `crop_fraction: [x, y, w, h]` (fractions 0..1
+/// of the picture). Explicit units: a crop of `[0, 0, 1, 1]` means one pixel, not the whole.
+fn apply_crop(img: image::RgbaImage, crop_px: Option<&Value>, crop_fraction: Option<&Value>) -> Result<image::RgbaImage> {
+ match (crop_px, crop_fraction) {
+ (Some(_), Some(_)) => Err(bad(CMD, "give `crop_px` or `crop_fraction`, not both")),
+ (None, None) => Ok(img),
+ (Some(v), None) => {
+ let nums = crop_nums(v, "crop_px")?;
+ let (w, h) = img.dimensions();
+ crop_to(img, nums[0].round() as usize, nums[1].round() as usize, nums[2].round() as usize, nums[3].round() as usize, w, h)
+ }
+ (None, Some(v)) => {
+ let nums = crop_nums(v, "crop_fraction")?;
+ if nums.iter().any(|f| *f > 1.0) {
+ return Err(bad(CMD, "`crop_fraction` values must be 0..1"));
+ }
+ let (w, h) = img.dimensions();
+ crop_to(
+ img,
+ (nums[0] * w as f64).round() as usize,
+ (nums[1] * h as f64).round() as usize,
+ (nums[2] * w as f64).round() as usize,
+ (nums[3] * h as f64).round() as usize,
+ w,
+ h,
+ )
+ }
+ }
+}
+
+fn crop_nums(v: &Value, name: &str) -> Result<Vec<f64>> {
+ let nums: Vec<f64> = v
+ .as_array()
+ .filter(|a| a.len() == 4)
+ .and_then(|a| a.iter().map(|v| v.as_f64()).collect())
+ .ok_or_else(|| bad(CMD, format!("`{name}` must be [x, y, w, h]")))?;
+ if nums.iter().any(|v| !v.is_finite() || *v < 0.0) {
+ return Err(bad(CMD, format!("`{name}` values must be non-negative")));
+ }
+ Ok(nums)
+}
+
+fn crop_to(img: image::RgbaImage, x: usize, y: usize, cw: usize, ch: usize, w: u32, h: u32) -> Result<image::RgbaImage> {
+ if x >= w as usize || y >= h as usize {
+ return Err(bad(CMD, "the crop starts outside the picture"));
+ }
+ let cw = cw.max(1).min(w as usize - x);
+ let ch = ch.max(1).min(h as usize - y);
+ Ok(image::imageops::crop_imm(&img, x as u32, y as u32, cw as u32, ch as u32).to_image())
+}
+
+/// Find the part. Alpha decides only when the backdrop really is transparent — most border
+/// pixels see-through — so one stray translucent pixel cannot switch the read; then any ink
+/// is part, so a part drawn at a faint alpha survives. Otherwise grow the background in from
+/// the border through small colour steps only — seeded from the border pixels close to the
+/// border's main colour, so a part touching the border is not eaten by its own edge — keep
+/// what the background did not reach, and keep the largest 4-connected piece. On a retry the
+/// step is stricter, so a white part on a (245, 245, 245) backdrop (a step of 10) still
+/// separates; on a second retry the backdrop is taken globally — within BG_NEAR of the
+/// border's main colour — because a light JPEG blurs the step at the part's outline below
+/// any local limit. Says which happened when no part is found.
+fn find_part(img: &image::RgbaImage) -> std::result::Result<Part, String> {
+ let (w, h) = (img.width() as usize, img.height() as usize);
+ if w < 2 || h < 2 {
+ return Err("the picture is smaller than 2 x 2 pixels".to_string());
+ }
+ let px = |x: usize, y: usize| -> [u16; 3] {
+ let p = img.get_pixel(x as u32, y as u32).0;
+ [p[0] as u16, p[1] as u16, p[2] as u16]
+ };
+ let step = |a: [u16; 3], b: [u16; 3]| (a[0].abs_diff(b[0])).max(a[1].abs_diff(b[1])).max(a[2].abs_diff(b[2]));
+
+ // Alpha decides only when the border is mostly see-through: a single backdrop pixel at
+ // alpha 254 in an otherwise opaque picture must not turn the whole read into alpha.
+ let mut border = 0usize;
+ let mut see_through = 0usize;
+ for x in 0..w {
+ for y in [0, h - 1] {
+ border += 1;
+ if img.get_pixel(x as u32, y as u32)[3] < ALPHA_BG {
+ see_through += 1;
+ }
+ }
+ }
+ for y in 0..h {
+ for x in [0, w - 1] {
+ border += 1;
+ if img.get_pixel(x as u32, y as u32)[3] < ALPHA_BG {
+ see_through += 1;
+ }
+ }
+ }
+ let uses_alpha = see_through * 2 > border;
+
+ let (part, too_small): (Vec<bool>, String) = if uses_alpha {
+ let mask: Vec<bool> = img.pixels().map(|p| p[3] > 0).collect();
+ (mask, "the picture is transparent: no pixel holds any ink".to_string())
+ } else {
+ let bg = flood_background(&px, w, h, BG_STEP, &step);
+ let mask: Vec<bool> = (0..w * h).map(|i| !bg[i]).collect();
+ if largest_piece(&mask, w, h).len() >= 16 {
+ (mask, "the picture holds no piece big enough to be a part".to_string())
+ } else {
+ let bg = flood_background(&px, w, h, BG_STEP_STRICT, &step);
+ let mask: Vec<bool> = (0..w * h).map(|i| !bg[i]).collect();
+ if largest_piece(&mask, w, h).len() >= 16 {
+ (mask, "the picture holds no piece big enough to be a part".to_string())
+ } else {
+ // The local step fails on a light JPEG: the codec blurs the 10-level step
+ // at the part's outline into steps of a few levels each, under any limit.
+ // Take the backdrop globally instead: near the border's main colour.
+ let main = border_main_colour(&px, w, h);
+ let mask: Vec<bool> = (0..w * h).map(|i| step(main, px(i % w, i / w)) > BG_NEAR).collect();
+ (mask, "the background grew from the border over the whole picture, even with a stricter colour step: the part's colour is too close to the backdrop's".to_string())
+ }
+ }
+ };
+
+ let best = largest_piece(&part, w, h);
+ if best.len() < 16 {
+ return Err(too_small);
+ }
+ // Tight frame, then downscale to the comparison grid at need.
+ let (x0, y0, x1, y1) = frame(&best, w);
+ let (pw, ph) = (x1 - x0 + 1, y1 - y0 + 1);
+ let mut mask = vec![vec![false; pw]; ph];
+ for &i in &best {
+ let (x, y) = (i % w, i / w);
+ mask[y - y0][x - x0] = true;
+ }
+ let (dw, dh) = fit_grid(pw, ph);
+ let mut mask = if (dw, dh) == (pw, ph) { mask } else { resample(&mask, pw, ph, dw, dh) };
+ fill_holes(&mut mask);
+ Ok(Part { width: dw, height: dh, mask })
+}
+
+/// The crop box is the part itself (`part_fills_crop`): no background search. This is how
+/// one view cropped tight off a board says so — a part touching three sides leaves the
+/// border's main colour the part's own, and the flood would keep the backdrop strip instead.
+fn part_from_box(img: &image::RgbaImage) -> std::result::Result<Part, String> {
+ let (w, h) = (img.width() as usize, img.height() as usize);
+ if w < 2 || h < 2 {
+ return Err("the crop is smaller than 2 x 2 pixels".to_string());
+ }
+ let (dw, dh) = fit_grid(w, h);
+ Ok(Part { width: dw, height: dh, mask: vec![vec![true; dw]; dh] })
+}
+
+/// The border's main colour: the border colour it has most of. A part crossing the border
+/// is in the border too, but the backdrop is what most of the border shows.
+fn border_main_colour(px: &impl Fn(usize, usize) -> [u16; 3], w: usize, h: usize) -> [u16; 3] {
+ let mut counts: HashMap<[u16; 3], usize> = HashMap::new();
+ for x in 0..w {
+ for y in [0, h - 1] {
+ *counts.entry(px(x, y)).or_default() += 1;
+ }
+ }
+ for y in 0..h {
+ for x in [0, w - 1] {
+ *counts.entry(px(x, y)).or_default() += 1;
+ }
+ }
+ counts.into_iter().max_by_key(|(_, n)| *n).map(|(c, _)| c).unwrap_or([0; 3])
+}
+
+/// The background mask: grown from every border pixel whose colour is close to the border's
+/// main colour, crossing only steps of at most `limit` per channel.
+fn flood_background(px: &impl Fn(usize, usize) -> [u16; 3], w: usize, h: usize, limit: u16, step: &impl Fn([u16; 3], [u16; 3]) -> u16) -> Vec<bool> {
+ let main = border_main_colour(px, w, h);
+
+ let mut bg = vec![false; w * h];
+ let mut queue: VecDeque<(usize, usize)> = VecDeque::new();
+ let mut seed = |x: usize, y: usize| {
+ if step(main, px(x, y)) <= limit && !bg[y * w + x] {
+ bg[y * w + x] = true;
+ queue.push_back((x, y));
+ }
+ };
+ for x in 0..w {
+ seed(x, 0);
+ seed(x, h - 1);
+ }
+ for y in 0..h {
+ seed(0, y);
+ seed(w - 1, y);
+ }
+ while let Some((x, y)) = queue.pop_front() {
+ let here = px(x, y);
+ for (nx, ny) in neighbours(x, y, w, h) {
+ if !bg[ny * w + nx] && step(here, px(nx, ny)) <= limit {
+ bg[ny * w + nx] = true;
+ queue.push_back((nx, ny));
+ }
+ }
+ }
+ bg
+}
+
+/// The largest 4-connected piece of a mask, as indices into `w`-wide rows.
+fn largest_piece(mask: &[bool], w: usize, h: usize) -> Vec<usize> {
+ let mut best: Vec<usize> = Vec::new();
+ let mut seen = vec![false; w * h];
+ for start in 0..w * h {
+ if !mask[start] || seen[start] {
+ continue;
+ }
+ let mut piece: Vec<usize> = Vec::new();
+ let mut queue: VecDeque<usize> = VecDeque::new();
+ seen[start] = true;
+ queue.push_back(start);
+ while let Some(i) = queue.pop_front() {
+ piece.push(i);
+ let (x, y) = (i % w, i / w);
+ for (nx, ny) in neighbours(x, y, w, h) {
+ let j = ny * w + nx;
+ if mask[j] && !seen[j] {
+ seen[j] = true;
+ queue.push_back(j);
+ }
+ }
+ }
+ if piece.len() > best.len() {
+ best = piece;
+ }
+ }
+ best
+}
+
+/// Fill the holes: pixels the outside never reaches become part — the rule the picture's
+/// border flood gives the picture side for free, given to the model's mask as well.
+fn fill_holes(mask: &mut [Vec<bool>]) {
+ let (h, w) = (mask.len(), mask.first().map_or(0, Vec::len));
+ if w < 2 || h < 2 {
+ return;
+ }
+ let mut outside = vec![false; w * h];
+ let mut queue: VecDeque<usize> = VecDeque::new();
+ for x in 0..w {
+ for y in [0, h - 1] {
+ if !mask[y][x] && !outside[y * w + x] {
+ outside[y * w + x] = true;
+ queue.push_back(y * w + x);
+ }
+ }
+ }
+ for y in 0..h {
+ for x in [0, w - 1] {
+ if !mask[y][x] && !outside[y * w + x] {
+ outside[y * w + x] = true;
+ queue.push_back(y * w + x);
+ }
+ }
+ }
+ while let Some(i) = queue.pop_front() {
+ let (x, y) = (i % w, i / w);
+ for (nx, ny) in neighbours(x, y, w, h) {
+ let j = ny * w + nx;
+ if !mask[ny][nx] && !outside[j] {
+ outside[j] = true;
+ queue.push_back(j);
+ }
+ }
+ }
+ for y in 0..h {
+ for x in 0..w {
+ if !mask[y][x] && !outside[y * w + x] {
+ mask[y][x] = true;
+ }
+ }
+ }
+}
+
+fn neighbours(x: usize, y: usize, w: usize, h: usize) -> impl Iterator<Item = (usize, usize)> {
+ [(x.wrapping_sub(1), y), (x + 1, y), (x, y.wrapping_sub(1)), (x, y + 1)].into_iter().filter(move |(nx, ny)| *nx < w && *ny < h)
+}
+
+fn frame(pixels: &[usize], w: usize) -> (usize, usize, usize, usize) {
+ let mut x0 = usize::MAX;
+ let mut y0 = usize::MAX;
+ let mut x1 = 0;
+ let mut y1 = 0;
+ for &i in pixels {
+ let (x, y) = (i % w, i / w);
+ x0 = x0.min(x);
+ y0 = y0.min(y);
+ x1 = x1.max(x);
+ y1 = y1.max(y);
+ }
+ (x0, y0, x1, y1)
+}
+
+fn fit_grid(w: usize, h: usize) -> (usize, usize) {
+ let scale = (MAX_GRID as f64 / w.max(h) as f64).min(1.0);
+ (((w as f64 * scale).round() as usize).max(2), ((h as f64 * scale).round() as usize).max(2))
+}
+
+/// Nearest-neighbour resample of a mask to (w1, h1).
+fn resample(mask: &[Vec<bool>], w0: usize, h0: usize, w1: usize, h1: usize) -> Vec<Vec<bool>> {
+ (0..h1)
+ .map(|y| {
+ let row = &mask[((y * h0) / h1.max(1)).min(h0 - 1)];
+ (0..w1).map(|x| row[((x * w0) / w1.max(1)).min(w0 - 1)]).collect()
+ })
+ .collect()
+}
+
+/// The model outline's size in picture pixels and the mm per pixel that follows. One
+/// uniform scale, taken from the dimension the two share — the axes are never stretched
+/// apart, which would hide the very mismatch the tool is for. `both` claims both dimensions
+/// and scales by width; the result's `aspect_mismatch` says how far the shapes are apart.
+/// The fitted size takes the same MAX_GRID cap as the picture's: a narrow model fitted by
+/// width against a wide picture would otherwise ask for hundreds of megabytes of mask.
+fn fit_model(model: &Silhouette, pw: usize, ph: usize, fit: Fit) -> (usize, usize, f64, f64) {
+ let s = match fit {
+ Fit::Width | Fit::Both => pw as f64 / model.width as f64,
+ Fit::Height => ph as f64 / model.height as f64,
+ };
+ let mw = ((model.width as f64 * s).round() as usize).max(2);
+ let mh = ((model.height as f64 * s).round() as usize).max(2);
+ let (mw, mh) = fit_grid(mw, mh);
+ (mw, mh, model.width as f64 * model.mm_per_px / mw as f64, model.height as f64 * model.mm_per_px / mh as f64)
+}
+
+struct Best {
+ overlap: f64,
+ dx: i64,
+ dy: i64,
+}
+
+/// Slide the model mask over the picture mask to the best intersection-over-union. The
+/// coarse pass covers the whole range of placements where the two frames can share a
+/// pixel — a tenth of the picture around the centre is not enough when the heights differ —
+/// and the fine pass walks every offset around its winner.
+fn best_overlap(model: &[Vec<bool>], picture: &Part) -> Best {
+ let pic = spans_of(&picture.mask);
+ let mdl = spans_of(model);
+ let model_area: i64 = mdl.iter().flatten().map(|[a, b]| (b - a) as i64).sum();
+ let pic_area: i64 = pic.iter().flatten().map(|[a, b]| (b - a) as i64).sum();
+
+ let (mw, mh) = (model[0].len() as i64, model.len() as i64);
+ let (lo_x, hi_x) = (1 - mw, picture.width as i64);
+ let (lo_y, hi_y) = (1 - mh, picture.height as i64);
+ // About 64 coarse samples per axis, never finer than 4 px, never coarser than 32.
+ let step_x = ((hi_x - lo_x) / 64).clamp(4, 32);
+ let step_y = ((hi_y - lo_y) / 64).clamp(4, 32);
+
+ // The frames' aligned corners: both masks are tightly framed, so the best placement of a
+ // matching picture is at or near here.
+ let (mut bx, mut by) = (0i64, 0i64);
+ for round in 0..2 {
+ // Coarse: the whole range at the step; fine: every offset within one coarse step
+ // of the coarse winner.
+ let (sx, sy, x0, x1, y0, y1) = if round == 0 {
+ (step_x, step_y, lo_x, hi_x, lo_y, hi_y)
+ } else {
+ (1, 1, (bx - step_x).max(lo_x), (bx + step_x).min(hi_x), (by - step_y).max(lo_y), (by + step_y).min(hi_y))
+ };
+ let (mut best_score, mut best_x, mut best_y) = (i64::MIN, bx, by);
+ let mut dy = y0;
+ while dy <= y1 {
+ let mut dx = x0;
+ while dx <= x1 {
+ let hits = intersection(&mdl, &pic, dx, dy);
+ let union = model_area + pic_area - hits;
+ let score = if union <= 0 { i64::MIN } else { hits * 1_000_000 / union };
+ if score > best_score {
+ best_score = score;
+ best_x = dx;
+ best_y = dy;
+ }
+ dx += sx;
+ }
+ dy += sy;
+ }
+ bx = best_x;
+ by = best_y;
+ }
+
+ let hits = intersection(&mdl, &pic, bx, by);
+ let union = model_area + pic_area - hits;
+ Best { overlap: if union <= 0 { 0.0 } else { hits as f64 / union as f64 }, dx: bx, dy: by }
+}
+
+fn spans_of(mask: &[Vec<bool>]) -> Vec<Vec<[usize; 2]>> {
+ mask.iter().map(|row| silhouette::spans(row)).collect()
+}
+
+/// Pixels the model shares with the picture at model offset (dx, dy), from run-length rows.
+fn intersection(mdl: &[Vec<[usize; 2]>], pic: &[Vec<[usize; 2]>], dx: i64, dy: i64) -> i64 {
+ let mut hits = 0i64;
+ for (r, spans) in mdl.iter().enumerate() {
+ if spans.is_empty() {
+ continue;
+ }
+ let Some(y) = (dy + r as i64).try_into().ok().and_then(|y: usize| pic.get(y)) else { continue };
+ if y.is_empty() {
+ continue;
+ }
+ for &[a, b] in spans {
+ let (mut lo, hi) = ((a as i64 + dx).max(0), b as i64 + dx);
+ if hi <= lo {
+ continue;
+ }
+ for &[pa, pb] in y {
+ let (s, e) = (lo.max(pa as i64), hi.min(pb as i64));
+ if e > s {
+ hits += e - s;
+ }
+ lo = lo.max(pb as i64);
+ if lo >= hi || pb as i64 >= hi {
+ break;
+ }
+ }
+ }
+ }
+ hits
+}
+
+/// The model's edge against the picture's, in mm, at nine stations along one side; positive
+/// where the model reaches past the picture, null where a station misses one of the two.
+fn edge_offsets(model: &Placed, picture: &Part, side: Edge, mmpp: f64) -> Vec<Value> {
+ (1..=STATIONS)
+ .map(|i| {
+ let station = match side {
+ Edge::Top | Edge::Bottom => (picture.width as u64 * i as u64 / (STATIONS as u64 + 1)) as usize,
+ Edge::Left | Edge::Right => (picture.height as u64 * i as u64 / (STATIONS as u64 + 1)) as usize,
+ };
+ let offset = match side {
+ Edge::Top | Edge::Bottom => {
+ let pic = column_edge(&picture.mask, station, matches!(side, Edge::Top));
+ let mdl = (station as i64 - model.dx)
+ .try_into()
+ .ok()
+ .and_then(|mx: usize| column_edge(model.mask, mx, matches!(side, Edge::Top)))
+ .map(|e| e as i64 + model.dy);
+ match (mdl, pic) {
+ (Some(m), Some(p)) => Some(match side {
+ Edge::Top => (p as f64 - m as f64) * mmpp,
+ _ => (m as f64 - p as f64) * mmpp,
+ }),
+ _ => None,
+ }
+ }
+ Edge::Left | Edge::Right => {
+ let pic = row_edge(&picture.mask, station, matches!(side, Edge::Left));
+ let mdl = (station as i64 - model.dy)
+ .try_into()
+ .ok()
+ .and_then(|my: usize| row_edge(model.mask, my, matches!(side, Edge::Left)))
+ .map(|e| e as i64 + model.dx);
+ match (mdl, pic) {
+ (Some(m), Some(p)) => Some(match side {
+ Edge::Left => (p as f64 - m as f64) * mmpp,
+ _ => (m as f64 - p as f64) * mmpp,
+ }),
+ _ => None,
+ }
+ }
+ };
+ offset.map(|mm| json!(round3(mm))).unwrap_or(Value::Null)
+ })
+ .collect()
+}
+
+/// The first (or last, when `first` is false) row holding column `x`; None if none does.
+fn column_edge(mask: &[Vec<bool>], x: usize, first: bool) -> Option<usize> {
+ let mut found = None;
+ for (y, row) in mask.iter().enumerate() {
+ if row.get(x).copied().unwrap_or(false) {
+ if first {
+ return Some(y);
+ }
+ found = Some(y);
+ }
+ }
+ found
+}
+
+/// The first (or last) column set in row `y`; None if the row is empty or missing.
+fn row_edge(mask: &[Vec<bool>], y: usize, first: bool) -> Option<usize> {
+ let row = mask.get(y)?;
+ if first { row.iter().position(|&on| on) } else { row.iter().rposition(|&on| on) }
+}
+
+/// The overlay PNG: the picture in blue, the model in red, on a canvas that holds both.
+fn overlay_png(picture: &Part, model: &Placed) -> Vec<u8> {
+ // A margin so a model that hangs off the picture still fits the canvas.
+ let m = 8i64;
+ let (x0, y0) = (0.min(model.dx) - m, 0.min(model.dy) - m);
+ let (x1, y1) = (
+ (picture.width as i64).max(model.dx + model.mask.first().map_or(0, |r| r.len()) as i64) + m,
+ (picture.height as i64).max(model.dy + model.mask.len() as i64) + m,
+ );
+ let (w, h) = ((x1 - x0).max(1) as u32, (y1 - y0).max(1) as u32);
+ let mut img = image::RgbaImage::from_pixel(w, h, image::Rgba([250, 250, 250, 255]));
+ for (y, row) in picture.mask.iter().enumerate() {
+ for (x, &on) in row.iter().enumerate() {
+ if on {
+ img.put_pixel((x as i64 - x0) as u32, (y as i64 - y0) as u32, image::Rgba([70, 110, 230, 255]));
+ }
+ }
+ }
+ for (r, row) in model.mask.iter().enumerate() {
+ for (c, &on) in row.iter().enumerate() {
+ if !on {
+ continue;
+ }
+ let (x, y) = (model.dx + c as i64 - x0, model.dy + r as i64 - y0);
+ if (0..w as i64).contains(&x) && (0..h as i64).contains(&y) {
+ img.put_pixel(x as u32, y as u32, image::Rgba([235, 60, 60, 255]));
+ }
+ }
+ }
+ let mut out = Cursor::new(Vec::new());
+ // Encoding a mask-sized PNG cannot realistically fail; if it ever does, no overlay.
+ let enc = image::codecs::png::PngEncoder::new(&mut out);
+ use image::ImageEncoder as _;
+ if enc.write_image(img.as_raw(), img.width(), img.height(), image::ExtendedColorType::Rgba8).is_err() {
+ return Vec::new();
+ }
+ out.into_inner()
+}
+
+fn round4(v: f64) -> f64 {
+ if v.abs() < 1e12 { (v * 10_000.0).round() / 10_000.0 } else { v }
+}
+
+fn round3(v: f64) -> f64 {
+ if v.abs() < 1e12 { (v * 1_000.0).round() / 1_000.0 } else { v }
+}
diff --git a/crates/engine/src/cmd/compare_picture_tests.rs b/crates/engine/src/cmd/compare_picture_tests.rs
new file mode 100644
index 0000000000..fe8f443902
--- /dev/null
+++ b/crates/engine/src/cmd/compare_picture_tests.rs
@@ -0,0 +1,490 @@
+//! Tests for `inspect.compare_picture` (#1486): synthetic reference pictures drawn in code,
+//! written to the temp dir and read back through the vfs like any other reference.
+
+use std::io::Cursor;
+
+use serde_json::{Value, json};
+
+use super::super::Session;
+use image::ImageEncoder as _;
+
+fn run(s: &mut Session, cmd: &str, params: Value) -> Value {
+ s.execute(cmd, &params).unwrap_or_else(|e| panic!("{cmd}: {e}"))
+}
+
+fn box_body(s: &mut Session, name: &str, x1: f64, y1: f64, z: f64) {
+ let sketch = format!("{name}Sketch");
+ run(s, "sketch.create", json!({"plane": "XY", "name": sketch}));
+ run(s, "sketch.rectangle.two_point", json!({"p0": [0.0, 0.0], "p1": [x1, y1]}));
+ run(s, "sketch.finish", json!({}));
+ run(s, "solid.extrude", json!({"sketch": sketch, "distance": z, "operation": "new_body", "body_name": name}));
+}
+
+/// A PNG whose pixels come from `paint`.
+fn png(w: usize, h: usize, paint: impl Fn(usize, usize) -> [u8; 3]) -> Vec<u8> {
+ png_rgba(w, h, |x, y| {
+ let [r, g, b] = paint(x, y);
+ [r, g, b, 255]
+ })
+}
+
+/// A PNG with alpha, whose pixels come from `paint`.
+fn png_rgba(w: usize, h: usize, paint: impl Fn(usize, usize) -> [u8; 4]) -> Vec<u8> {
+ let mut img = image::RgbaImage::new(w as u32, h as u32);
+ for y in 0..h {
+ for x in 0..w {
+ img.put_pixel(x as u32, y as u32, image::Rgba(paint(x, y)));
+ }
+ }
+ let mut out = Cursor::new(Vec::new());
+ image::codecs::png::PngEncoder::new(&mut out).write_image(img.as_raw(), w as u32, h as u32, image::ExtendedColorType::Rgba8).unwrap();
+ out.into_inner()
+}
+
+fn jpeg(w: usize, h: usize, paint: impl Fn(usize, usize) -> [u8; 3]) -> Vec<u8> {
+ jpeg_q(w, h, 90, paint)
+}
+
+fn jpeg_q(w: usize, h: usize, q: u8, paint: impl Fn(usize, usize) -> [u8; 3]) -> Vec<u8> {
+ let mut img = image::RgbaImage::new(w as u32, h as u32);
+ for y in 0..h {
+ for x in 0..w {
+ let [r, g, b] = paint(x, y);
+ img.put_pixel(x as u32, y as u32, image::Rgba([r, g, b, 255]));
+ }
+ }
+ let rgb = image::DynamicImage::ImageRgba8(img).to_rgb8();
+ let mut out = Cursor::new(Vec::new());
+ image::codecs::jpeg::JpegEncoder::new_with_quality(&mut out, q)
+ .write_image(rgb.as_raw(), w as u32, h as u32, image::ExtendedColorType::Rgb8)
+ .unwrap();
+ out.into_inner()
+}
+
+/// Write a reference into the temp dir and return its path.
+fn reference(name: &str, bytes: &[u8]) -> String {
+ let path = std::env::temp_dir().join(format!("solidcraft-compare-picture-{}-{name}", std::process::id()));
+ std::fs::write(&path, bytes).unwrap();
+ path.to_string_lossy().into_owned()
+}
+
+/// A white rectangle on a grey backdrop: `x0..x1, y0..y1` in pixels.
+fn rect_picture(w: usize, h: usize, x0: usize, y0: usize, x1: usize, y1: usize) -> Vec<u8> {
+ png(w, h, |x, y| if (x0..x1).contains(&x) && (y0..y1).contains(&y) { [255, 255, 255] } else { [200, 200, 200] })
+}
+
+fn edges(v: &Value) -> Vec<Vec<Option<f64>>> {
+ ["top", "bottom", "left", "right"]
+ .iter()
+ .map(|side| v["edges"][side].as_array().unwrap().iter().map(|e| e.as_f64()).collect::<Vec<Option<f64>>>())
+ .collect()
+}
+
+fn near(v: Option<f64>, want: f64, tol: f64) -> bool {
+ v.is_some_and(|e| (e - want).abs() <= tol)
+}
+
+#[test]
+fn a_matching_picture_scores_high_with_flat_edges() {
+ let mut s = Session::default();
+ box_body(&mut s, "Box", 20.0, 10.0, 30.0); // front view: 20 wide, 30 tall
+ let path = reference("match.png", &rect_picture(200, 280, 20, 20, 180, 260));
+ let out = run(&mut s, "inspect.compare_picture", json!({"reference": path, "view": "front"}));
+ assert!(out["overlap"].as_f64().unwrap() > 0.95, "overlap {}", out["overlap"]);
+ for side in edges(&out) {
+ for e in side {
+ assert!(near(e, 0.0, 0.6), "edge {e:?}");
+ }
+ }
+ assert!(!out["overlay_png"].as_str().unwrap().is_empty());
+}
+
+#[test]
+fn a_step_in_the_bottom_edge_shows_up_in_mm() {
+ let mut s = Session::default();
+ box_body(&mut s, "Box", 20.0, 10.0, 30.0);
+ // The aligned picture: a plain part 160 x 240 px — the model's own shape at 8 px per mm.
+ let aligned_path = reference("aligned.png", &rect_picture(200, 280, 20, 20, 180, 260));
+ let aligned = run(&mut s, "inspect.compare_picture", json!({"reference": aligned_path, "view": "front"}));
+ // The stepped picture: over its right half the part's bottom edge sits 35 px higher.
+ // The width fit gives 8 px per mm, so 35 px is 4.375 mm of the model reaching past
+ // the picture.
+ let stepped_path = reference(
+ "stepped.png",
+ &png(200, 280, |x, y| {
+ let bottom = if x >= 100 { 225 } else { 260 };
+ if (20..180).contains(&x) && (20..bottom).contains(&y) { [255, 255, 255] } else { [200, 200, 200] }
+ }),
+ );
+ let stepped = run(&mut s, "inspect.compare_picture", json!({"reference": stepped_path, "view": "front"}));
+ let flat = edges(&aligned)[1].clone();
+ let step = edges(&stepped)[1].clone();
+ for e in flat.iter() {
+ assert!(near(*e, 0.0, 0.6), "flat bottom {e:?}");
+ }
+ // Stations sit at local x = 16..144 px; the notch starts at local x = 80 (global 100),
+ // so stations 5..9 are over it.
+ for e in step.iter().take(4) {
+ assert!(near(*e, 0.0, 0.6), "step bottom left {e:?}");
+ }
+ for e in step.iter().skip(4) {
+ assert!(near(*e, 4.375, 0.6), "step bottom right {e:?}");
+ }
+ assert!(stepped["overlap"].as_f64().unwrap() < aligned["overlap"].as_f64().unwrap());
+}
+
+/// #1486's example, pinned: `examples/bracket.json` as it is (upright 30 mm) against the
+/// same bracket at 40 mm, both from the front, scaled by the 40 mm arm they share.
+/// The issue's prototype numbers: overlap 0.865, bottom −9.9 mm at the upright (6 % across),
+/// 0.0 mm everywhere else.
+#[test]
+fn the_issue_bracket_pins_overlap_and_the_short_upright() {
+ let mut s = Session::default();
+ // The bracket profile of examples/bracket.json: a 40 x 6 arm with a 6-wide upright to
+ // 30 mm, on XZ so the front view shows the upright hanging below the arm.
+ run(&mut s, "sketch.create", json!({"plane": "XZ", "name": "Profile"}));
+ run(&mut s, "sketch.line", json!({"points": [[0, 0], [40, 0], [40, 6], [6, 6], [6, 30], [0, 30]], "closed": true}));
+ run(&mut s, "sketch.finish", json!({}));
+ run(&mut s, "solid.extrude", json!({"distance": 20.0, "direction": "negative", "body_name": "Bracket"}));
+ // The same bracket with the upright at 40 mm, at 20 px per mm: an 800 x 800 part, the
+ // 800 x 120 arm on top and the 120-wide upright down to 800, on a grey margin.
+ let path = reference(
+ "bracket.png",
+ &png(840, 840, |x, y| {
+ let part = ((20..820).contains(&x) && (20..140).contains(&y)) || ((20..140).contains(&x) && (20..820).contains(&y));
+ if part { [255, 255, 255] } else { [200, 200, 200] }
+ }),
+ );
+ let out = run(&mut s, "inspect.compare_picture", json!({"reference": path, "view": "front", "rows": 480, "overlay": false}));
+ let overlap = out["overlap"].as_f64().unwrap();
+ assert!((overlap - 0.865).abs() <= 0.01, "overlap {overlap}");
+ let e = edges(&out);
+ // The upright is the first station (x = 80 of 800): the model is 10 mm short there.
+ assert!(near(e[1][0], -10.0, 0.2), "bottom at the upright {:?}", e[1][0]);
+ for side in 1..9 {
+ assert!(near(e[1][side], 0.0, 0.1), "bottom elsewhere {:?}", e[1]);
+ }
+ for i in 0..9 {
+ assert!(near(e[0][i], 0.0, 0.1), "top {:?}", e[0]);
+ // Below the model's 30 mm upright the model has no row to answer with: those
+ // side stations are null, as documented.
+ for side in [e[2].clone(), e[3].clone()] {
+ match side[i] {
+ None => {}
+ Some(v) => assert!(v.abs() <= 0.1, "side {side:?}"),
+ }
+ }
+ }
+}
+
+#[test]
+fn a_gradient_backdrop_still_separates_from_the_part() {
+ let mut s = Session::default();
+ box_body(&mut s, "Box", 20.0, 10.0, 30.0);
+ // A backdrop that shifts one step per row: the border flood follows it; the step at the
+ // part's outline (to white) stops it.
+ let path = reference(
+ "gradient.png",
+ &png(200, 300, |x, y| if (20..180).contains(&x) && (20..280).contains(&y) { [255, 255, 255] } else { [200 + (y / 30) as u8, 200, 200] }),
+ );
+ let out = run(&mut s, "inspect.compare_picture", json!({"reference": path, "view": "front"}));
+ assert!(out["overlap"].as_f64().unwrap() > 0.9, "overlap {}", out["overlap"]);
+}
+
+#[test]
+fn a_jpeg_reference_works_too() {
+ let mut s = Session::default();
+ box_body(&mut s, "Box", 20.0, 10.0, 30.0);
+ let path =
+ reference("match.jpg", &jpeg(200, 300, |x, y| if (20..180).contains(&x) && (20..280).contains(&y) { [255, 255, 255] } else { [40, 40, 40] }));
+ let out = run(&mut s, "inspect.compare_picture", json!({"reference": path, "view": "front"}));
+ assert!(out["overlap"].as_f64().unwrap() > 0.9, "overlap {}", out["overlap"]);
+}
+
+#[test]
+fn crop_takes_one_part_off_the_board() {
+ let mut s = Session::default();
+ box_body(&mut s, "Box", 20.0, 10.0, 30.0);
+ // Two parts side by side; the crop keeps the left one, a plain rectangle that matches
+ // the model. The right part is bigger overall but L-shaped, so without the crop the
+ // largest piece wins and the score drops.
+ let path = reference(
+ "two.png",
+ &png(400, 300, |x, y| {
+ let left = (20..180).contains(&x) && (20..260).contains(&y);
+ let right = (210..390).contains(&x) && (5..295).contains(&y) && !((300..390).contains(&x) && (100..200).contains(&y));
+ if left || right { [255, 255, 255] } else { [200, 200, 200] }
+ }),
+ );
+ let out = run(&mut s, "inspect.compare_picture", json!({"reference": path.clone(), "view": "front", "crop_fraction": [0.0, 0.0, 0.5, 1.0]}));
+ // The picture size is the found part's tight frame: the left rectangle is 160 x 240 px.
+ assert_eq!(out["picture"]["width"], 160);
+ assert!(out["overlap"].as_f64().unwrap() > 0.95, "overlap {}", out["overlap"]);
+ let whole = run(&mut s, "inspect.compare_picture", json!({"reference": path, "view": "front"}));
+ assert!(whole["overlap"].as_f64().unwrap() < out["overlap"].as_f64().unwrap());
+}
+
+#[test]
+fn crop_px_has_explicit_units() {
+ let mut s = Session::default();
+ box_body(&mut s, "Box", 20.0, 10.0, 30.0);
+ let path = reference("px.png", &rect_picture(200, 280, 20, 20, 180, 260));
+ // Pixels mean pixels: [10, 10, 180, 260] keeps the part with a margin, and a crop of
+ // one pixel is one pixel, not the whole picture.
+ let out = run(&mut s, "inspect.compare_picture", json!({"reference": path.clone(), "view": "front", "crop_px": [10, 10, 180, 260]}));
+ assert!(out["overlap"].as_f64().unwrap() > 0.95, "overlap {}", out["overlap"]);
+ let dot = reference("dot.png", &rect_picture(200, 280, 100, 100, 101, 101));
+ assert!(
+ s.execute("inspect.compare_picture", &json!({"reference": dot, "crop_px": [0, 0, 1, 1]})).unwrap_err().to_string().contains("no part found")
+ );
+ // Giving both spellings is refused, and so is a fraction above 1.
+ assert!(
+ s.execute("inspect.compare_picture", &json!({"reference": path, "crop_px": [0, 0, 10, 10], "crop_fraction": [0.0, 0.0, 0.1, 0.1]}))
+ .unwrap_err()
+ .to_string()
+ .contains("not both")
+ );
+ assert!(
+ s.execute("inspect.compare_picture", &json!({"reference": path, "crop_fraction": [0.0, 0.0, 1.5, 1.0]}))
+ .unwrap_err()
+ .to_string()
+ .contains("0..1")
+ );
+}
+
+#[test]
+fn fit_both_never_stretches_the_axes_apart() {
+ let mut s = Session::default();
+ box_body(&mut s, "Box", 20.0, 10.0, 30.0); // front view: 20 wide, 30 tall
+ // A 20 x 20 mm square picture of a 20 x 30 mm model: `both` used to stretch the model
+ // to the square and report a perfect 1.000 with every edge at 0.0 mm. One uniform
+ // scale (from the width they share) leaves the 10 mm of overhang visible instead.
+ let path = reference("square.png", &rect_picture(220, 220, 10, 10, 210, 210));
+ let both = run(&mut s, "inspect.compare_picture", json!({"reference": path.clone(), "view": "front", "fit": "both", "overlay": false}));
+ assert!((both["overlap"].as_f64().unwrap() - 200.0 / 300.0).abs() < 0.02, "overlap {}", both["overlap"]);
+ assert!((both["aspect_mismatch"].as_f64().unwrap() - 1.5).abs() < 0.01, "aspect_mismatch {}", both["aspect_mismatch"]);
+ // And `width` is the default: leaving `fit` out answers the same as naming it.
+ let plain = run(&mut s, "inspect.compare_picture", json!({"reference": path, "view": "front", "overlay": false}));
+ assert_eq!(plain["overlap"], both["overlap"]);
+ assert_eq!(plain["edges"], both["edges"]);
+}
+
+#[test]
+fn fit_width_keeps_the_models_aspect() {
+ let mut s = Session::default();
+ box_body(&mut s, "Box", 20.0, 10.0, 30.0);
+ // The picture part is 160 x 160 px: fit=width scales the model uniformly to 160 x 240,
+ // so it hangs 80 px (10 mm) over the picture's height in total, wherever the slide puts
+ // it. The overlap of a contained model is 160/240 wherever it sits.
+ let path = reference("short.png", &rect_picture(200, 220, 20, 20, 180, 180));
+ let out = run(&mut s, "inspect.compare_picture", json!({"reference": path, "view": "front", "fit": "width"}));
+ assert!((out["overlap"].as_f64().unwrap() - 160.0 / 240.0).abs() < 0.05, "overlap {}", out["overlap"]);
+ let bottom: Vec<Option<f64>> = out["edges"]["bottom"].as_array().unwrap().iter().map(|e| e.as_f64()).collect();
+ let top: Vec<Option<f64>> = out["edges"]["top"].as_array().unwrap().iter().map(|e| e.as_f64()).collect();
+ for i in 0..9 {
+ // The overhang splits between top and bottom but always totals the extra 10 mm.
+ assert!(bottom[i].unwrap() >= 0.0 && top[i].unwrap() >= 0.0, "bottom {:?} top {:?}", bottom[i], top[i]);
+ assert!((bottom[i].unwrap() + top[i].unwrap() - 10.0).abs() < 0.6, "bottom {:?} top {:?}", bottom[i], top[i]);
+ }
+}
+
+#[test]
+fn a_narrow_model_fits_the_grid_not_the_memory() {
+ let mut s = Session::default();
+ // A 2 mm x 1000 mm sliver seen from the front: fitted by width against a 1024-wide
+ // picture it used to become 1024 x 524 288 booleans. The fitted model now takes the
+ // same MAX_GRID cap as the picture, so the comparison just runs.
+ box_body(&mut s, "Sliver", 2.0, 1.0, 1000.0);
+ let path = reference("sliver.png", &rect_picture(1030, 1010, 3, 5, 1027, 1005));
+ let out = run(&mut s, "inspect.compare_picture", json!({"reference": path, "view": "front", "overlay": false}));
+ // The shapes are nothing alike (the sliver is 1:500, the picture 1024:1000), and the
+ // capped model grid keeps the overlap small — what matters is that it ran at all.
+ assert!(out["overlap"].as_f64().unwrap() < 0.1, "overlap {}", out["overlap"]);
+}
+
+#[test]
+fn ordinary_pictures_keep_their_part() {
+ let mut s = Session::default();
+ box_body(&mut s, "Box", 20.0, 10.0, 30.0);
+ // A part touching the picture's border: only border pixels close to the border's main
+ // colour seed the background, so the part's own border pixels do not flood it.
+ let border = reference("border.png", &rect_picture(200, 280, 0, 20, 160, 260));
+ let out = run(&mut s, "inspect.compare_picture", json!({"reference": border, "view": "front"}));
+ assert!(out["overlap"].as_f64().unwrap() > 0.95, "border overlap {}", out["overlap"]);
+ // A white part on a (245, 245, 245) studio backdrop: the step of 10 is below BG_STEP,
+ // but the stricter retry separates it.
+ let studio = reference(
+ "studio.png",
+ &png(200, 280, |x, y| if (20..180).contains(&x) && (20..260).contains(&y) { [255, 255, 255] } else { [245, 245, 245] }),
+ );
+ let out = run(&mut s, "inspect.compare_picture", json!({"reference": studio, "view": "front"}));
+ assert!(out["overlap"].as_f64().unwrap() > 0.95, "studio overlap {}", out["overlap"]);
+ // A part on a transparent background: alpha below 128 is background.
+ let alpha = reference(
+ "alpha.png",
+ &png_rgba(200, 280, |x, y| if (20..180).contains(&x) && (20..260).contains(&y) { [90, 90, 90, 255] } else { [0, 0, 0, 0] }),
+ );
+ let out = run(&mut s, "inspect.compare_picture", json!({"reference": alpha, "view": "front"}));
+ assert!(out["overlap"].as_f64().unwrap() > 0.95, "alpha overlap {}", out["overlap"]);
+}
+
+#[test]
+fn a_through_hole_counts_the_same_on_both_sides() {
+ let mut s = Session::default();
+ // A 200 x 200 mm plate with a 100 x 100 mm through-hole, against a picture of itself
+ // at 1 px per mm, seen from the top (the hole goes through the plate's faces). The
+ // picture's hole is filled by the border flood; the model's hole now gets the same
+ // rule, so the plate scores ~1.0 instead of 0.75.
+ run(&mut s, "sketch.create", json!({"plane": "XY", "name": "Plate"}));
+ run(&mut s, "sketch.rectangle.two_point", json!({"p0": [0.0, 0.0], "p1": [200.0, 200.0]}));
+ run(&mut s, "sketch.finish", json!({}));
+ run(&mut s, "solid.extrude", json!({"sketch": "Plate", "distance": 10.0, "operation": "new_body", "body_name": "Plate"}));
+ run(&mut s, "sketch.create", json!({"plane": "XY", "name": "Hole"}));
+ run(&mut s, "sketch.rectangle.two_point", json!({"p0": [50.0, 50.0], "p1": [150.0, 150.0]}));
+ run(&mut s, "sketch.finish", json!({}));
+ run(&mut s, "solid.extrude", json!({"sketch": "Hole", "distance": 20.0, "operation": "cut"}));
+ let path = reference(
+ "plate.png",
+ &png(220, 220, |x, y| {
+ let plate = (10..210).contains(&x) && (10..210).contains(&y);
+ let hole = (60..160).contains(&x) && (60..160).contains(&y);
+ if plate && !hole { [255, 255, 255] } else { [200, 200, 200] }
+ }),
+ );
+ let out = run(&mut s, "inspect.compare_picture", json!({"reference": path, "view": "top", "overlay": false}));
+ assert!(out["overlap"].as_f64().unwrap() > 0.98, "overlap {}", out["overlap"]);
+ for side in edges(&out) {
+ for e in side {
+ assert!(near(e, 0.0, 0.6), "edge {e:?}");
+ }
+ }
+}
+
+#[test]
+fn hostile_and_empty_references_fail_cleanly() {
+ let mut s = Session::default();
+ box_body(&mut s, "Box", 20.0, 10.0, 30.0);
+ assert!(s.execute("inspect.compare_picture", &json!({})).is_err());
+ // Not an image at all.
+ let text = reference("not.png", b"just text, not a picture");
+ assert!(s.execute("inspect.compare_picture", &json!({"reference": text})).unwrap_err().to_string().contains("PNG or a JPEG"));
+ // A uniform picture: the background eats everything, and says so.
+ let flat = reference("flat.png", &png(64, 64, |_, _| [200, 200, 200]));
+ assert!(s.execute("inspect.compare_picture", &json!({"reference": flat})).unwrap_err().to_string().contains("no part found"));
+ // A fully transparent picture: alpha says there is nothing there.
+ let clear = reference("clear.png", &png_rgba(64, 64, |_, _| [0, 0, 0, 0]));
+ assert!(s.execute("inspect.compare_picture", &json!({"reference": clear})).unwrap_err().to_string().contains("transparent"));
+ // A missing file.
+ assert!(s.execute("inspect.compare_picture", &json!({"reference": "no-such-file.png"})).is_err());
+ // A bad fit name.
+ let path = reference("match2.png", &rect_picture(200, 280, 20, 20, 180, 260));
+ assert!(s.execute("inspect.compare_picture", &json!({"reference": path, "fit": "diagonal"})).is_err());
+ // overlay: false leaves the PNG out.
+ let out = run(&mut s, "inspect.compare_picture", json!({"reference": path, "view": "front", "overlay": false}));
+ assert!(out.get("overlay_png").is_none());
+}
+
+/// Reviewer re-test case: a 60 x 60 opaque PNG with a white 40 x 40 square and a single
+/// backdrop pixel at alpha 254 used to be read as all part — 1.000 against the 40 x 40
+/// square at 0.067 mm/px instead of 0.1. Alpha now decides only when the border is mostly
+/// see-through, and a part drawn at alpha 100 on a transparent backdrop is still the part.
+#[test]
+fn one_translucent_pixel_does_not_switch_to_alpha() {
+ let mut s = Session::default();
+ box_body(&mut s, "Square", 4.0, 1.0, 4.0); // front view: 4 x 4 mm
+ let path = reference(
+ "speck.png",
+ &png_rgba(60, 60, |x, y| {
+ let alpha = if x == 5 && y == 55 { 254 } else { 255 };
+ let c = if (10..50).contains(&x) && (10..50).contains(&y) { 255u8 } else { 128 };
+ [c, c, c, alpha]
+ }),
+ );
+ let out = run(&mut s, "inspect.compare_picture", json!({"reference": path, "view": "front", "overlay": false}));
+ // The part is the 40 x 40 square: 4 mm over 40 px is 0.1 mm/px, not 4/60 = 0.067.
+ let mmpp = out["mm_per_px"]["x"].as_f64().unwrap();
+ assert!((mmpp - 0.1).abs() < 0.01, "mm_per_px {mmpp}");
+ assert!(out["overlap"].as_f64().unwrap() > 0.9, "overlap {}", out["overlap"]);
+ // A part drawn at alpha 100 on a transparent backdrop is the part, not the background.
+ let faint = reference(
+ "faint.png",
+ &png_rgba(60, 60, |x, y| if (10..50).contains(&x) && (10..50).contains(&y) { [255, 255, 255, 100] } else { [0, 0, 0, 0] }),
+ );
+ let out = run(&mut s, "inspect.compare_picture", json!({"reference": faint, "view": "front", "overlay": false}));
+ assert!(out["overlap"].as_f64().unwrap() > 0.9, "faint overlap {}", out["overlap"]);
+ let mmpp = out["mm_per_px"]["x"].as_f64().unwrap();
+ assert!((mmpp - 0.1).abs() < 0.01, "faint mm_per_px {mmpp}");
+}
+
+/// Reviewer re-test case: one view cropped off a board — the part touches three sides and
+/// takes 270 of the 400 border samples, so the border's main colour is the part's own and
+/// the flood eats it: the 15 px backdrop strip becomes "the part" and an exact model scores
+/// 0.18. `part_fills_crop: true` says the crop box is the part: no flood, exact model ~1.0.
+#[test]
+fn a_tight_crop_can_declare_the_box_is_the_part() {
+ let mut s = Session::default();
+ box_body(&mut s, "Box", 17.0, 10.0, 20.0); // front view: 17 wide, 20 tall
+ // The tight crop: the white part fills x 0..85 of a 100 x 100 box, touching three
+ // sides — 270 of the border samples — with a 15 px backdrop strip down the right.
+ let path = reference("tight.png", &png(100, 100, |x, _| if x < 85 { [255, 255, 255] } else { [200, 200, 200] }));
+ let flood = run(&mut s, "inspect.compare_picture", json!({"reference": path.clone(), "view": "front", "overlay": false}));
+ assert!(flood["overlap"].as_f64().unwrap() < 0.5, "the flood keeps the strip: overlap {}", flood["overlap"]);
+ let out = run(
+ &mut s,
+ "inspect.compare_picture",
+ json!({"reference": path, "view": "front", "crop_px": [0, 0, 85, 100], "part_fills_crop": true, "overlay": false}),
+ );
+ assert!(out["overlap"].as_f64().unwrap() > 0.95, "overlap {}", out["overlap"]);
+ // 17 mm over the 85 px the part fills: 0.2 mm/px.
+ assert!((out["mm_per_px"]["x"].as_f64().unwrap() - 0.2).abs() < 0.01, "mm_per_px {}", out["mm_per_px"]);
+ // A crop of a 2 x 2 box is the part; a 1 px box is refused.
+ let dot = reference("tight-dot.png", &png(4, 4, |_, _| [255, 255, 255]));
+ assert!(
+ s.execute("inspect.compare_picture", &json!({"reference": dot, "part_fills_crop": true, "crop_px": [0, 0, 1, 1]}))
+ .unwrap_err()
+ .to_string()
+ .contains("2 x 2")
+ );
+}
+
+/// Reviewer re-test case: holes are filled on both sides, so a model missing a 100 x 100
+/// through-hole scores ~1.000 against a picture that shows the hole. That is the contract —
+/// outline only — and the command description must say so, so an agent does not read 1.000
+/// as "matches".
+#[test]
+fn the_compare_is_outline_only_holes_are_not_compared() {
+ assert!(
+ crate::cmd::compare_picture::COMMANDS[0].params.contains("holes are not compared"),
+ "the command description must say the compare is outline only"
+ );
+ let mut s = Session::default();
+ // A solid 200 x 200 mm plate against a picture of a plate with a 100 x 100 through-hole.
+ box_body(&mut s, "Plate", 200.0, 10.0, 200.0);
+ let path = reference(
+ "hole-only.png",
+ &png(220, 220, |x, y| {
+ let plate = (10..210).contains(&x) && (10..210).contains(&y);
+ let hole = (60..160).contains(&x) && (60..160).contains(&y);
+ if plate && !hole { [255, 255, 255] } else { [200, 200, 200] }
+ }),
+ );
+ let out = run(&mut s, "inspect.compare_picture", json!({"reference": path, "view": "front", "overlay": false}));
+ // The outlines match: the hole is not compared, as the description says.
+ assert!(out["overlap"].as_f64().unwrap() > 0.98, "overlap {}", out["overlap"]);
+}
+
+/// Reviewer re-test case: a white part on (245, 245, 245) saved as JPEG at quality 75 gave
+/// "no part" even on the stricter retry — the codec blurs the 10-level step at the part's
+/// outline into steps of a few levels each, under any local limit. The third pass takes the
+/// backdrop globally: near the border's main colour.
+#[test]
+fn a_light_jpeg_keeps_its_part() {
+ let mut s = Session::default();
+ box_body(&mut s, "Box", 20.0, 10.0, 30.0);
+ let path = reference(
+ "light.jpg",
+ &jpeg_q(200, 300, 75, |x, y| if (20..180).contains(&x) && (20..280).contains(&y) { [255, 255, 255] } else { [245, 245, 245] }),
+ );
+ let out = run(&mut s, "inspect.compare_picture", json!({"reference": path, "view": "front", "overlay": false}));
+ assert!(out["overlap"].as_f64().unwrap() > 0.9, "overlap {}", out["overlap"]);
+}
diff --git a/crates/engine/src/cmd/mod.rs b/crates/engine/src/cmd/mod.rs
index ca96a61991..c76ed7bd87 100644
--- a/crates/engine/src/cmd/mod.rs
+++ b/crates/engine/src/cmd/mod.rs
@@ -26,6 +26,9 @@ pub(crate) mod clearance;
mod clipboard;
mod clipboard_foreign;
mod compare;
+mod compare_picture;
+#[cfg(test)]
+mod compare_picture_tests;
mod component;
mod component_clone;
mod component_clone_assembly;
@@ -346,6 +349,7 @@ pub fn command_specs() -> Vec<&'static CommandSpec> {
v.extend(metadata::COMMANDS.iter());
v.extend(inspect::COMMANDS.iter());
v.extend(compare::COMMANDS.iter());
+ v.extend(compare_picture::COMMANDS.iter());
v.extend(silhouette::COMMANDS.iter());
v.extend(materials::COMMANDS.iter());
v.extend(material_properties::COMMANDS.iter());
diff --git a/crates/engine/src/cmd/silhouette.rs b/crates/engine/src/cmd/silhouette.rs
index f44a9f02ff..17b3f9b0a7 100644
--- a/crates/engine/src/cmd/silhouette.rs
+++ b/crates/engine/src/cmd/silhouette.rs
@@ -20,19 +20,50 @@ fn run(s: &mut Session, p: &Value) -> Result<Value> {
let camera = camera(p)?;
let rows = rows(p)?;
let wanted = body_names(p)?;
+ let out = outline(s, &camera, wanted.as_deref(), rows)?;
+ let mask: Vec<Vec<[usize; 2]>> = out.mask.iter().map(|row| spans(row)).collect();
+ Ok(json!({
+ "width": out.width,
+ "height": out.height,
+ "mm_per_px": round4(out.mm_per_px),
+ "box_mm": [[round4(out.lo[0]), round4(out.lo[1])], [round4(out.hi[0]), round4(out.hi[1])]],
+ "right": [clean(out.right.x), clean(out.right.y), clean(out.right.z)],
+ "up": [clean(out.up.x), clean(out.up.y), clean(out.up.z)],
+ "projection": if out.perspective { "perspective" } else { "parallel" },
+ "bodies": out.bodies,
+ "triangles": out.triangles,
+ "mask": mask,
+ }))
+}
+
+/// A projected, tightly framed silhouette: the pixel grid, its scale and placement, and the
+/// filled mask as one row of booleans per pixel row. `inspect.silhouette` renders this as
+/// run-length rows; `inspect.compare_picture` uses the filled rows directly.
+pub(crate) struct Silhouette {
+ pub width: usize,
+ pub height: usize,
+ pub mm_per_px: f64,
+ /// The view-plane box the mask frames, in mm (u along `right`, v along `up`).
+ pub lo: [f64; 2],
+ pub hi: [f64; 2],
+ pub right: Vec3,
+ pub up: Vec3,
+ pub perspective: bool,
+ pub bodies: usize,
+ pub triangles: usize,
+ pub mask: Vec<Vec<bool>>,
+}
+
+pub(crate) fn outline(s: &Session, camera: &Camera, wanted: Option<&[String]>, rows: usize) -> Result<Silhouette> {
let state = s.world_state();
- if let Some(names) = &wanted {
+ if let Some(names) = wanted {
for name in names {
if !state.bodies.iter().any(|b| b.name == *name) {
return Err(bad(CMD, format!("unknown body name `{name}`")));
}
}
}
- let bodies: Vec<_> = state
- .bodies
- .iter()
- .filter(|b| wanted.as_ref().map_or_else(|| s.body_visible(&b.name), |names| names.iter().any(|name| name == &b.name)))
- .collect();
+ let bodies: Vec<_> = state.bodies.iter().filter(|b| wanted.map_or_else(|| s.body_visible(&b.name), |names| names.contains(&b.name))).collect();
if bodies.is_empty() {
return Err(bad(CMD, "nothing visible to outline"));
}
@@ -133,22 +164,22 @@ fn run(s: &mut Session, p: &Value) -> Result<Value> {
}
}
- let mask: Vec<Vec<[usize; 2]>> = covered.chunks_exact(width).map(spans).collect();
- Ok(json!({
- "width": width,
- "height": height,
- "mm_per_px": round4(mm_per_px),
- "box_mm": [[round4(lo[0]), round4(lo[1])], [round4(hi[0]), round4(hi[1])]],
- "right": [clean(right.x), clean(right.y), clean(right.z)],
- "up": [clean(up.x), clean(up.y), clean(up.z)],
- "projection": if perspective { "perspective" } else { "parallel" },
- "bodies": bodies.len(),
- "triangles": triangle_count,
- "mask": mask,
- }))
+ Ok(Silhouette {
+ width,
+ height,
+ mm_per_px,
+ lo,
+ hi,
+ right,
+ up,
+ perspective,
+ bodies: bodies.len(),
+ triangles: triangle_count,
+ mask: covered.chunks_exact(width).map(|row| row.to_vec()).collect(),
+ })
}
-fn camera(p: &Value) -> Result<Camera> {
+pub(crate) fn camera(p: &Value) -> Result<Camera> {
if p.get("view").is_some() && p.get("camera").is_some() {
return Err(bad(CMD, "give `view` or `camera`, not both"));
}
@@ -171,7 +202,7 @@ fn camera(p: &Value) -> Result<Camera> {
Ok(camera)
}
-fn body_names(p: &Value) -> Result<Option<Vec<String>>> {
+pub(crate) fn body_names(p: &Value) -> Result<Option<Vec<String>>> {
match p.get("bodies") {
None => Ok(None),
Some(Value::String(name)) => Ok(Some(vec![name.clone()])),
@@ -184,7 +215,7 @@ fn body_names(p: &Value) -> Result<Option<Vec<String>>> {
}
}
-fn rows(p: &Value) -> Result<usize> {
+pub(crate) fn rows(p: &Value) -> Result<usize> {
match p.get("rows") {
None => Ok(240),
Some(v) => {
@@ -209,7 +240,7 @@ fn edge(a: [f64; 2], b: [f64; 2], p: [f64; 2]) -> f64 {
(b[0] - a[0]) * (p[1] - a[1]) - (b[1] - a[1]) * (p[0] - a[0])
}
-fn spans(row: &[bool]) -> Vec<[usize; 2]> {
+pub(crate) fn spans(row: &[bool]) -> Vec<[usize; 2]> {
let mut out = Vec::new();
let mut start = None;
for (x, &on) in row.iter().enumerate() {
diff --git a/crates/mcp/Cargo.toml b/crates/mcp/Cargo.toml
index 057cca2586..47c03f4da1 100644
--- a/crates/mcp/Cargo.toml
+++ b/crates/mcp/Cargo.toml
@@ -15,3 +15,5 @@ serde_json = { workspace = true }
[dev-dependencies]
# The renderer `solidcraft-cli mcp` installs, so the tests' screenshots are drawn as the server's.
solidcraft-gpu = { workspace = true }
+# Synthetic reference pictures for the compare_picture test.
+image = { workspace = true }
diff --git a/crates/mcp/src/tests.rs b/crates/mcp/src/tests.rs
index 23957f826d..4a0e6cc0af 100644
--- a/crates/mcp/src/tests.rs
+++ b/crates/mcp/src/tests.rs
@@ -137,6 +137,7 @@ fn handshake_and_tools_list() {
"set_parameter",
"check_parameters",
"screenshot",
+ "compare_picture",
"export",
"undo",
"redo",
@@ -1469,6 +1470,46 @@ fn legacy_design_files_and_resource_uris() {
let _ = std::fs::remove_dir_all(&dir);
}
+/// `compare_picture` (#1486): the overlay comes back as an image block and the numbers as text.
+#[test]
+fn compare_picture_returns_the_overlay_as_an_image() {
+ use image::ImageEncoder as _;
+ use std::io::Cursor;
+ // A scoped server, like `solidcraft-cli mcp`: the reference must be inside the folders.
+ let base = solidcraft_engine::io::vfs::temp_dir().join(format!("solidcraft-mcp-compare-{}", std::process::id()));
+ let recovery = base.join("recovery");
+ std::fs::create_dir_all(&recovery).unwrap();
+ let scope = crate::agent_scope_with(Some(recovery.clone()), []);
+ let mut s = Server::new(Box::new(Headless::default())).with_path_scope(Some(scope));
+ tool(&mut s, "execute", json!({"command": "solid.box", "params": {"length": 20, "width": 10, "height": 30}}));
+ // A white rectangle on grey with the box's front-view proportions (20 x 30 mm).
+ let (w, h) = (200u32, 300u32);
+ let mut img = image::RgbaImage::from_pixel(w, h, image::Rgba([200, 200, 200, 255]));
+ for y in 20..280 {
+ for x in 20..180 {
+ img.put_pixel(x, y, image::Rgba([255, 255, 255, 255]));
+ }
+ }
+ let mut png = Cursor::new(Vec::new());
+ image::codecs::png::PngEncoder::new(&mut png).write_image(img.as_raw(), w, h, image::ExtendedColorType::Rgba8).unwrap();
+ let path = recovery.join("target.png");
+ std::fs::write(&path, png.into_inner()).unwrap();
+
+ let r = raw_tool(&mut s, "compare_picture", json!({"reference": path.to_string_lossy(), "view": "front"}));
+ assert_ne!(r["result"]["isError"], json!(true), "{r}");
+ let content = r["result"]["content"].as_array().unwrap().clone();
+ assert_eq!(content[0]["type"], "image", "{content:?}");
+ assert_eq!(content[0]["mimeType"], "image/png");
+ assert!(content[0]["data"].as_str().unwrap().starts_with("iVBORw0KGgo"), "not a PNG");
+ let info: Value = serde_json::from_str(content[1]["text"].as_str().unwrap()).unwrap();
+ assert!(info["overlap"].as_f64().unwrap() > 0.9, "{info}");
+ assert!(info.get("overlay_png").is_none(), "the base64 blob belongs in the image block only");
+ // A reference outside the server's folders is refused through the existing path rules.
+ let outside = raw_tool(&mut s, "compare_picture", json!({"reference": "/etc/hostname"}));
+ assert_eq!(outside["result"]["isError"], json!(true), "{outside}");
+ let _ = std::fs::remove_dir_all(&base);
+}
+
/// The default folders (#205): the recovery folder works out of the box, `screenshot` without a
/// `path` uses the server's own scratch folder, and the open design's folder counts while the
/// design is open there, without its hidden files. The working folder doesn't count at all.
diff --git a/crates/mcp/src/tools.rs b/crates/mcp/src/tools.rs
index 4c8c57cba1..10edfe0856 100644
--- a/crates/mcp/src/tools.rs
+++ b/crates/mcp/src/tools.rs
@@ -147,7 +147,7 @@ pub fn tool_definitions() -> Vec<Value> {
(as list_commands lists them). A step may also name one of these tools with that tool's arguments as params, \
and runs the engine command it stands for: set_parameter (parameters.change; `value` or `expression`), \
list_commands (engine.commands, with `filter`), describe_command (engine.describe), execute ({command, params}), \
- measure (inspect.measure), \
+ measure (inspect.measure), compare_picture (inspect.compare_picture, numbers only: no overlay PNG), \
inspect_design (document.inspect), undo, redo, export, save, open, new_design, crash_reports; screenshot, \
body_topology, batch and pending can't be steps. On an error the steps already run are undone (unless \
rollback: false), so the batch is all-or-nothing; the reply names the failing step (index from 0), its error, \
@@ -307,6 +307,34 @@ pub fn tool_definitions() -> Vec<Value> {
),
Effect::Files,
),
+ tool(
+ "compare_picture",
+ "Compare with a reference picture",
+ "Compare the model's outline with the part in a reference picture — a concept render, a product photo, one view off a design board. \
+ The model's outline is `inspect.silhouette` from `view` (default front), `camera` and `bodies`; the picture's part is found by growing \
+ the background in from the border, so a backdrop's gradual shading stays background. The compare is outline only: holes are not \
+ compared — both the picture's and the model's holes are filled. `reference` is a PNG or JPEG inside the server's \
+ folders. `crop_px` / `crop_fraction` take one view off a board: [x, y, w, h] in pixels or as fractions 0..1 of the picture; \
+ `part_fills_crop: true` says the crop box is the part itself, for a crop that tightly frames it. `fit` names \
+ the dimension the scale comes from (\"width\", \"height\", \"both\"; default \"width\") — one uniform scale, never stretched per axis; \
+ \"both\" scales by width and reports `aspect_mismatch` when the shapes differ. Returns as text the overlap (0..1) and the model's top, \
+ bottom, left and right edges against the picture's in mm at nine stations (positive where the model reaches past the picture), and as \
+ an image the overlay PNG: model red, picture blue.",
+ obj(
+ json!({
+ "reference": string("PNG or JPEG to match, inside the server's folders"),
+ "view": string("Standard view (front, top, right, iso, ...) the picture is seen from (default front)"),
+ "camera": {"type": "object", "description": "Look from this camera instead: {target: [x,y,z], yaw, pitch, distance, fov?} (radians, mm; as view.apply). A photo taken in perspective gets an fov"},
+ "bodies": {"type": "array", "items": {"type": "string"}, "description": "Body names to outline (default: the visible bodies)"},
+ "crop_px": {"type": "array", "items": {"type": "number"}, "description": "[x, y, w, h] in pixels"},
+ "crop_fraction": {"type": "array", "items": {"type": "number"}, "description": "[x, y, w, h] as fractions 0..1 of the picture"},
+ "fit": {"type": "string", "enum": ["width", "height", "both"], "description": "The dimension the one uniform scale comes from (default width); both scales by width and reports aspect_mismatch"},
+ "rows": {"type": "integer", "description": "The model outline's height in pixels (default 240)"},
+ }),
+ &["reference"],
+ ),
+ Effect::Files,
+ ),
tool(
"export",
"Export",
@@ -497,6 +525,7 @@ pub fn call_tool(b: &mut dyn Backend, name: &str, args: &Value) -> Value {
let a = args.as_object().unwrap_or(&empty);
let r = match name {
"screenshot" => return screenshot(b, a),
+ "compare_picture" => return compare_picture(b, a),
"batch" => return batch(b, a),
"check_parameters" => return check_parameters(b, a),
"execute" => return execute(b, a),
@@ -992,6 +1021,14 @@ fn batch_step(step: &Value) -> Result<(String, Value, Option<Narrow>), String> {
Some(items) => mapped("inspect.measure", json!({"items": items})),
None => Ok(("inspect.measure".into(), measure_params(a), measure_summary_wanted(a).then_some(Narrow::MeasureSummary))),
},
+ "compare_picture" => {
+ // A batch step gets the numbers; the overlay PNG would be a base64 blob in text.
+ let mut p = pick(a, &["reference", "view", "camera", "bodies", "crop_px", "crop_fraction", "fit", "rows"]);
+ if let Some(obj) = p.as_object_mut() {
+ obj.entry("overlay".to_string()).or_insert(json!(false));
+ }
+ mapped("inspect.compare_picture", p)
+ }
"inspect_design" => mapped("document.inspect", json!({"measure": a.get("measure").and_then(Value::as_bool).unwrap_or(true)})),
"undo" => mapped("edit.undo", json!({})),
"redo" => mapped("edit.redo", json!({})),
@@ -1020,6 +1057,7 @@ const TOOL_STEPS: &[&str] = &[
"batch",
"body_topology",
"check_parameters",
+ "compare_picture",
"crash_reports",
"describe_command",
"execute",
@@ -1777,6 +1815,28 @@ fn screenshot(b: &mut dyn Backend, a: &Map<String, Value>) -> Value {
}
}
+/// `compare_picture`: run the engine comparison and hand the overlay PNG back as an MCP
+/// image block, the numbers as text — an agent sees the picture, not a base64 blob.
+fn compare_picture(b: &mut dyn Backend, a: &Map<String, Value>) -> Value {
+ let mut params = pick(a, &["reference", "view", "camera", "bodies", "crop_px", "crop_fraction", "fit", "rows"]);
+ params["overlay"] = json!(true);
+ match exec(b, "inspect.compare_picture", params) {
+ Ok(v) => {
+ let overlay = v.get("overlay_png").and_then(Value::as_str).unwrap_or_default().to_string();
+ let mut info = v;
+ if let Some(obj) = info.as_object_mut() {
+ obj.remove("overlay_png");
+ }
+ let mut content = vec![json!({"type": "text", "text": info.to_string()})];
+ if !overlay.is_empty() {
+ content.insert(0, json!({"type": "image", "data": overlay, "mimeType": "image/png"}));
+ }
+ json!({"content": content})
+ }
+ Err(e) => text_result(&Value::String(e), true),
+ }
+}
+
fn capture(b: &mut dyn Backend, a: &Map<String, Value>) -> Result<(Vec<u8>, Value), String> {
// `grid` and `axes` shape a model render; the window shows what it shows.
let guides: Vec<(&str, bool)> = ["grid", "axes"].into_iter().filter_map(|k| a.get(k).and_then(Value::as_bool).map(|v| (k, v))).collect();
diff --git a/docs/mcp.md b/docs/mcp.md
index 8260913a3f..a12f8f4423 100644
--- a/docs/mcp.md
+++ b/docs/mcp.md
@@ -154,8 +154,9 @@ Each tool says what it may do, for clients that ask before running tools (or aut
these hints). `list_commands`, `describe_command`, `inspect_design`, `measure`, `body_topology`,
`skill_reference`, `crash_reports`, `pending` and `running` are read-only. `set_parameter`, `undo` and `redo`
change the design undoably, and so does `check_parameters` (it changes the design and puts it back:
-not read-only, not destructive). `new_design` replaces it (destructive). `execute`, `batch`, `open`, `save`, `export` and
-`screenshot` (whose `path` writes a file) can read, write, overwrite or delete files: they are
+not read-only, not destructive). `new_design` replaces it (destructive). `execute`, `batch`, `open`, `save`, `export`,
+`screenshot` (whose `path` writes a file) and `compare_picture` (which reads the reference picture) can read,
+write, overwrite or delete files: they are
not read-only, `destructiveHint` and `openWorldHint` are true.
### Client configuration
@@ -288,7 +289,7 @@ The `skill_reference` tool returns these documents from the copies built into So
| `list_commands` | `filter?` | The command catalog: id, label, tab/panel, shortcut, parameter docs, enabled now. `filter` matches id, label, tab and panel. |
| `describe_command` | `id` | One command's parameter docs (`params`) and, for commands with a typed contract (every new command, and more of the older ones as they are converted), the JSON Schemas (2020-12) of its parameters (`input`) and of a successful result (`output`); `null` for the others. The same as `execute` `engine.describe {command}`. |
| `execute` | `command`, `params?`, `verbose?` (default true), `expect?` | Run any command with JSON parameters (`list_commands` shows them). Failures leave the design unchanged. A feature command replies with the feature (`feature`, `name`), every body (`bodies`) and the bodies it created, changed and removed (`created_bodies`, `changed_bodies`, `removed_bodies`). `verbose: false` uses the same compact reply as `batch`. `expect` checks the design after the command, as it does for `batch` (see below). |
-| `batch` | `commands: [{command, params}]`, `rollback?` (default true), `verbose?` (default true), `timings?`, `profile?`, `coalesce_parameters?` (default false), `expect?` | Run steps in order, stop at the first error and undo the completed steps. The reply names the failing step (`failed.index`, `failed.command`, `failed.error`), every result before it, `rolled_back` (true only when the design really went back to where the batch started), `undone`, `applied` (the indices of the steps whose changes are in the design) and `not_run` (the steps never sent). A step may name a tool instead of a command, with that tool's arguments as `params`: `set_parameter` (runs `parameters.change`; `value` or `expression`), `list_commands` (`engine.commands`, filtered by `filter` as the tool filters it), `describe_command` (`engine.describe`), `execute` (`{command, params}`), `measure` (`summary: true` summarised as the tool summarises it), `inspect_design` (`document.inspect`: each sketch as its counts, not its full contents, and no joints), `undo`, `redo`, `export`, `save`, `open`, `new_design` and `crash_reports`. `screenshot`, `body_topology` (use `model.edges` and `model.faces`), `batch`, `check_parameters`, `pending`, `running` and `skill_reference` can't be steps, nor can `component.edit_in_place` and `component.finish_edit`, however a step names them, as they go to another design's tab (send them with the `execute` tool on their own): such a batch fails before anything is sent. So does a step the batch would certainly refuse at its turn (#5853): a step without a command, an unknown command id (the error names the closest ids) or, with `--param-check strict`, parameters that break a command's contract. A headless session checks this itself; an app checks it against its own commands when it gets the batch, and a bridged server at `--param-check strict` checks the contracts before it sends the batch. `verbose: false` keeps the reply small in large designs: feature steps leave out the full body list (`created_bodies`, `changed_bodies`, `removed_bodies` stay) and sketch steps the ids they repeat outside `created`. Bridged to an app that has `engine.batch`, the whole batch is one control request (one frame of the app). Older apps, and apps showing the agent cursor, get the steps one by one. `timings: true` adds `timings: [{index, command, ms}]` for every step run, the failing one included, and `total_ms`. `profile: true` also adds each step's `profile: {boolean_ms, tessellate_ms, other_ms}`, its time in kernel booleans, in tessellation and the rest. Steps are timed where they run: in the app when bridged. An app without `engine.batch` is timed step by step, without `profile`. `coalesce_parameters: true` sets each run of consecutive plain parameter edits (`parameters.change` or `set_parameter` with a name and a value, or `changes`) together, so the design rebuilds once per run instead of once per edit. Steps between runs see the design up to date. Each step keeps its own result (`coalesced`: the run's length), and the run is one undo step. The run skips the states between its edits, so an edit refused on its own can go through with the run's later ones. A refused run goes step by step, naming the refused step. |
+| `batch` | `commands: [{command, params}]`, `rollback?` (default true), `verbose?` (default true), `timings?`, `profile?`, `coalesce_parameters?` (default false), `expect?` | Run steps in order, stop at the first error and undo the completed steps. The reply names the failing step (`failed.index`, `failed.command`, `failed.error`), every result before it, `rolled_back` (true only when the design really went back to where the batch started), `undone`, `applied` (the indices of the steps whose changes are in the design) and `not_run` (the steps never sent). A step may name a tool instead of a command, with that tool's arguments as `params`: `set_parameter` (runs `parameters.change`; `value` or `expression`), `list_commands` (`engine.commands`, filtered by `filter` as the tool filters it), `describe_command` (`engine.describe`), `execute` (`{command, params}`), `measure` (`summary: true` summarised as the tool summarises it), `compare_picture` (`inspect.compare_picture`, numbers only: no overlay PNG), `inspect_design` (`document.inspect`: each sketch as its counts, not its full contents, and no joints), `undo`, `redo`, `export`, `save`, `open`, `new_design` and `crash_reports`. `screenshot`, `body_topology` (use `model.edges` and `model.faces`), `batch`, `check_parameters`, `pending`, `running` and `skill_reference` can't be steps, nor can `component.edit_in_place` and `component.finish_edit`, however a step names them, as they go to another design's tab (send them with the `execute` tool on their own): such a batch fails before anything is sent. So does a step the batch would certainly refuse at its turn (#5853): a step without a command, an unknown command id (the error names the closest ids) or, with `--param-check strict`, parameters that break a command's contract. A headless session checks this itself; an app checks it against its own commands when it gets the batch, and a bridged server at `--param-check strict` checks the contracts before it sends the batch. `verbose: false` keeps the reply small in large designs: feature steps leave out the full body list (`created_bodies`, `changed_bodies`, `removed_bodies` stay) and sketch steps the ids they repeat outside `created`. Bridged to an app that has `engine.batch`, the whole batch is one control request (one frame of the app). Older apps, and apps showing the agent cursor, get the steps one by one. `timings: true` adds `timings: [{index, command, ms}]` for every step run, the failing one included, and `total_ms`. `profile: true` also adds each step's `profile: {boolean_ms, tessellate_ms, other_ms}`, its time in kernel booleans, in tessellation and the rest. Steps are timed where they run: in the app when bridged. An app without `engine.batch` is timed step by step, without `profile`. `coalesce_parameters: true` sets each run of consecutive plain parameter edits (`parameters.change` or `set_parameter` with a name and a value, or `changes`) together, so the design rebuilds once per run instead of once per edit. Steps between runs see the design up to date. Each step keeps its own result (`coalesced`: the run's length), and the run is one undo step. The run skips the states between its edits, so an edit refused on its own can go through with the run's later ones. A refused run goes step by step, naming the refused step. |
**`expect`** on `batch` and `execute` turns the verify step into part of the build. An agent
computes the volume, body count or bounding box a stage should produce before building it;
@@ -315,6 +316,7 @@ builds it.
| `set_parameter` | `name`, `value` or `expression` (number or expression), `unit?`, `comment?` | Create or change a user parameter and recompute. `expression` is another name for `value` (the one `parameters.change` uses): give one of them, or both only when they agree; neither, or two that differ, is an error. Reports `recomputed` (features rebuilt), `restored` (features whose earlier result for the same inputs was reused, e.g. a value set back) and features that now fail. An expression `new_name = expression` renames the parameter as it sets it; in any other tool's expression parameter it adds the user parameter `new_name` and the input uses it (`defined`, `definitions` in the reply). |
| `check_parameters` | `set: {name: value}` (a number or an expression each, as `set_parameter`'s `value`), `report?` (`measure`, `cylinders`, `errors`; default all three) | Try parameter values without keeping them: one `edit.checkpoint`, every value set in one `parameters.change {changes}` (one rebuild; an app without `changes` gets one change per name), the reports read, then `edit.restore_checkpoint`, so the design, its parameters and its undo history are exactly as before. The reply has `values` (each parameter's value with the tried expressions in), `recomputed`, `measure` (volume, bbox, bodies and faces per body, as the `measure` tool reports them), `errors` and `warnings` (the features that fail or warn: `{feature, name, error}`) with `parameter_errors` (the parameters that don't evaluate), `cylinders` (every cylindrical face of every body as `{body, radius, origin, axis, faces, name, area}`; the halves a kernel splits one cylinder into are one entry, `origin` is the axis point closest to the world origin and `axis` points so that its first non-zero component is positive) and `restored: true`. A name the design has no parameter of, a value that is not a number or a string, an expression that does not evaluate or an unknown report item is an error, and nothing changes. If the checkpoint cannot be restored the call is an error with `restored: false`, `restore_error` and a note that the tried values are still in the design: set them back before saving. Replaces the loop `set_parameter`, `measure`, `body_topology`, `set_parameter` back, which cost every agent session three to five calls and up to 16 s of rebuilds. Not a `batch` step. |
| `screenshot` | `view?` (iso, top, front, back, bottom, left, right, home, fit, or a named view saved with `view.save`), `camera?` (`{target, yaw, pitch, distance, fov?}`, as `view.apply`), `path?`, `width?`, `height?`, `source?` (window, model), `grid?`, `axes?` | A PNG image content block. Headless: a render with the viewport's pipelines on a headless GPU device (see `solidcraft-cli gpu`; with no adapter, the CPU rasterizer, without analysis colours), fitted to the model, or from the named view or camera; the reply's `renderer` says which (`gpu`, `gpu-software`, `cpu`, with a `warning` for `cpu`). Bridged: the app window (default) or a model render with the app's camera; a named view or camera moves the app's viewport there first (`execute` `view.apply` does the same without a screenshot). `grid: false` and `axes: false` leave the ground grid (it lies on Z 0 and cuts through a part that straddles it) and the origin's axes out of a model render; axes hidden with `browser.visibility` stay out whatever `axes` says. Bridged, a screenshot that gives `grid` or `axes` is a model render, which follows the app's Grid and Origin switches for what it does not give; `source: "window"` with either is an error. A window that is minimised or on another macOS Space presents no frame: with no `source` the reply is a model render and says so (`fallback`); `source: "window"` is an error saying why, at once. Agents that don't need the app's chrome can ask for `source: "model"`. |
+| `compare_picture` | `reference`, `view?`, `camera?`, `bodies?`, `crop_px?`, `crop_fraction?`, `part_fills_crop?`, `fit?` (width, height, both; default width), `rows?` | Compare the model's outline with the part in a reference picture (a PNG or JPEG inside the server's folders, read through the same path rules as `screenshot`). The compare is outline only: holes are not compared — both the picture's and the model's holes are filled. The model's outline is `inspect.silhouette` seen from `view` (default front) or `camera`; the picture's part is found by growing the background in from the border through small colour steps, filling holes and keeping the largest piece, so a backdrop's shading stays background and labels on a board drop out; a part touching the border, a white part on a light grey backdrop, and a part on a transparent background all separate. `crop_px` (`[x, y, w, h]` in pixels) or `crop_fraction` (the same as fractions 0..1) takes one view off a board; with `part_fills_crop: true` the crop box is the part itself and no background search runs, for a crop that tightly frames the part; `fit` names the dimension the one uniform scale comes from, never stretched per axis (`both` scales by width and reports `aspect_mismatch`). Returns an image content block (the overlay PNG: model red, picture blue) and text: `overlap` (0..1) and the model's top, bottom, left and right edges against the picture's in mm at nine stations, positive where the model reaches past the picture, null where a station misses one of the two. As a `batch` step it runs `inspect.compare_picture` without the overlay. |
| `export` | `path`, `format?` (step, 3mf, 3mf-tagged, stl, stla, obj, urdf), `bodies?`, `mesh_prefix?`, `effort?`, `velocity?` | STEP, 3MF or mesh export; URDF writes the whole assembly with its meshes next to it (`docs/urdf.md`). |
| `undo`, `redo` | — | Undo / redo one change. |
| `pending` | — | Bridged: the result of a request that was still running when its call returned (an error starting `still running:`), waiting up to 60 s for it; `done: false` while it runs (call again), `pending: null` when nothing runs. See [Long-running requests](#long-running-requests). |