Skip to content

SolidCraft: an open-source parametric 3D CAD application built in pure Rust. Desktop, web, and automation through MCP.

view.pick: map screenshot pixels to exact geometry (#6304) (#6329)

Dahak Buildsadbd2adf8985
diff --git a/crates/mcp/src/tools.rs b/crates/mcp/src/tools.rs
index 231e2b67e3..8104a29fc5 100644
--- a/crates/mcp/src/tools.rs
+++ b/crates/mcp/src/tools.rs
@@ -92,7 +92,8 @@ pub fn tool_definitions() -> Vec<Value> {
created: {points, curves, constraints, dimensions}: the ids of what they added, stable and not reused \
after a delete (undo rewinds the counters with the design), usable right away \
in sketch.dimension, sketch.constraint.* and sketch.delete. Feature commands return the feature's id \
- and name, every body (bodies) and the bodies the feature created, changed and removed (created_bodies, changed_bodies, removed_bodies).",
+ and name, every body (bodies) and the bodies the feature created, changed and removed (created_bodies, changed_bodies, removed_bodies). \
+ view.pick {pixels: [[x,y]…]} answers the body, face, exact world point, normal and depth under pixels of a screenshot (read-only; docs/control-protocol.md).",
obj(
json!({
"command": string("Command id (see list_commands)"),
diff --git a/crates/ui-egui/src/control.rs b/crates/ui-egui/src/control.rs
index 7dad4493bd..9685ec85cc 100644
--- a/crates/ui-egui/src/control.rs
+++ b/crates/ui-egui/src/control.rs
@@ -117,6 +117,13 @@ fn key_from(name: &str) -> Option<egui::Key> {
/// Run one command as `engine.execute` does: through `app.run`, then the first-body fit and the
/// camera the command asked for.
fn execute(app: &mut SolveApp, id: &str, params: Value) -> Result<Value, String> {
+ // `view.pick` is answered by the viewport, not the engine: pixels and rays need the
+ // window's camera and pick index, and the command must not touch the document, the
+ // selection or the undo stack. Intercepted here so `engine.execute` and `engine.batch`
+ // (and through them MCP `execute` and `batch`) reach it like any other command.
+ if id == "view.pick" {
+ return crate::view_pick::pick_command(app, &params);
+ }
let r = app.run(id, params);
if r.is_ok()
&& !matches!(
@@ -604,6 +611,7 @@ fn dispatch(app: &mut SolveApp, ctx: &egui::Context, method: &str, p: &Value) ->
let list = crate::scenario::handles(s("prefix").unwrap_or(""));
ok(json!(list.into_iter().map(|(n, p)| json!({"name": n, "x": p.x, "y": p.y})).collect::<Vec<_>>()))
}
+ "ui.pick" => wrap(crate::view_pick::pick_command(app, p)),
"ui.at" => {
if let Some(path) = p.get("occurrence").and_then(Value::as_array) {
let ids: Option<Vec<u64>> = path.iter().map(Value::as_u64).collect();
diff --git a/crates/ui-egui/src/lib.rs b/crates/ui-egui/src/lib.rs
index ff4429f5b2..cda7d7fbbb 100644
--- a/crates/ui-egui/src/lib.rs
+++ b/crates/ui-egui/src/lib.rs
@@ -258,6 +258,7 @@ pub mod ui_kit;
#[cfg(test)]
mod ui_kit_guard_tests;
pub mod update;
+pub mod view_pick;
pub mod viewport;
mod viewport_layout;
pub mod workspace;
diff --git a/crates/ui-egui/src/view_pick.rs b/crates/ui-egui/src/view_pick.rs
new file mode 100644
index 0000000000..3da1ee6e65
--- /dev/null
+++ b/crates/ui-egui/src/view_pick.rs
@@ -0,0 +1,519 @@
+//! `view.pick` (control: `ui.pick`): screenshot pixels or world rays → the exact geometry
+//! under each one (#6304).
+//!
+//! An agent that has just looked at a screenshot sends back the pixels it saw and gets the
+//! body, face index, world point, surface normal and depth under each — the inputs
+//! `solid.fillet`, `inspect.measure` and `construct.point` take. The command is read-only:
+//! it never changes the selection, the camera or the document, and it answers from the same
+//! per-body pick index the mouse uses, so a pick here matches what a click on the same
+//! pixel would pick.
+//!
+//! Frames: `window` pixels are physical pixels of a window screenshot (divided by
+//! `pixels_per_point`, offset by the viewport rect); `model` pixels are pixels of a
+//! `ui.render` of the same size, so an agent can render, look, and pick without a window.
+
+use std::collections::HashMap;
+
+use egui::{Pos2, Rect, pos2, vec2};
+use serde_json::{Value, json};
+
+use solidcraft_engine::doc::{apply_point, is_identity};
+use solidcraft_engine::geom::{Mesh, Vec3};
+
+use crate::SolveApp;
+
+/// At most this many pixels + rays per call: a batch of picks is one request, but it still
+/// runs on the window thread.
+const MAX_PICKS: usize = 1024;
+
+/// The default model-render size, as the MCP `screenshot` tool renders by default.
+const DEFAULT_MODEL_W: f64 = 960.0;
+const DEFAULT_MODEL_H: f64 = 600.0;
+
+/// How far past the section plane a face point may lie and still count as kept (the renderer
+/// clips at `n·p <= n·o`; this is the same hairline `clip_segment` allows).
+const SECTION_EPS: f64 = 1e-6;
+
+/// Crossings one body may contribute before the section-cap walk gives up on it.
+const MAX_CROSSINGS: usize = 64;
+
+/// One input: the raw request (echoed back), its screen pixel when it came from one, and
+/// the world ray it maps to.
+struct Input<'a> {
+ raw: &'a Value,
+ pixel: Option<Pos2>,
+ o: Vec3,
+ d: Vec3,
+}
+
+/// The nearest triangle of one body along one ray, refined onto its face.
+struct Cand {
+ body: String,
+ face: usize,
+ point: Vec3,
+ normal: Vec3,
+ t: f64,
+ ghosted: bool,
+}
+
+/// A body's outermost placement: (occurrence path, occurrence name).
+struct Placed {
+ path: Vec<u64>,
+ name: String,
+}
+
+/// `view.pick` / `ui.pick`: `pixels` (screenshot pixels) and/or `rays` (world-space
+/// `{origin, dir}`) → one entry per input, in order; `null` where nothing is hit.
+pub fn pick_command(app: &mut SolveApp, p: &Value) -> Result<Value, String> {
+ let f = |k: &str| p.get(k).and_then(Value::as_f64).filter(|x| x.is_finite());
+ let frame = match p.get("frame").and_then(Value::as_str) {
+ Some("window") => true,
+ Some("model") => false,
+ None => app.viewport.rect.is_some(),
+ Some(other) => return Err(format!(r#"view.pick: frame must be "window" or "model", not {other:?}"#)),
+ };
+ let edges_on = p.get("edges").and_then(Value::as_bool).unwrap_or(true);
+ let edge_px = f("edge_px").filter(|x| *x > 0.0).unwrap_or(6.0) as f32;
+ let all = p.get("all").and_then(Value::as_bool).unwrap_or(false);
+ let include_ghosted = p.get("include_ghosted").and_then(Value::as_bool).unwrap_or(false);
+
+ // The projection the pixels live in: the window's, or a model render's own w×h.
+ let (proj, ppp) = if frame {
+ let rect = app.viewport.rect.ok_or(r#"view.pick: frame "window" needs a drawn window; headless callers use frame "model""#)?;
+ (crate::viewport::projection(app, rect), app.viewport.pixels_per_point())
+ } else {
+ let w = f("width").unwrap_or(DEFAULT_MODEL_W).clamp(16.0, 8192.0);
+ let h = f("height").unwrap_or(DEFAULT_MODEL_H).clamp(16.0, 8192.0);
+ let rect = Rect::from_min_size(Pos2::ZERO, vec2(w as f32, h as f32));
+ let cam = app.cam;
+ (crate::viewport::Proj { cam, vp: cam.view_proj(w / h, crate::viewport::scene_radius(app)), rect }, 1.0)
+ };
+
+ let mut inputs: Vec<Input> = Vec::new();
+ if let Some(px) = p.get("pixels") {
+ for q in px.as_array().ok_or("view.pick: `pixels` must be an array of [x, y]".to_string())? {
+ let mut xy = [0.0f64; 2];
+ for (i, v) in xy.iter_mut().enumerate() {
+ *v = q.get(i).and_then(Value::as_f64).filter(|x| x.is_finite()).ok_or("view.pick: a pixel must be [x, y] numbers")?;
+ }
+ let (x, y) = (xy[0], xy[1]);
+ // Window screenshots are whole-window, in physical pixels; `Proj` speaks
+ // absolute window points, so only the HiDPI scale comes off.
+ let pos = if frame { pos2((x / ppp as f64) as f32, (y / ppp as f64) as f32) } else { pos2(x as f32, y as f32) };
+ let (o, d) = proj.ray(pos);
+ inputs.push(Input { raw: q, pixel: Some(pos), o, d });
+ }
+ }
+ if let Some(rs) = p.get("rays") {
+ for r in rs.as_array().ok_or("view.pick: `rays` must be an array of {origin, dir}".to_string())? {
+ let triple = |key: &str| -> Result<Vec3, String> {
+ let v = r.get(key).ok_or_else(|| format!("view.pick: a ray needs `{key}` [x, y, z]"))?;
+ let mut xyz = [0.0f64; 3];
+ for (i, x) in xyz.iter_mut().enumerate() {
+ *x = v.get(i).and_then(Value::as_f64).filter(|x| x.is_finite()).ok_or(format!("view.pick: `{key}` must be [x, y, z] numbers"))?;
+ }
+ Ok(Vec3::new(xyz[0], xyz[1], xyz[2]))
+ };
+ let d = triple("dir")?.normalized().ok_or("view.pick: a ray needs a non-zero `dir`")?;
+ inputs.push(Input { raw: r, pixel: None, o: triple("origin")?, d });
+ }
+ }
+ if inputs.is_empty() {
+ return Err("view.pick: give `pixels` and/or `rays`".to_string());
+ }
+ if inputs.len() > MAX_PICKS {
+ return Err(format!("view.pick: at most {MAX_PICKS} pixels+rays per call"));
+ }
+
+ // What the viewport shows now, and what a pick may not answer for.
+ let st = crate::open_meshing::shown_world(app);
+ let bounds = crate::viewport::body_bounds(app, &st);
+ let active_bodies = app.session.active_occurrence_path.as_ref().and_then(|q| app.session.occurrence_bodies(q, true).ok());
+ let replaced: &[String] = app.preview.effective_replaced();
+ let section = crate::viewport::section_plane(app);
+ let wireframe = app.ui.visual_style == 2;
+ let slack = crate::viewport::scene_radius(app) * 0.02;
+ let occ = occurrence_map(app);
+ let comps = component_names(app);
+
+ let mut hits: Vec<Value> = Vec::with_capacity(inputs.len());
+ for input in &inputs {
+ let mut cands = face_candidates(app, &st, &bounds, input, &active_bodies, replaced, include_ghosted);
+ cands.sort_by(|a, b| a.t.total_cmp(&b.t));
+ // A section hides the material its normal points away from: faces on the removed
+ // side do not stop a pick, and the cut cap answers where the plane crosses material.
+ let kept: Vec<&Cand> = match section {
+ Some((so, sn)) => cands.iter().filter(|c| sn.dot(c.point) <= sn.dot(so) + SECTION_EPS).collect(),
+ None => cands.iter().collect(),
+ };
+ let cap = section
+ .and_then(|(so, sn)| section_cap(app, &st, &bounds, input, so, sn, kept.first().map(|c| c.t), include_ghosted, replaced, &active_bodies));
+ let face_t = kept.first().map(|c| c.t).or(cap.as_ref().map(|(t, _, _)| *t));
+ let (edge, vertex) = match (edges_on, input.pixel) {
+ (true, Some(pos)) => {
+ near_edges(app, &st, &bounds, &proj, input, pos, face_t, edge_px, &active_bodies, replaced, include_ghosted, wireframe, slack)
+ }
+ _ => (None, None),
+ };
+ hits.push(match (all, cap, kept) {
+ (true, cap, kept) => {
+ let mut row: Vec<Value> = Vec::new();
+ if let Some((t, body, sn)) = cap {
+ row.push(cap_json(input, &body, t, input.o + input.d * t, sn));
+ }
+ row.extend(kept.iter().map(|c| hit_json(input, c, &occ, &comps, None, None)));
+ Value::Array(row)
+ }
+ (false, Some((t, body, sn)), kept) => match kept.first() {
+ // The cap is nearer than any kept face: the pick lands on the cut.
+ Some(c) if c.t < t => hit_json(input, c, &occ, &comps, edge.as_ref(), vertex.as_ref()),
+ _ => cap_json(input, &body, t, input.o + input.d * t, sn),
+ },
+ (false, None, kept) => match kept.first() {
+ Some(c) => hit_json(input, c, &occ, &comps, edge.as_ref(), vertex.as_ref()),
+ None => match (&edge, &vertex) {
+ (None, None) => Value::Null,
+ _ => json!({
+ "pixel": input.raw, "body": null, "face": null,
+ "edge": edge.as_ref().map(edge_json), "vertex": vertex.as_ref().map(vertex_json),
+ }),
+ },
+ },
+ });
+ }
+ Ok(json!({
+ "frame": if frame { "window" } else { "model" },
+ "camera": {
+ "projection": if proj.cam.fov > 0.0 { "perspective" } else { "orthographic" },
+ "eye": v3(proj.cam.eye()),
+ "target": v3(proj.cam.target),
+ },
+ "hits": hits,
+ }))
+}
+
+/// The nearest face of every pickable body along the ray, nearest first. Bodies the
+/// viewport hides, ghosts (unless asked for) or stands in for with a preview are skipped,
+/// exactly as the mouse picker skips them.
+fn face_candidates(
+ app: &SolveApp,
+ st: &solidcraft_engine::doc::ModelState,
+ bounds: &[(String, solidcraft_engine::geom::Aabb3)],
+ input: &Input,
+ active_bodies: &Option<Vec<String>>,
+ replaced: &[String],
+ include_ghosted: bool,
+) -> Vec<Cand> {
+ let mut out = Vec::new();
+ for (i, b) in st.bodies.iter().enumerate() {
+ if !app.session.body_visible(&b.name) || replaced.contains(&b.name) {
+ continue;
+ }
+ let ghosted = crate::viewport::body_faded(app, b, active_bodies);
+ if ghosted && !include_ghosted {
+ continue;
+ }
+ if let Some((_, bb)) = bounds.get(i) {
+ let pad = bb.diagonal() * 1e-6 + 1e-9;
+ if crate::viewport::ray_box(input.o, input.d, bb, pad).is_none() {
+ continue;
+ }
+ }
+ // A placed part is picked in its own frame, as the mouse picker does: the ray moves
+ // there (a rigid move keeps distances along it), so no moved copy of the mesh is made.
+ let (m, place) = b.mesh_in_place();
+ let Some((lo, ld)) = crate::viewport::local_ray(&place, input.o, input.d) else { continue };
+ let Some((t, ti)) = crate::pick_index::raycast(app.viewport.pick_index.of(&m).as_deref(), &m, lo, ld) else { continue };
+ let Some((point, normal)) = refine(&m, ti, t, lo, ld, &place) else { continue };
+ out.push(Cand { body: b.name.clone(), face: m.tri_face.get(ti).copied().unwrap_or(0) as usize, point, normal, t, ghosted });
+ }
+ out
+}
+
+/// The hit point and outward surface normal in world space. A flat face (constant mesh
+/// normals) is solved exactly against its plane, so a box picks to the last digit; a curved
+/// face keeps the mesh hit (within the display tolerance) and takes its normal interpolated
+/// from the mesh's per-vertex surface normals.
+fn refine(m: &Mesh, ti: usize, t: f64, lo: Vec3, ld: Vec3, place: &solidcraft_engine::doc::Mat) -> Option<(Vec3, Vec3)> {
+ let [a, b, c] = *m.triangles.get(ti)?;
+ let (p0, p1, p2) = (m.positions.get(a as usize)?, m.positions.get(b as usize)?, m.positions.get(c as usize)?);
+ let lp = lo + ld * t;
+ let mesh_normal = || (*p1 - *p0).cross(*p2 - *p0).normalized().or_else(|| (ld * -1.0).normalized());
+ let (point_local, n_local) = match (m.normals.get(a as usize), m.normals.get(b as usize), m.normals.get(c as usize)) {
+ (Some(na), Some(nb), Some(nc)) if nb.dist(*na) < 1e-6 && nc.dist(*na) < 1e-6 && nc.dist(*nb) < 1e-6 => {
+ // Flat: the exact ray-plane intersection, on the face's own plane.
+ let n = na.normalized()?;
+ let den = n.dot(ld);
+ let tp = if den.abs() < 1e-12 { t } else { (n.dot(*p0 - lo) / den).max(0.0) };
+ (lo + ld * tp, n)
+ }
+ (Some(na), Some(nb), Some(nc)) => {
+ let (w0, w1, w2) = barycentric(lp, *p0, *p1, *p2);
+ let n = (Vec3::new(na.x * w0 + nb.x * w1 + nc.x * w2, na.y * w0 + nb.y * w1 + nc.y * w2, na.z * w0 + nb.z * w1 + nc.z * w2))
+ .normalized()
+ .or_else(mesh_normal)?;
+ (lp, n)
+ }
+ _ => (lp, mesh_normal()?),
+ };
+ let point = apply_point(place, point_local);
+ let normal = apply_dir(place, point_local, n_local);
+ Some((point, normal))
+}
+
+/// A placement is rigid, so a direction rides along it as a point difference.
+fn apply_dir(place: &solidcraft_engine::doc::Mat, at: Vec3, n: Vec3) -> Vec3 {
+ if is_identity(place) {
+ return n;
+ }
+ (apply_point(place, at + n) - apply_point(place, at)).normalized().unwrap_or(n)
+}
+
+/// Affine weights of `p` in triangle `a b c`.
+fn barycentric(p: Vec3, a: Vec3, b: Vec3, c: Vec3) -> (f64, f64, f64) {
+ let (e0, e1, e2) = (b - a, c - a, p - a);
+ let (d00, d01, d11) = (e0.dot(e0), e0.dot(e1), e1.dot(e1));
+ let (d20, d21) = (e2.dot(e0), e2.dot(e1));
+ let den = d00 * d11 - d01 * d01;
+ if !(den.is_finite() && den.abs() > 1e-24) {
+ return (1.0, 0.0, 0.0);
+ }
+ let v = (d11 * d20 - d01 * d21) / den;
+ let w = (d00 * d21 - d01 * d20) / den;
+ (1.0 - v - w, v, w)
+}
+
+/// Whether the section plane crosses material at the ray (where a cut cap answers): the
+/// plane crossing lies inside a body, told by an odd number of mesh crossings before it.
+/// The walk re-raycasts each body past every crossing it finds, bounded per body.
+#[allow(clippy::too_many_arguments)]
+fn section_cap(
+ app: &SolveApp,
+ st: &solidcraft_engine::doc::ModelState,
+ bounds: &[(String, solidcraft_engine::geom::Aabb3)],
+ input: &Input,
+ so: Vec3,
+ sn: Vec3,
+ first_kept: Option<f64>,
+ include_ghosted: bool,
+ replaced: &[String],
+ active_bodies: &Option<Vec<String>>,
+) -> Option<(f64, String, Vec3)> {
+ let den = sn.dot(input.d);
+ if den.abs() < 1e-12 {
+ return None;
+ }
+ let t_cap = (sn.dot(so) - sn.dot(input.o)) / den;
+ if !(t_cap > 0.0) || t_cap >= first_kept.unwrap_or(f64::INFINITY) {
+ return None;
+ }
+ let walk_eps = 1e-6 * (1.0 + crate::viewport::scene_radius(app));
+ for (i, b) in st.bodies.iter().enumerate() {
+ if !app.session.body_visible(&b.name) || replaced.contains(&b.name) {
+ continue;
+ }
+ if !include_ghosted && crate::viewport::body_faded(app, b, active_bodies) {
+ continue;
+ }
+ if let Some((_, bb)) = bounds.get(i) {
+ let pad = bb.diagonal() * 1e-6 + 1e-9;
+ match crate::viewport::ray_box(input.o, input.d, bb, pad) {
+ None => continue,
+ Some(t0) if t0 > t_cap => continue,
+ _ => {}
+ }
+ }
+ let (m, place) = b.mesh_in_place();
+ let index = app.viewport.pick_index.of(&m);
+ let (mut t, mut crossings) = (0.0f64, 0usize);
+ while let Some((lo, ld)) = crate::viewport::local_ray(&place, input.o + input.d * t, input.d) {
+ let Some((dt, _)) = crate::pick_index::raycast(index.as_deref(), &m, lo, ld) else { break };
+ let tt = t + dt;
+ if tt >= t_cap - SECTION_EPS {
+ break;
+ }
+ crossings += 1;
+ t = tt + walk_eps;
+ if crossings > MAX_CROSSINGS {
+ break;
+ }
+ }
+ if crossings % 2 == 1 {
+ return Some((t_cap, b.name.clone(), sn));
+ }
+ }
+ None
+}
+
+/// The edge and vertex nearest the pixel, within `edge_px`, hidden geometry excluded the way
+/// the mouse picker excludes it. Rays without a pixel get neither (they have no screen reach).
+#[allow(clippy::too_many_arguments)]
+fn near_edges(
+ app: &SolveApp,
+ st: &solidcraft_engine::doc::ModelState,
+ bounds: &[(String, solidcraft_engine::geom::Aabb3)],
+ proj: &crate::viewport::Proj,
+ input: &Input,
+ pos: Pos2,
+ face_t: Option<f64>,
+ edge_px: f32,
+ active_bodies: &Option<Vec<String>>,
+ replaced: &[String],
+ include_ghosted: bool,
+ wireframe: bool,
+ slack: f64,
+) -> (Option<EdgeHit>, Option<VertexHit>) {
+ let reach = crate::pick_index::reach(proj, pos, edge_px.max(6.0));
+ let visible = |p: Vec3| wireframe || face_t.is_none_or(|t| (p - input.o).dot(input.d) <= t + slack);
+ let mut best_e: Option<EdgeHit> = None;
+ let mut best_v: Option<VertexHit> = None;
+ for (i, b) in st.bodies.iter().enumerate() {
+ if !app.session.body_visible(&b.name) || replaced.contains(&b.name) {
+ continue;
+ }
+ if !include_ghosted && crate::viewport::body_faded(app, b, active_bodies) {
+ continue;
+ }
+ if let Some((_, bb)) = bounds.get(i)
+ && crate::viewport::screen_box(proj, bb, edge_px).is_some_and(|r| !r.contains(pos))
+ {
+ continue;
+ }
+ let (m, place) = b.mesh_in_place();
+ let Some((lo, ld)) = crate::viewport::local_ray(&place, input.o, input.d) else { continue };
+ let moved = !is_identity(&place);
+ let at = |p: Vec3| if moved { apply_point(&place, p) } else { p };
+ let near = crate::pick_index::segments_near(app.viewport.pick_index.of(&m).as_deref(), &m, lo, ld, reach);
+ let mut last_edge = None;
+ for (ei, si) in near {
+ let Some(e) = m.edges.get(ei) else { continue };
+ if !m.edge_pickable(ei) {
+ continue;
+ }
+ if last_edge != Some(ei) {
+ last_edge = Some(ei);
+ for v in [e.first(), e.last()].into_iter().flatten().map(|v| at(*v)) {
+ if let Some(sp) = proj.to_screen(v) {
+ let dd = sp.distance(pos);
+ if dd <= edge_px + 1.0 && visible(v) && best_v.as_ref().is_none_or(|x| dd < x.dist_px) {
+ best_v = Some(VertexHit { dist_px: dd, point: v });
+ }
+ }
+ }
+ }
+ let (Some(p0), Some(p1)) = (e.get(si).map(|p| at(*p)), e.get(si + 1).map(|p| at(*p))) else { continue };
+ let (Some(a), Some(c)) = (proj.to_screen(p0), proj.to_screen(p1)) else { continue };
+ let dd = crate::viewport::seg_dist(pos, a, c);
+ if dd <= edge_px
+ && visible(crate::viewport::ray_segment_closest(input.o, input.d, p0, p1))
+ && best_e.as_ref().is_none_or(|x| dd < x.dist_px)
+ {
+ let k = seg_param(pos, a, c);
+ best_e = Some(EdgeHit { index: ei, dist_px: dd, point: p0 + (p1 - p0) * f64::from(k) });
+ }
+ }
+ }
+ (best_e, best_v)
+}
+
+struct EdgeHit {
+ index: usize,
+ dist_px: f32,
+ point: Vec3,
+}
+
+struct VertexHit {
+ dist_px: f32,
+ point: Vec3,
+}
+
+/// Where on the screen segment `a`–`b` the pixel `p` lies closest, as a 0..1 parameter.
+fn seg_param(p: Pos2, a: Pos2, b: Pos2) -> f32 {
+ let ab = b - a;
+ let l2 = ab.length_sq();
+ if l2 < 1e-9 { 0.0 } else { ((p - a).dot(ab) / l2).clamp(0.0, 1.0) }
+}
+
+/// Body world name → outermost placement, walked breadth-first so a shallower placement
+/// wins; bounded like the engine's own occurrence expansion.
+fn occurrence_map(app: &SolveApp) -> HashMap<String, Placed> {
+ let mut out: HashMap<String, Placed> = HashMap::new();
+ let mut queue: std::collections::VecDeque<Vec<u64>> = std::collections::VecDeque::from([vec![]]);
+ let mut budget = 10_000usize;
+ while let Some(path) = queue.pop_front() {
+ if budget == 0 {
+ break;
+ }
+ let Ok(children) = app.session.occurrence_children(&path) else { continue };
+ for placement in children {
+ budget -= 1;
+ for b in &placement.bodies {
+ out.entry(b.world.clone()).or_insert_with(|| Placed { path: placement.path.clone(), name: placement.name.clone() });
+ }
+ queue.push_back(placement.path.clone());
+ }
+ }
+ out
+}
+
+/// Body name → the component that owns it, by name (the root component names nothing).
+fn component_names(app: &SolveApp) -> HashMap<String, String> {
+ let doc = &app.session.doc;
+ let names: HashMap<u64, String> = doc.components.iter().map(|c| (c.id, c.name.clone())).collect();
+ doc.body_components.iter().filter_map(|(body, id)| names.get(id).map(|n| (body.clone(), n.clone()))).collect()
+}
+
+fn v3(v: Vec3) -> [f64; 3] {
+ [v.x, v.y, v.z]
+}
+
+fn hit_json(
+ input: &Input,
+ c: &Cand,
+ occ: &HashMap<String, Placed>,
+ comps: &HashMap<String, String>,
+ edge: Option<&EdgeHit>,
+ vertex: Option<&VertexHit>,
+) -> Value {
+ let mut j = json!({
+ "pixel": input.raw,
+ "body": c.body,
+ "face": c.face,
+ "point": v3(c.point),
+ "normal": v3(c.normal),
+ "depth": c.t,
+ });
+ if let Some(p) = occ.get(&c.body) {
+ j["occurrence"] = json!(p.path);
+ j["occurrence_name"] = json!(p.name);
+ }
+ if let Some(name) = comps.get(&c.body) {
+ j["component"] = json!(name);
+ }
+ if c.ghosted {
+ j["ghosted"] = json!(true);
+ }
+ if let Some(e) = edge {
+ j["edge"] = edge_json(e);
+ }
+ if let Some(v) = vertex {
+ j["vertex"] = vertex_json(v);
+ }
+ j
+}
+
+fn cap_json(input: &Input, body: &str, t: f64, point: Vec3, normal: Vec3) -> Value {
+ json!({ "pixel": input.raw, "body": body, "face": null, "section_cap": true, "point": v3(point), "normal": v3(normal), "depth": t })
+}
+
+fn edge_json(e: &EdgeHit) -> Value {
+ json!({ "index": e.index, "point": v3(e.point), "distance_px": e.dist_px })
+}
+
+fn vertex_json(v: &VertexHit) -> Value {
+ json!({ "point": v3(v.point), "distance_px": v.dist_px })
+}
+
+#[cfg(test)]
+#[path = "view_pick_tests.rs"]
+mod tests;
diff --git a/crates/ui-egui/src/view_pick_tests.rs b/crates/ui-egui/src/view_pick_tests.rs
new file mode 100644
index 0000000000..1f5967ed6c
--- /dev/null
+++ b/crates/ui-egui/src/view_pick_tests.rs
@@ -0,0 +1,203 @@
+//! `view.pick` / `ui.pick` (#6304): pixels and rays answer with the geometry under them,
+//! read-only, from both frames, section-aware.
+
+use serde_json::json;
+
+use solidcraft_engine::geom::Vec3;
+use solidcraft_engine::render::Camera;
+
+use crate::scenario::Harness;
+
+/// A 10×10×10 box with an orthographic camera straight over its top face: the centre pixel
+/// of a 960×600 model render looks down the +Z face at (5, 5, 10), 95 mm from the eye.
+fn top_view() -> Harness {
+ let mut h = Harness::new();
+ h.call("engine.execute", json!({"command": "solid.box", "params": {"length": 10, "width": 10, "height": 10}}));
+ h.app.cam = Camera {
+ target: Vec3::new(5.0, 5.0, 5.0),
+ yaw: 0.0,
+ pitch: std::f64::consts::FRAC_PI_2,
+ roll: 0.0,
+ distance: 100.0,
+ fov: 0.0,
+ up: Default::default(),
+ };
+ h
+}
+
+fn pick(h: &mut Harness, mut params: serde_json::Value) -> serde_json::Value {
+ params.as_object_mut().unwrap().entry("frame").or_insert(json!("model"));
+ let v = h.call("ui.pick", params);
+ assert_eq!(v["ok"], true, "{v}");
+ v["result"].clone()
+}
+
+/// The centre pixel of the top face answers with the exact face, point, normal and depth;
+/// a pixel off the box answers with nothing.
+#[test]
+fn the_centre_pixel_of_a_box_hits_its_facing_face() {
+ let mut h = top_view();
+ let r = pick(&mut h, json!({"pixels": [[480, 300], [40, 40]]}));
+ assert_eq!(r["frame"], "model");
+ assert_eq!(r["camera"]["projection"], "orthographic");
+ let hit = &r["hits"][0];
+ assert!(hit["body"].is_string(), "{hit}");
+ assert!(hit["face"].as_u64().is_some_and(|f| f < 6), "{hit}");
+ let [x, y, z] = hit["point"].as_array().unwrap().iter().map(|v| v.as_f64().unwrap()).collect::<Vec<_>>().try_into().unwrap();
+ assert!((x - 5.0).abs() < 1e-9 && (y - 5.0).abs() < 1e-9 && (z - 10.0).abs() < 1e-9, "{hit}");
+ let n = hit["normal"].as_array().unwrap();
+ assert!((n[2].as_f64().unwrap() - 1.0).abs() < 1e-9, "{hit}");
+ assert!((hit["depth"].as_f64().unwrap() - 395.0).abs() < 1e-9, "orthographic depth runs from the ray origin, 4 distances ahead: {hit}");
+ assert_eq!(r["hits"][1], serde_json::Value::Null);
+}
+
+/// Perspective and orthographic agree on the centre pixel's face and point.
+#[test]
+fn perspective_and_orthographic_agree_on_the_centre_pixel() {
+ let mut h = top_view();
+ let flat = pick(&mut h, json!({"pixels": [[480, 300]]}));
+ h.app.cam.fov = 0.6;
+ let round = pick(&mut h, json!({"pixels": [[480, 300]]}));
+ assert_eq!(round["camera"]["projection"], "perspective");
+ assert_eq!(round["hits"][0]["face"], flat["hits"][0]["face"]);
+ let (a, b) = (round["hits"][0]["point"].as_array().unwrap(), flat["hits"][0]["point"].as_array().unwrap());
+ for i in 0..3 {
+ assert!((a[i].as_f64().unwrap() - b[i].as_f64().unwrap()).abs() < 1e-6, "{round} vs {flat}");
+ }
+}
+
+/// A world ray answers without any pixel, and a zero direction is refused.
+#[test]
+fn world_rays_pick_without_a_pixel() {
+ let mut h = top_view();
+ let r = pick(&mut h, json!({"rays": [{"origin": [5.0, 5.0, 50.0], "dir": [0.0, 0.0, -1.0]}]}));
+ assert!((r["hits"][0]["depth"].as_f64().unwrap() - 40.0).abs() < 1e-9, "{}", r["hits"][0]);
+ assert!((r["hits"][0]["point"][2].as_f64().unwrap() - 10.0).abs() < 1e-9);
+ let v = h.call("ui.pick", json!({"rays": [{"origin": [0.0, 0.0, 0.0], "dir": [0.0, 0.0, 0.0]}]}));
+ assert_eq!(v["ok"], false);
+}
+
+/// A hidden body answers with nothing, as the viewport does not show it.
+#[test]
+fn a_hidden_body_is_not_picked() {
+ let mut h = top_view();
+ let name = h.app.session.world_state().bodies[0].name.clone();
+ h.app.session.visibility.hidden_bodies.push(name);
+ let r = pick(&mut h, json!({"pixels": [[480, 300]]}));
+ assert_eq!(r["hits"][0], serde_json::Value::Null);
+}
+
+/// A section hides the material its normal points at: over the removed half the pick
+/// lands on the cut cap (or the far side), never on the removed face.
+#[test]
+fn a_section_pick_lands_on_the_cap_not_the_removed_face() {
+ let mut h = top_view();
+ h.call("engine.execute", json!({"command": "inspect.section", "params": {"plane": {"origin": [0, 0, 5], "normal": [0, 0, 1]}}}));
+ let r = pick(&mut h, json!({"pixels": [[480, 300]]}));
+ let hit = &r["hits"][0];
+ assert_eq!(hit["section_cap"], true, "{hit}");
+ assert!(hit["face"].is_null(), "{hit}");
+ assert!((hit["point"][2].as_f64().unwrap() - 5.0).abs() < 1e-9, "{hit}");
+ assert!((hit["normal"][2].as_f64().unwrap() - 1.0).abs() < 1e-9, "{hit}");
+ // Off the sectioned column (the plane cuts the whole box here, so a side pixel that
+ // missed the box entirely still answers with nothing).
+ assert_eq!(r["hits"].as_array().map(Vec::len), Some(1));
+}
+
+/// A pixel near a silhouette edge reports that edge with its distance; a pixel in the
+/// middle of the face reports none.
+#[test]
+fn edge_proximity_follows_the_pixel() {
+ let mut h = top_view();
+ // The front edge of the top face lies at y = 0: a pixel two points off it (0.2 mm at
+ // this zoom is well inside 6 px) reports the edge.
+ let near = pick(&mut h, json!({"pixels": [[480, 335.5]]}));
+ assert!(near["hits"][0]["edge"].get("index").is_some(), "{}", near["hits"][0]);
+ let middle = pick(&mut h, json!({"pixels": [[480, 300]]}));
+ assert!(middle["hits"][0].get("edge").is_none(), "{}", middle["hits"][0]);
+}
+
+/// `ui.at` (world → screen) and `ui.pick` (screen → world) round-trip through the window.
+#[test]
+fn window_picks_round_trip_through_ui_at() {
+ let mut h = Harness::new();
+ h.call("engine.execute", json!({"command": "solid.box", "params": {"length": 10, "width": 10, "height": 10}}));
+ h.frames(60); // the first-body fit animates; the camera must hold still between the two calls
+ let at = h.call("ui.at", json!({"world": [5.0, 5.0, 10.0]}));
+ assert_eq!(at["ok"], true, "{at}");
+ let xy = at["result"].as_array().unwrap();
+ let r = pick(&mut h, json!({"frame": "window", "pixels": [[xy[0], xy[1]]]}));
+ assert_eq!(r["frame"], "window");
+ let hit = &r["hits"][0];
+ let n = hit["normal"].as_array().unwrap();
+ assert!((n[2].as_f64().unwrap() - 1.0).abs() < 1e-6, "the pick must land on the top face: {hit}");
+ let p = hit["point"].as_array().unwrap();
+ for (got, want) in p.iter().zip([5.0, 5.0, 10.0]) {
+ assert!((got.as_f64().unwrap() - want).abs() < 1e-6, "{hit}");
+ }
+}
+
+/// The same round-trip at a 2× HiDPI scale: window pixels are physical, so they double.
+/// Driven through `control::handle` directly: a harness frame would reset the scale factor.
+#[test]
+fn window_picks_divide_physical_pixels_at_hidpi() {
+ let mut h = Harness::new();
+ h.call("engine.execute", json!({"command": "solid.box", "params": {"length": 10, "width": 10, "height": 10}}));
+ h.app.viewport.set_pixels_per_point(2.0);
+ let at = raw_call(&mut h, "ui.at", json!({"world": [5.0, 5.0, 10.0]}));
+ let xy = at["result"].as_array().unwrap();
+ let doubled: Vec<f64> = xy.iter().map(|v| v.as_f64().unwrap() * 2.0).collect();
+ let v = raw_call(&mut h, "ui.pick", json!({"frame": "window", "pixels": [doubled]}));
+ assert_eq!(v["ok"], true, "{v}");
+ let hit = &v["result"]["hits"][0];
+ let n = hit["normal"].as_array().unwrap();
+ assert!((n[2].as_f64().unwrap() - 1.0).abs() < 1e-6, "the HiDPI pick must land on the same face: {hit}");
+}
+
+/// One control request without the harness settling frames (which would reset the scale).
+fn raw_call(h: &mut Harness, method: &str, params: serde_json::Value) -> serde_json::Value {
+ let (req, _rx) = crate::control::ControlRequest::new(method, params);
+ match crate::control::handle(&mut h.app, &h.ctx, &req) {
+ crate::control::Outcome::Done(v) => v,
+ crate::control::Outcome::Screenshot { .. } => json!({"ok": true}),
+ }
+}
+
+/// The command is read-only: selection, camera, revision and undo stand exactly where they were.
+#[test]
+fn picking_changes_nothing() {
+ let mut h = top_view();
+ let (cam, undo, revision) = (h.app.cam, h.app.session.undo.len(), h.app.session.revision);
+ let r = pick(&mut h, json!({"pixels": [[480, 300], [40, 40]], "rays": [{"origin": [5.0, 5.0, 50.0], "dir": [0.0, 0.0, -1.0]}]}));
+ assert_eq!(r["hits"].as_array().map(Vec::len), Some(3));
+ assert_eq!((h.app.cam.distance, h.app.cam.target.z), (cam.distance, cam.target.z));
+ assert_eq!(h.app.session.undo.len(), undo);
+ assert_eq!(h.app.session.revision, revision);
+}
+
+/// MCP reaches the command through `engine.execute` and `engine.batch` like any command.
+#[test]
+fn the_command_runs_through_execute_and_batch() {
+ let mut h = top_view();
+ let params = json!({"frame": "model", "pixels": [[480, 300]]});
+ let v = h.call("engine.execute", json!({"command": "view.pick", "params": params}));
+ assert_eq!(v["ok"], true, "{v}");
+ assert!(v["result"]["hits"][0]["face"].is_number(), "{v}");
+ let v = h.call("engine.batch", json!({"commands": [{"command": "view.pick", "params": params}]}));
+ assert_eq!(v["ok"], true, "{v}");
+ assert!(v["result"]["results"][0]["hits"][0]["face"].is_number(), "{v}");
+}
+
+/// `all` lists every body the ray meets, nearest first; the input cap is enforced.
+#[test]
+fn all_lists_the_faces_along_the_ray_and_the_cap_is_enforced() {
+ let mut h = top_view();
+ let r = pick(&mut h, json!({"pixels": [[480, 300]], "all": true}));
+ let row = r["hits"][0].as_array().unwrap();
+ assert_eq!(row.len(), 1, "one body under the ray: {r}");
+ assert!(row[0]["face"].is_number());
+ let pixels: Vec<Vec<f64>> = (0..1025).map(|i| vec![i as f64 % 960.0, 300.0]).collect();
+ let v = h.call("ui.pick", json!({"pixels": pixels}));
+ assert_eq!(v["ok"], false);
+ assert!(v["error"].as_str().is_some_and(|e| e.contains("1024")), "{v}");
+}
diff --git a/crates/ui-egui/src/viewport.rs b/crates/ui-egui/src/viewport.rs
index cc830df04c..1cad066413 100644
--- a/crates/ui-egui/src/viewport.rs
+++ b/crates/ui-egui/src/viewport.rs
@@ -440,7 +440,7 @@ fn sketch_color(t: &Tokens, c: Rgb) -> [u8; 4] {
}
/// With a component active, bodies outside it are drawn faded.
-fn body_faded(app: &SolveApp, b: &solidcraft_engine::doc::ModelBody, active_bodies: &Option<Vec<String>>) -> bool {
+pub(crate) fn body_faded(app: &SolveApp, b: &solidcraft_engine::doc::ModelBody, active_bodies: &Option<Vec<String>>) -> bool {
let s = &app.session;
if let Some(names) = active_bodies {
!names.contains(&b.name)
@@ -1314,7 +1314,7 @@ pub fn construction_quads(app: &SolveApp) -> Vec<(String, Vec3, [Vec3; 4])> {
/// Pick what is under `pos`: sketch points, sketch curves, vertices, edges and axes first, then
/// the surfaces along the ray (profiles, faces, planes) nearest first.
/// Bounding boxes of the shown model's bodies (cached until the model changes).
-fn body_bounds(
+pub(crate) fn body_bounds(
app: &SolveApp,
st: &std::sync::Arc<solidcraft_engine::doc::ModelState>,
) -> std::sync::Arc<Vec<(String, solidcraft_engine::geom::Aabb3)>> {
@@ -1393,7 +1393,7 @@ fn sketch_bounds(app: &SolveApp, st: &std::sync::Arc<solidcraft_engine::doc::Mod
}
/// Where a ray enters a box (None: it misses).
-fn ray_box(o: Vec3, d: Vec3, b: &solidcraft_engine::geom::Aabb3, pad: f64) -> Option<f64> {
+pub(crate) fn ray_box(o: Vec3, d: Vec3, b: &solidcraft_engine::geom::Aabb3, pad: f64) -> Option<f64> {
let (mut t0, mut t1) = (f64::NEG_INFINITY, f64::INFINITY);
for (oo, dd, lo, hi) in [(o.x, d.x, b.min.x, b.max.x), (o.y, d.y, b.min.y, b.max.y), (o.z, d.z, b.min.z, b.max.z)] {
let (lo, hi) = (lo - pad, hi + pad);
@@ -1411,7 +1411,7 @@ fn ray_box(o: Vec3, d: Vec3, b: &solidcraft_engine::geom::Aabb3, pad: f64) -> Op
}
/// The point of segment `a`–`b` nearest the ray from `o` along `d`.
-fn ray_segment_closest(o: Vec3, d: Vec3, a: Vec3, b: Vec3) -> Vec3 {
+pub(crate) fn ray_segment_closest(o: Vec3, d: Vec3, a: Vec3, b: Vec3) -> Vec3 {
let u = b - a;
let w = a - o;
let (dd, du, uu) = (d.dot(d), d.dot(u), u.dot(u));
@@ -1428,7 +1428,7 @@ fn ray_segment_closest(o: Vec3, d: Vec3, a: Vec3, b: Vec3) -> Vec3 {
}
/// A box's outline on screen, grown by `px` (None: behind the eye).
-fn screen_box(proj: &Proj, b: &solidcraft_engine::geom::Aabb3, px: f32) -> Option<Rect> {
+pub(crate) fn screen_box(proj: &Proj, b: &solidcraft_engine::geom::Aabb3, px: f32) -> Option<Rect> {
let mut r = Rect::NOTHING;
for i in 0..8 {
let p = Vec3::new(
@@ -2887,7 +2887,7 @@ fn body_rgb(app: &SolveApp, name: &str) -> (u8, u8, u8) {
/// The ray `o` + t `d` in the frame of a mesh that `place` (a rigid move) puts in the world, with
/// the same t along it; `None` for a move with no inverse.
-fn local_ray(place: &solidcraft_engine::doc::Mat, o: Vec3, d: Vec3) -> Option<(Vec3, Vec3)> {
+pub(crate) fn local_ray(place: &solidcraft_engine::doc::Mat, o: Vec3, d: Vec3) -> Option<(Vec3, Vec3)> {
if solidcraft_engine::doc::is_identity(place) {
return Some((o, d));
}
@@ -2908,7 +2908,7 @@ pub(crate) fn polyline_mid(pts: &[Vec3]) -> Vec3 {
pts.first().copied().unwrap_or_default()
}
-fn seg_dist(p: Pos2, a: Pos2, b: Pos2) -> f32 {
+pub(crate) fn seg_dist(p: Pos2, a: Pos2, b: Pos2) -> f32 {
let ab = b - a;
let l2 = ab.length_sq();
let t = if l2 > 0.0 { ((p - a).dot(ab) / l2).clamp(0.0, 1.0) } else { 0.0 };
diff --git a/docs/control-protocol.md b/docs/control-protocol.md
index 4773f65e6f..2a16e4e293 100644
--- a/docs/control-protocol.md
+++ b/docs/control-protocol.md
@@ -230,6 +230,7 @@ labels; `prefs:measure_axis_distances` identifies the checkbox.
| `ui.sketchToScreen` | `points`: `[[x, y]…]` in active-sketch coordinates | their screen points (null when behind the camera) |
| `ui.dialogInput` | `index` | what clicking an input's Select chip does: picks go to that input |
| `ui.at` | `world: [x,y,z]` \| `sketch: [x,y]` \| `plane: "XY"` \| `axis: "Z"` \| `dimension: "d1"` (its value text) \| `handle: "arrow"` (a manipulator handle, see `scenario::publish_handle`) | the screen point (a scenario step's target; see `crates/ui-egui/src/scenario.rs`) |
+| `ui.pick` | `pixels?: [[x,y]…]`, `rays?: [{origin, dir}…]`, `frame?: "window"\|"model"`, `width?`, `height?`, `edges?` (default true), `edge_px?` (default 6), `all?`, `include_ghosted?` | read-only geometry under each pixel or ray (#6304): one entry per input, `null` where nothing is hit — body, face index, exact world `point` and outward `normal`, `depth`, `occurrence`/`component` for placed parts, `edge`/`vertex` within `edge_px` of a pixel, `section_cap: true` on a cut face. `window` pixels are physical pixels of a window screenshot; `model` pixels are pixels of a `ui.render` of `width`×`height` (default 960×600), so it works headless. As `view.pick` through `engine.execute`/`engine.batch`. See `crates/ui-egui/src/view_pick.rs` |
| `ui.handles` | `prefix?` | the handles drawn last frame, `[{name, x, y}]`: toolbar buttons and drop-down items `toolbar:<command id>` (the button alone: `button:<command id>`; a drop-down row: `dropdown:<command id>`, highlighted `dropdown_hot:<command id>`, its pin `dropdown_pin:<command id>`; Customize Toolbar's check boxes `customize:<command id>`), ribbon tabs `tab:<name>` (MODEL, SURFACE, ASSEMBLE, INSERT, INSPECT, MANAGE, VIEW; contextual SKETCH and SHEET METAL; workspace RENDER), panel labels `panel:<tab>:<panel>` (also `ribbon:section:<TAB>:<SECTION>`), document overflow list `tab:overflow`, workspace column `ribbon:workspace`, command-search launcher `ribbon:search`, browser rows `row:<label>` and fold arrows `fold:<label>`, manipulator handles; signed-axis tips `axis_view:z`, `axis_view:-y`; optional cube facets `navigation_cube:top`, `navigation_cube:top_front_right`, controls `navigation_cube:control:home` (and other actions), plus signed-axis aliases for visible cube faces; Preferences navigator picker `prefs:navigator`, options `prefs:navigator:Signed axes` and `prefs:navigator:Cube`; in the Motion Study dialog, where the chart puts its first and last step `motion:chart_start` and `motion:chart_end`, the Check collisions button `motion:check` and each collision found `motion:collision:<n>` |
| `ui.editFeature` | `feature` (id or name) | what double-clicking a timeline item does: rolls back to it and opens its dialog filled in (sketches open in sketch mode) |
| `ui.scroll` | `x`, `y`, `delta?` | wheel zoom at the cursor |