SolidCraft: an open-source parametric 3D CAD application built in pure Rust. Desktop, web, and automation through MCP.
view.pick: map screenshot pixels to exact geometry (#6304) (#6329)
Dahak Buildsadbd2adf8985
diff --git a/crates/mcp/src/tools.rs b/crates/mcp/src/tools.rsindex 231e2b67e3..8104a29fc5 100644--- a/crates/mcp/src/tools.rs+++ b/crates/mcp/src/tools.rs@@ -92,7 +92,8 @@ pub fn tool_definitions() -> Vec<Value> {created: {points, curves, constraints, dimensions}: the ids of what they added, stable and not reused \after a delete (undo rewinds the counters with the design), usable right away \in sketch.dimension, sketch.constraint.* and sketch.delete. Feature commands return the feature's id \- and name, every body (bodies) and the bodies the feature created, changed and removed (created_bodies, changed_bodies, removed_bodies).",+ and name, every body (bodies) and the bodies the feature created, changed and removed (created_bodies, changed_bodies, removed_bodies). \+ view.pick {pixels: [[x,y]…]} answers the body, face, exact world point, normal and depth under pixels of a screenshot (read-only; docs/control-protocol.md).",obj(json!({"command": string("Command id (see list_commands)"),diff --git a/crates/ui-egui/src/control.rs b/crates/ui-egui/src/control.rsindex 7dad4493bd..9685ec85cc 100644--- a/crates/ui-egui/src/control.rs+++ b/crates/ui-egui/src/control.rs@@ -117,6 +117,13 @@ fn key_from(name: &str) -> Option<egui::Key> {/// Run one command as `engine.execute` does: through `app.run`, then the first-body fit and the/// camera the command asked for.fn execute(app: &mut SolveApp, id: &str, params: Value) -> Result<Value, String> {+ // `view.pick` is answered by the viewport, not the engine: pixels and rays need the+ // window's camera and pick index, and the command must not touch the document, the+ // selection or the undo stack. Intercepted here so `engine.execute` and `engine.batch`+ // (and through them MCP `execute` and `batch`) reach it like any other command.+ if id == "view.pick" {+ return crate::view_pick::pick_command(app, ¶ms);+ }let r = app.run(id, params);if r.is_ok()&& !matches!(@@ -604,6 +611,7 @@ fn dispatch(app: &mut SolveApp, ctx: &egui::Context, method: &str, p: &Value) ->let list = crate::scenario::handles(s("prefix").unwrap_or(""));ok(json!(list.into_iter().map(|(n, p)| json!({"name": n, "x": p.x, "y": p.y})).collect::<Vec<_>>()))}+ "ui.pick" => wrap(crate::view_pick::pick_command(app, p)),"ui.at" => {if let Some(path) = p.get("occurrence").and_then(Value::as_array) {let ids: Option<Vec<u64>> = path.iter().map(Value::as_u64).collect();diff --git a/crates/ui-egui/src/lib.rs b/crates/ui-egui/src/lib.rsindex ff4429f5b2..cda7d7fbbb 100644--- a/crates/ui-egui/src/lib.rs+++ b/crates/ui-egui/src/lib.rs@@ -258,6 +258,7 @@ pub mod ui_kit;#[cfg(test)]mod ui_kit_guard_tests;pub mod update;+pub mod view_pick;pub mod viewport;mod viewport_layout;pub mod workspace;diff --git a/crates/ui-egui/src/view_pick.rs b/crates/ui-egui/src/view_pick.rsnew file mode 100644index 0000000000..3da1ee6e65--- /dev/null+++ b/crates/ui-egui/src/view_pick.rs@@ -0,0 +1,519 @@+//! `view.pick` (control: `ui.pick`): screenshot pixels or world rays → the exact geometry+//! under each one (#6304).+//!+//! An agent that has just looked at a screenshot sends back the pixels it saw and gets the+//! body, face index, world point, surface normal and depth under each — the inputs+//! `solid.fillet`, `inspect.measure` and `construct.point` take. The command is read-only:+//! it never changes the selection, the camera or the document, and it answers from the same+//! per-body pick index the mouse uses, so a pick here matches what a click on the same+//! pixel would pick.+//!+//! Frames: `window` pixels are physical pixels of a window screenshot (divided by+//! `pixels_per_point`, offset by the viewport rect); `model` pixels are pixels of a+//! `ui.render` of the same size, so an agent can render, look, and pick without a window.++use std::collections::HashMap;++use egui::{Pos2, Rect, pos2, vec2};+use serde_json::{Value, json};++use solidcraft_engine::doc::{apply_point, is_identity};+use solidcraft_engine::geom::{Mesh, Vec3};++use crate::SolveApp;++/// At most this many pixels + rays per call: a batch of picks is one request, but it still+/// runs on the window thread.+const MAX_PICKS: usize = 1024;++/// The default model-render size, as the MCP `screenshot` tool renders by default.+const DEFAULT_MODEL_W: f64 = 960.0;+const DEFAULT_MODEL_H: f64 = 600.0;++/// How far past the section plane a face point may lie and still count as kept (the renderer+/// clips at `n·p <= n·o`; this is the same hairline `clip_segment` allows).+const SECTION_EPS: f64 = 1e-6;++/// Crossings one body may contribute before the section-cap walk gives up on it.+const MAX_CROSSINGS: usize = 64;++/// One input: the raw request (echoed back), its screen pixel when it came from one, and+/// the world ray it maps to.+struct Input<'a> {+ raw: &'a Value,+ pixel: Option<Pos2>,+ o: Vec3,+ d: Vec3,+}++/// The nearest triangle of one body along one ray, refined onto its face.+struct Cand {+ body: String,+ face: usize,+ point: Vec3,+ normal: Vec3,+ t: f64,+ ghosted: bool,+}++/// A body's outermost placement: (occurrence path, occurrence name).+struct Placed {+ path: Vec<u64>,+ name: String,+}++/// `view.pick` / `ui.pick`: `pixels` (screenshot pixels) and/or `rays` (world-space+/// `{origin, dir}`) → one entry per input, in order; `null` where nothing is hit.+pub fn pick_command(app: &mut SolveApp, p: &Value) -> Result<Value, String> {+ let f = |k: &str| p.get(k).and_then(Value::as_f64).filter(|x| x.is_finite());+ let frame = match p.get("frame").and_then(Value::as_str) {+ Some("window") => true,+ Some("model") => false,+ None => app.viewport.rect.is_some(),+ Some(other) => return Err(format!(r#"view.pick: frame must be "window" or "model", not {other:?}"#)),+ };+ let edges_on = p.get("edges").and_then(Value::as_bool).unwrap_or(true);+ let edge_px = f("edge_px").filter(|x| *x > 0.0).unwrap_or(6.0) as f32;+ let all = p.get("all").and_then(Value::as_bool).unwrap_or(false);+ let include_ghosted = p.get("include_ghosted").and_then(Value::as_bool).unwrap_or(false);++ // The projection the pixels live in: the window's, or a model render's own w×h.+ let (proj, ppp) = if frame {+ let rect = app.viewport.rect.ok_or(r#"view.pick: frame "window" needs a drawn window; headless callers use frame "model""#)?;+ (crate::viewport::projection(app, rect), app.viewport.pixels_per_point())+ } else {+ let w = f("width").unwrap_or(DEFAULT_MODEL_W).clamp(16.0, 8192.0);+ let h = f("height").unwrap_or(DEFAULT_MODEL_H).clamp(16.0, 8192.0);+ let rect = Rect::from_min_size(Pos2::ZERO, vec2(w as f32, h as f32));+ let cam = app.cam;+ (crate::viewport::Proj { cam, vp: cam.view_proj(w / h, crate::viewport::scene_radius(app)), rect }, 1.0)+ };++ let mut inputs: Vec<Input> = Vec::new();+ if let Some(px) = p.get("pixels") {+ for q in px.as_array().ok_or("view.pick: `pixels` must be an array of [x, y]".to_string())? {+ let mut xy = [0.0f64; 2];+ for (i, v) in xy.iter_mut().enumerate() {+ *v = q.get(i).and_then(Value::as_f64).filter(|x| x.is_finite()).ok_or("view.pick: a pixel must be [x, y] numbers")?;+ }+ let (x, y) = (xy[0], xy[1]);+ // Window screenshots are whole-window, in physical pixels; `Proj` speaks+ // absolute window points, so only the HiDPI scale comes off.+ let pos = if frame { pos2((x / ppp as f64) as f32, (y / ppp as f64) as f32) } else { pos2(x as f32, y as f32) };+ let (o, d) = proj.ray(pos);+ inputs.push(Input { raw: q, pixel: Some(pos), o, d });+ }+ }+ if let Some(rs) = p.get("rays") {+ for r in rs.as_array().ok_or("view.pick: `rays` must be an array of {origin, dir}".to_string())? {+ let triple = |key: &str| -> Result<Vec3, String> {+ let v = r.get(key).ok_or_else(|| format!("view.pick: a ray needs `{key}` [x, y, z]"))?;+ let mut xyz = [0.0f64; 3];+ for (i, x) in xyz.iter_mut().enumerate() {+ *x = v.get(i).and_then(Value::as_f64).filter(|x| x.is_finite()).ok_or(format!("view.pick: `{key}` must be [x, y, z] numbers"))?;+ }+ Ok(Vec3::new(xyz[0], xyz[1], xyz[2]))+ };+ let d = triple("dir")?.normalized().ok_or("view.pick: a ray needs a non-zero `dir`")?;+ inputs.push(Input { raw: r, pixel: None, o: triple("origin")?, d });+ }+ }+ if inputs.is_empty() {+ return Err("view.pick: give `pixels` and/or `rays`".to_string());+ }+ if inputs.len() > MAX_PICKS {+ return Err(format!("view.pick: at most {MAX_PICKS} pixels+rays per call"));+ }++ // What the viewport shows now, and what a pick may not answer for.+ let st = crate::open_meshing::shown_world(app);+ let bounds = crate::viewport::body_bounds(app, &st);+ let active_bodies = app.session.active_occurrence_path.as_ref().and_then(|q| app.session.occurrence_bodies(q, true).ok());+ let replaced: &[String] = app.preview.effective_replaced();+ let section = crate::viewport::section_plane(app);+ let wireframe = app.ui.visual_style == 2;+ let slack = crate::viewport::scene_radius(app) * 0.02;+ let occ = occurrence_map(app);+ let comps = component_names(app);++ let mut hits: Vec<Value> = Vec::with_capacity(inputs.len());+ for input in &inputs {+ let mut cands = face_candidates(app, &st, &bounds, input, &active_bodies, replaced, include_ghosted);+ cands.sort_by(|a, b| a.t.total_cmp(&b.t));+ // A section hides the material its normal points away from: faces on the removed+ // side do not stop a pick, and the cut cap answers where the plane crosses material.+ let kept: Vec<&Cand> = match section {+ Some((so, sn)) => cands.iter().filter(|c| sn.dot(c.point) <= sn.dot(so) + SECTION_EPS).collect(),+ None => cands.iter().collect(),+ };+ let cap = section+ .and_then(|(so, sn)| section_cap(app, &st, &bounds, input, so, sn, kept.first().map(|c| c.t), include_ghosted, replaced, &active_bodies));+ let face_t = kept.first().map(|c| c.t).or(cap.as_ref().map(|(t, _, _)| *t));+ let (edge, vertex) = match (edges_on, input.pixel) {+ (true, Some(pos)) => {+ near_edges(app, &st, &bounds, &proj, input, pos, face_t, edge_px, &active_bodies, replaced, include_ghosted, wireframe, slack)+ }+ _ => (None, None),+ };+ hits.push(match (all, cap, kept) {+ (true, cap, kept) => {+ let mut row: Vec<Value> = Vec::new();+ if let Some((t, body, sn)) = cap {+ row.push(cap_json(input, &body, t, input.o + input.d * t, sn));+ }+ row.extend(kept.iter().map(|c| hit_json(input, c, &occ, &comps, None, None)));+ Value::Array(row)+ }+ (false, Some((t, body, sn)), kept) => match kept.first() {+ // The cap is nearer than any kept face: the pick lands on the cut.+ Some(c) if c.t < t => hit_json(input, c, &occ, &comps, edge.as_ref(), vertex.as_ref()),+ _ => cap_json(input, &body, t, input.o + input.d * t, sn),+ },+ (false, None, kept) => match kept.first() {+ Some(c) => hit_json(input, c, &occ, &comps, edge.as_ref(), vertex.as_ref()),+ None => match (&edge, &vertex) {+ (None, None) => Value::Null,+ _ => json!({+ "pixel": input.raw, "body": null, "face": null,+ "edge": edge.as_ref().map(edge_json), "vertex": vertex.as_ref().map(vertex_json),+ }),+ },+ },+ });+ }+ Ok(json!({+ "frame": if frame { "window" } else { "model" },+ "camera": {+ "projection": if proj.cam.fov > 0.0 { "perspective" } else { "orthographic" },+ "eye": v3(proj.cam.eye()),+ "target": v3(proj.cam.target),+ },+ "hits": hits,+ }))+}++/// The nearest face of every pickable body along the ray, nearest first. Bodies the+/// viewport hides, ghosts (unless asked for) or stands in for with a preview are skipped,+/// exactly as the mouse picker skips them.+fn face_candidates(+ app: &SolveApp,+ st: &solidcraft_engine::doc::ModelState,+ bounds: &[(String, solidcraft_engine::geom::Aabb3)],+ input: &Input,+ active_bodies: &Option<Vec<String>>,+ replaced: &[String],+ include_ghosted: bool,+) -> Vec<Cand> {+ let mut out = Vec::new();+ for (i, b) in st.bodies.iter().enumerate() {+ if !app.session.body_visible(&b.name) || replaced.contains(&b.name) {+ continue;+ }+ let ghosted = crate::viewport::body_faded(app, b, active_bodies);+ if ghosted && !include_ghosted {+ continue;+ }+ if let Some((_, bb)) = bounds.get(i) {+ let pad = bb.diagonal() * 1e-6 + 1e-9;+ if crate::viewport::ray_box(input.o, input.d, bb, pad).is_none() {+ continue;+ }+ }+ // A placed part is picked in its own frame, as the mouse picker does: the ray moves+ // there (a rigid move keeps distances along it), so no moved copy of the mesh is made.+ let (m, place) = b.mesh_in_place();+ let Some((lo, ld)) = crate::viewport::local_ray(&place, input.o, input.d) else { continue };+ let Some((t, ti)) = crate::pick_index::raycast(app.viewport.pick_index.of(&m).as_deref(), &m, lo, ld) else { continue };+ let Some((point, normal)) = refine(&m, ti, t, lo, ld, &place) else { continue };+ out.push(Cand { body: b.name.clone(), face: m.tri_face.get(ti).copied().unwrap_or(0) as usize, point, normal, t, ghosted });+ }+ out+}++/// The hit point and outward surface normal in world space. A flat face (constant mesh+/// normals) is solved exactly against its plane, so a box picks to the last digit; a curved+/// face keeps the mesh hit (within the display tolerance) and takes its normal interpolated+/// from the mesh's per-vertex surface normals.+fn refine(m: &Mesh, ti: usize, t: f64, lo: Vec3, ld: Vec3, place: &solidcraft_engine::doc::Mat) -> Option<(Vec3, Vec3)> {+ let [a, b, c] = *m.triangles.get(ti)?;+ let (p0, p1, p2) = (m.positions.get(a as usize)?, m.positions.get(b as usize)?, m.positions.get(c as usize)?);+ let lp = lo + ld * t;+ let mesh_normal = || (*p1 - *p0).cross(*p2 - *p0).normalized().or_else(|| (ld * -1.0).normalized());+ let (point_local, n_local) = match (m.normals.get(a as usize), m.normals.get(b as usize), m.normals.get(c as usize)) {+ (Some(na), Some(nb), Some(nc)) if nb.dist(*na) < 1e-6 && nc.dist(*na) < 1e-6 && nc.dist(*nb) < 1e-6 => {+ // Flat: the exact ray-plane intersection, on the face's own plane.+ let n = na.normalized()?;+ let den = n.dot(ld);+ let tp = if den.abs() < 1e-12 { t } else { (n.dot(*p0 - lo) / den).max(0.0) };+ (lo + ld * tp, n)+ }+ (Some(na), Some(nb), Some(nc)) => {+ let (w0, w1, w2) = barycentric(lp, *p0, *p1, *p2);+ let n = (Vec3::new(na.x * w0 + nb.x * w1 + nc.x * w2, na.y * w0 + nb.y * w1 + nc.y * w2, na.z * w0 + nb.z * w1 + nc.z * w2))+ .normalized()+ .or_else(mesh_normal)?;+ (lp, n)+ }+ _ => (lp, mesh_normal()?),+ };+ let point = apply_point(place, point_local);+ let normal = apply_dir(place, point_local, n_local);+ Some((point, normal))+}++/// A placement is rigid, so a direction rides along it as a point difference.+fn apply_dir(place: &solidcraft_engine::doc::Mat, at: Vec3, n: Vec3) -> Vec3 {+ if is_identity(place) {+ return n;+ }+ (apply_point(place, at + n) - apply_point(place, at)).normalized().unwrap_or(n)+}++/// Affine weights of `p` in triangle `a b c`.+fn barycentric(p: Vec3, a: Vec3, b: Vec3, c: Vec3) -> (f64, f64, f64) {+ let (e0, e1, e2) = (b - a, c - a, p - a);+ let (d00, d01, d11) = (e0.dot(e0), e0.dot(e1), e1.dot(e1));+ let (d20, d21) = (e2.dot(e0), e2.dot(e1));+ let den = d00 * d11 - d01 * d01;+ if !(den.is_finite() && den.abs() > 1e-24) {+ return (1.0, 0.0, 0.0);+ }+ let v = (d11 * d20 - d01 * d21) / den;+ let w = (d00 * d21 - d01 * d20) / den;+ (1.0 - v - w, v, w)+}++/// Whether the section plane crosses material at the ray (where a cut cap answers): the+/// plane crossing lies inside a body, told by an odd number of mesh crossings before it.+/// The walk re-raycasts each body past every crossing it finds, bounded per body.+#[allow(clippy::too_many_arguments)]+fn section_cap(+ app: &SolveApp,+ st: &solidcraft_engine::doc::ModelState,+ bounds: &[(String, solidcraft_engine::geom::Aabb3)],+ input: &Input,+ so: Vec3,+ sn: Vec3,+ first_kept: Option<f64>,+ include_ghosted: bool,+ replaced: &[String],+ active_bodies: &Option<Vec<String>>,+) -> Option<(f64, String, Vec3)> {+ let den = sn.dot(input.d);+ if den.abs() < 1e-12 {+ return None;+ }+ let t_cap = (sn.dot(so) - sn.dot(input.o)) / den;+ if !(t_cap > 0.0) || t_cap >= first_kept.unwrap_or(f64::INFINITY) {+ return None;+ }+ let walk_eps = 1e-6 * (1.0 + crate::viewport::scene_radius(app));+ for (i, b) in st.bodies.iter().enumerate() {+ if !app.session.body_visible(&b.name) || replaced.contains(&b.name) {+ continue;+ }+ if !include_ghosted && crate::viewport::body_faded(app, b, active_bodies) {+ continue;+ }+ if let Some((_, bb)) = bounds.get(i) {+ let pad = bb.diagonal() * 1e-6 + 1e-9;+ match crate::viewport::ray_box(input.o, input.d, bb, pad) {+ None => continue,+ Some(t0) if t0 > t_cap => continue,+ _ => {}+ }+ }+ let (m, place) = b.mesh_in_place();+ let index = app.viewport.pick_index.of(&m);+ let (mut t, mut crossings) = (0.0f64, 0usize);+ while let Some((lo, ld)) = crate::viewport::local_ray(&place, input.o + input.d * t, input.d) {+ let Some((dt, _)) = crate::pick_index::raycast(index.as_deref(), &m, lo, ld) else { break };+ let tt = t + dt;+ if tt >= t_cap - SECTION_EPS {+ break;+ }+ crossings += 1;+ t = tt + walk_eps;+ if crossings > MAX_CROSSINGS {+ break;+ }+ }+ if crossings % 2 == 1 {+ return Some((t_cap, b.name.clone(), sn));+ }+ }+ None+}++/// The edge and vertex nearest the pixel, within `edge_px`, hidden geometry excluded the way+/// the mouse picker excludes it. Rays without a pixel get neither (they have no screen reach).+#[allow(clippy::too_many_arguments)]+fn near_edges(+ app: &SolveApp,+ st: &solidcraft_engine::doc::ModelState,+ bounds: &[(String, solidcraft_engine::geom::Aabb3)],+ proj: &crate::viewport::Proj,+ input: &Input,+ pos: Pos2,+ face_t: Option<f64>,+ edge_px: f32,+ active_bodies: &Option<Vec<String>>,+ replaced: &[String],+ include_ghosted: bool,+ wireframe: bool,+ slack: f64,+) -> (Option<EdgeHit>, Option<VertexHit>) {+ let reach = crate::pick_index::reach(proj, pos, edge_px.max(6.0));+ let visible = |p: Vec3| wireframe || face_t.is_none_or(|t| (p - input.o).dot(input.d) <= t + slack);+ let mut best_e: Option<EdgeHit> = None;+ let mut best_v: Option<VertexHit> = None;+ for (i, b) in st.bodies.iter().enumerate() {+ if !app.session.body_visible(&b.name) || replaced.contains(&b.name) {+ continue;+ }+ if !include_ghosted && crate::viewport::body_faded(app, b, active_bodies) {+ continue;+ }+ if let Some((_, bb)) = bounds.get(i)+ && crate::viewport::screen_box(proj, bb, edge_px).is_some_and(|r| !r.contains(pos))+ {+ continue;+ }+ let (m, place) = b.mesh_in_place();+ let Some((lo, ld)) = crate::viewport::local_ray(&place, input.o, input.d) else { continue };+ let moved = !is_identity(&place);+ let at = |p: Vec3| if moved { apply_point(&place, p) } else { p };+ let near = crate::pick_index::segments_near(app.viewport.pick_index.of(&m).as_deref(), &m, lo, ld, reach);+ let mut last_edge = None;+ for (ei, si) in near {+ let Some(e) = m.edges.get(ei) else { continue };+ if !m.edge_pickable(ei) {+ continue;+ }+ if last_edge != Some(ei) {+ last_edge = Some(ei);+ for v in [e.first(), e.last()].into_iter().flatten().map(|v| at(*v)) {+ if let Some(sp) = proj.to_screen(v) {+ let dd = sp.distance(pos);+ if dd <= edge_px + 1.0 && visible(v) && best_v.as_ref().is_none_or(|x| dd < x.dist_px) {+ best_v = Some(VertexHit { dist_px: dd, point: v });+ }+ }+ }+ }+ let (Some(p0), Some(p1)) = (e.get(si).map(|p| at(*p)), e.get(si + 1).map(|p| at(*p))) else { continue };+ let (Some(a), Some(c)) = (proj.to_screen(p0), proj.to_screen(p1)) else { continue };+ let dd = crate::viewport::seg_dist(pos, a, c);+ if dd <= edge_px+ && visible(crate::viewport::ray_segment_closest(input.o, input.d, p0, p1))+ && best_e.as_ref().is_none_or(|x| dd < x.dist_px)+ {+ let k = seg_param(pos, a, c);+ best_e = Some(EdgeHit { index: ei, dist_px: dd, point: p0 + (p1 - p0) * f64::from(k) });+ }+ }+ }+ (best_e, best_v)+}++struct EdgeHit {+ index: usize,+ dist_px: f32,+ point: Vec3,+}++struct VertexHit {+ dist_px: f32,+ point: Vec3,+}++/// Where on the screen segment `a`–`b` the pixel `p` lies closest, as a 0..1 parameter.+fn seg_param(p: Pos2, a: Pos2, b: Pos2) -> f32 {+ let ab = b - a;+ let l2 = ab.length_sq();+ if l2 < 1e-9 { 0.0 } else { ((p - a).dot(ab) / l2).clamp(0.0, 1.0) }+}++/// Body world name → outermost placement, walked breadth-first so a shallower placement+/// wins; bounded like the engine's own occurrence expansion.+fn occurrence_map(app: &SolveApp) -> HashMap<String, Placed> {+ let mut out: HashMap<String, Placed> = HashMap::new();+ let mut queue: std::collections::VecDeque<Vec<u64>> = std::collections::VecDeque::from([vec![]]);+ let mut budget = 10_000usize;+ while let Some(path) = queue.pop_front() {+ if budget == 0 {+ break;+ }+ let Ok(children) = app.session.occurrence_children(&path) else { continue };+ for placement in children {+ budget -= 1;+ for b in &placement.bodies {+ out.entry(b.world.clone()).or_insert_with(|| Placed { path: placement.path.clone(), name: placement.name.clone() });+ }+ queue.push_back(placement.path.clone());+ }+ }+ out+}++/// Body name → the component that owns it, by name (the root component names nothing).+fn component_names(app: &SolveApp) -> HashMap<String, String> {+ let doc = &app.session.doc;+ let names: HashMap<u64, String> = doc.components.iter().map(|c| (c.id, c.name.clone())).collect();+ doc.body_components.iter().filter_map(|(body, id)| names.get(id).map(|n| (body.clone(), n.clone()))).collect()+}++fn v3(v: Vec3) -> [f64; 3] {+ [v.x, v.y, v.z]+}++fn hit_json(+ input: &Input,+ c: &Cand,+ occ: &HashMap<String, Placed>,+ comps: &HashMap<String, String>,+ edge: Option<&EdgeHit>,+ vertex: Option<&VertexHit>,+) -> Value {+ let mut j = json!({+ "pixel": input.raw,+ "body": c.body,+ "face": c.face,+ "point": v3(c.point),+ "normal": v3(c.normal),+ "depth": c.t,+ });+ if let Some(p) = occ.get(&c.body) {+ j["occurrence"] = json!(p.path);+ j["occurrence_name"] = json!(p.name);+ }+ if let Some(name) = comps.get(&c.body) {+ j["component"] = json!(name);+ }+ if c.ghosted {+ j["ghosted"] = json!(true);+ }+ if let Some(e) = edge {+ j["edge"] = edge_json(e);+ }+ if let Some(v) = vertex {+ j["vertex"] = vertex_json(v);+ }+ j+}++fn cap_json(input: &Input, body: &str, t: f64, point: Vec3, normal: Vec3) -> Value {+ json!({ "pixel": input.raw, "body": body, "face": null, "section_cap": true, "point": v3(point), "normal": v3(normal), "depth": t })+}++fn edge_json(e: &EdgeHit) -> Value {+ json!({ "index": e.index, "point": v3(e.point), "distance_px": e.dist_px })+}++fn vertex_json(v: &VertexHit) -> Value {+ json!({ "point": v3(v.point), "distance_px": v.dist_px })+}++#[cfg(test)]+#[path = "view_pick_tests.rs"]+mod tests;diff --git a/crates/ui-egui/src/view_pick_tests.rs b/crates/ui-egui/src/view_pick_tests.rsnew file mode 100644index 0000000000..1f5967ed6c--- /dev/null+++ b/crates/ui-egui/src/view_pick_tests.rs@@ -0,0 +1,203 @@+//! `view.pick` / `ui.pick` (#6304): pixels and rays answer with the geometry under them,+//! read-only, from both frames, section-aware.++use serde_json::json;++use solidcraft_engine::geom::Vec3;+use solidcraft_engine::render::Camera;++use crate::scenario::Harness;++/// A 10×10×10 box with an orthographic camera straight over its top face: the centre pixel+/// of a 960×600 model render looks down the +Z face at (5, 5, 10), 95 mm from the eye.+fn top_view() -> Harness {+ let mut h = Harness::new();+ h.call("engine.execute", json!({"command": "solid.box", "params": {"length": 10, "width": 10, "height": 10}}));+ h.app.cam = Camera {+ target: Vec3::new(5.0, 5.0, 5.0),+ yaw: 0.0,+ pitch: std::f64::consts::FRAC_PI_2,+ roll: 0.0,+ distance: 100.0,+ fov: 0.0,+ up: Default::default(),+ };+ h+}++fn pick(h: &mut Harness, mut params: serde_json::Value) -> serde_json::Value {+ params.as_object_mut().unwrap().entry("frame").or_insert(json!("model"));+ let v = h.call("ui.pick", params);+ assert_eq!(v["ok"], true, "{v}");+ v["result"].clone()+}++/// The centre pixel of the top face answers with the exact face, point, normal and depth;+/// a pixel off the box answers with nothing.+#[test]+fn the_centre_pixel_of_a_box_hits_its_facing_face() {+ let mut h = top_view();+ let r = pick(&mut h, json!({"pixels": [[480, 300], [40, 40]]}));+ assert_eq!(r["frame"], "model");+ assert_eq!(r["camera"]["projection"], "orthographic");+ let hit = &r["hits"][0];+ assert!(hit["body"].is_string(), "{hit}");+ assert!(hit["face"].as_u64().is_some_and(|f| f < 6), "{hit}");+ let [x, y, z] = hit["point"].as_array().unwrap().iter().map(|v| v.as_f64().unwrap()).collect::<Vec<_>>().try_into().unwrap();+ assert!((x - 5.0).abs() < 1e-9 && (y - 5.0).abs() < 1e-9 && (z - 10.0).abs() < 1e-9, "{hit}");+ let n = hit["normal"].as_array().unwrap();+ assert!((n[2].as_f64().unwrap() - 1.0).abs() < 1e-9, "{hit}");+ assert!((hit["depth"].as_f64().unwrap() - 395.0).abs() < 1e-9, "orthographic depth runs from the ray origin, 4 distances ahead: {hit}");+ assert_eq!(r["hits"][1], serde_json::Value::Null);+}++/// Perspective and orthographic agree on the centre pixel's face and point.+#[test]+fn perspective_and_orthographic_agree_on_the_centre_pixel() {+ let mut h = top_view();+ let flat = pick(&mut h, json!({"pixels": [[480, 300]]}));+ h.app.cam.fov = 0.6;+ let round = pick(&mut h, json!({"pixels": [[480, 300]]}));+ assert_eq!(round["camera"]["projection"], "perspective");+ assert_eq!(round["hits"][0]["face"], flat["hits"][0]["face"]);+ let (a, b) = (round["hits"][0]["point"].as_array().unwrap(), flat["hits"][0]["point"].as_array().unwrap());+ for i in 0..3 {+ assert!((a[i].as_f64().unwrap() - b[i].as_f64().unwrap()).abs() < 1e-6, "{round} vs {flat}");+ }+}++/// A world ray answers without any pixel, and a zero direction is refused.+#[test]+fn world_rays_pick_without_a_pixel() {+ let mut h = top_view();+ let r = pick(&mut h, json!({"rays": [{"origin": [5.0, 5.0, 50.0], "dir": [0.0, 0.0, -1.0]}]}));+ assert!((r["hits"][0]["depth"].as_f64().unwrap() - 40.0).abs() < 1e-9, "{}", r["hits"][0]);+ assert!((r["hits"][0]["point"][2].as_f64().unwrap() - 10.0).abs() < 1e-9);+ let v = h.call("ui.pick", json!({"rays": [{"origin": [0.0, 0.0, 0.0], "dir": [0.0, 0.0, 0.0]}]}));+ assert_eq!(v["ok"], false);+}++/// A hidden body answers with nothing, as the viewport does not show it.+#[test]+fn a_hidden_body_is_not_picked() {+ let mut h = top_view();+ let name = h.app.session.world_state().bodies[0].name.clone();+ h.app.session.visibility.hidden_bodies.push(name);+ let r = pick(&mut h, json!({"pixels": [[480, 300]]}));+ assert_eq!(r["hits"][0], serde_json::Value::Null);+}++/// A section hides the material its normal points at: over the removed half the pick+/// lands on the cut cap (or the far side), never on the removed face.+#[test]+fn a_section_pick_lands_on_the_cap_not_the_removed_face() {+ let mut h = top_view();+ h.call("engine.execute", json!({"command": "inspect.section", "params": {"plane": {"origin": [0, 0, 5], "normal": [0, 0, 1]}}}));+ let r = pick(&mut h, json!({"pixels": [[480, 300]]}));+ let hit = &r["hits"][0];+ assert_eq!(hit["section_cap"], true, "{hit}");+ assert!(hit["face"].is_null(), "{hit}");+ assert!((hit["point"][2].as_f64().unwrap() - 5.0).abs() < 1e-9, "{hit}");+ assert!((hit["normal"][2].as_f64().unwrap() - 1.0).abs() < 1e-9, "{hit}");+ // Off the sectioned column (the plane cuts the whole box here, so a side pixel that+ // missed the box entirely still answers with nothing).+ assert_eq!(r["hits"].as_array().map(Vec::len), Some(1));+}++/// A pixel near a silhouette edge reports that edge with its distance; a pixel in the+/// middle of the face reports none.+#[test]+fn edge_proximity_follows_the_pixel() {+ let mut h = top_view();+ // The front edge of the top face lies at y = 0: a pixel two points off it (0.2 mm at+ // this zoom is well inside 6 px) reports the edge.+ let near = pick(&mut h, json!({"pixels": [[480, 335.5]]}));+ assert!(near["hits"][0]["edge"].get("index").is_some(), "{}", near["hits"][0]);+ let middle = pick(&mut h, json!({"pixels": [[480, 300]]}));+ assert!(middle["hits"][0].get("edge").is_none(), "{}", middle["hits"][0]);+}++/// `ui.at` (world → screen) and `ui.pick` (screen → world) round-trip through the window.+#[test]+fn window_picks_round_trip_through_ui_at() {+ let mut h = Harness::new();+ h.call("engine.execute", json!({"command": "solid.box", "params": {"length": 10, "width": 10, "height": 10}}));+ h.frames(60); // the first-body fit animates; the camera must hold still between the two calls+ let at = h.call("ui.at", json!({"world": [5.0, 5.0, 10.0]}));+ assert_eq!(at["ok"], true, "{at}");+ let xy = at["result"].as_array().unwrap();+ let r = pick(&mut h, json!({"frame": "window", "pixels": [[xy[0], xy[1]]]}));+ assert_eq!(r["frame"], "window");+ let hit = &r["hits"][0];+ let n = hit["normal"].as_array().unwrap();+ assert!((n[2].as_f64().unwrap() - 1.0).abs() < 1e-6, "the pick must land on the top face: {hit}");+ let p = hit["point"].as_array().unwrap();+ for (got, want) in p.iter().zip([5.0, 5.0, 10.0]) {+ assert!((got.as_f64().unwrap() - want).abs() < 1e-6, "{hit}");+ }+}++/// The same round-trip at a 2× HiDPI scale: window pixels are physical, so they double.+/// Driven through `control::handle` directly: a harness frame would reset the scale factor.+#[test]+fn window_picks_divide_physical_pixels_at_hidpi() {+ let mut h = Harness::new();+ h.call("engine.execute", json!({"command": "solid.box", "params": {"length": 10, "width": 10, "height": 10}}));+ h.app.viewport.set_pixels_per_point(2.0);+ let at = raw_call(&mut h, "ui.at", json!({"world": [5.0, 5.0, 10.0]}));+ let xy = at["result"].as_array().unwrap();+ let doubled: Vec<f64> = xy.iter().map(|v| v.as_f64().unwrap() * 2.0).collect();+ let v = raw_call(&mut h, "ui.pick", json!({"frame": "window", "pixels": [doubled]}));+ assert_eq!(v["ok"], true, "{v}");+ let hit = &v["result"]["hits"][0];+ let n = hit["normal"].as_array().unwrap();+ assert!((n[2].as_f64().unwrap() - 1.0).abs() < 1e-6, "the HiDPI pick must land on the same face: {hit}");+}++/// One control request without the harness settling frames (which would reset the scale).+fn raw_call(h: &mut Harness, method: &str, params: serde_json::Value) -> serde_json::Value {+ let (req, _rx) = crate::control::ControlRequest::new(method, params);+ match crate::control::handle(&mut h.app, &h.ctx, &req) {+ crate::control::Outcome::Done(v) => v,+ crate::control::Outcome::Screenshot { .. } => json!({"ok": true}),+ }+}++/// The command is read-only: selection, camera, revision and undo stand exactly where they were.+#[test]+fn picking_changes_nothing() {+ let mut h = top_view();+ let (cam, undo, revision) = (h.app.cam, h.app.session.undo.len(), h.app.session.revision);+ let r = pick(&mut h, json!({"pixels": [[480, 300], [40, 40]], "rays": [{"origin": [5.0, 5.0, 50.0], "dir": [0.0, 0.0, -1.0]}]}));+ assert_eq!(r["hits"].as_array().map(Vec::len), Some(3));+ assert_eq!((h.app.cam.distance, h.app.cam.target.z), (cam.distance, cam.target.z));+ assert_eq!(h.app.session.undo.len(), undo);+ assert_eq!(h.app.session.revision, revision);+}++/// MCP reaches the command through `engine.execute` and `engine.batch` like any command.+#[test]+fn the_command_runs_through_execute_and_batch() {+ let mut h = top_view();+ let params = json!({"frame": "model", "pixels": [[480, 300]]});+ let v = h.call("engine.execute", json!({"command": "view.pick", "params": params}));+ assert_eq!(v["ok"], true, "{v}");+ assert!(v["result"]["hits"][0]["face"].is_number(), "{v}");+ let v = h.call("engine.batch", json!({"commands": [{"command": "view.pick", "params": params}]}));+ assert_eq!(v["ok"], true, "{v}");+ assert!(v["result"]["results"][0]["hits"][0]["face"].is_number(), "{v}");+}++/// `all` lists every body the ray meets, nearest first; the input cap is enforced.+#[test]+fn all_lists_the_faces_along_the_ray_and_the_cap_is_enforced() {+ let mut h = top_view();+ let r = pick(&mut h, json!({"pixels": [[480, 300]], "all": true}));+ let row = r["hits"][0].as_array().unwrap();+ assert_eq!(row.len(), 1, "one body under the ray: {r}");+ assert!(row[0]["face"].is_number());+ let pixels: Vec<Vec<f64>> = (0..1025).map(|i| vec![i as f64 % 960.0, 300.0]).collect();+ let v = h.call("ui.pick", json!({"pixels": pixels}));+ assert_eq!(v["ok"], false);+ assert!(v["error"].as_str().is_some_and(|e| e.contains("1024")), "{v}");+}diff --git a/crates/ui-egui/src/viewport.rs b/crates/ui-egui/src/viewport.rsindex cc830df04c..1cad066413 100644--- a/crates/ui-egui/src/viewport.rs+++ b/crates/ui-egui/src/viewport.rs@@ -440,7 +440,7 @@ fn sketch_color(t: &Tokens, c: Rgb) -> [u8; 4] {}/// With a component active, bodies outside it are drawn faded.-fn body_faded(app: &SolveApp, b: &solidcraft_engine::doc::ModelBody, active_bodies: &Option<Vec<String>>) -> bool {+pub(crate) fn body_faded(app: &SolveApp, b: &solidcraft_engine::doc::ModelBody, active_bodies: &Option<Vec<String>>) -> bool {let s = &app.session;if let Some(names) = active_bodies {!names.contains(&b.name)@@ -1314,7 +1314,7 @@ pub fn construction_quads(app: &SolveApp) -> Vec<(String, Vec3, [Vec3; 4])> {/// Pick what is under `pos`: sketch points, sketch curves, vertices, edges and axes first, then/// the surfaces along the ray (profiles, faces, planes) nearest first./// Bounding boxes of the shown model's bodies (cached until the model changes).-fn body_bounds(+pub(crate) fn body_bounds(app: &SolveApp,st: &std::sync::Arc<solidcraft_engine::doc::ModelState>,) -> std::sync::Arc<Vec<(String, solidcraft_engine::geom::Aabb3)>> {@@ -1393,7 +1393,7 @@ fn sketch_bounds(app: &SolveApp, st: &std::sync::Arc<solidcraft_engine::doc::Mod}/// Where a ray enters a box (None: it misses).-fn ray_box(o: Vec3, d: Vec3, b: &solidcraft_engine::geom::Aabb3, pad: f64) -> Option<f64> {+pub(crate) fn ray_box(o: Vec3, d: Vec3, b: &solidcraft_engine::geom::Aabb3, pad: f64) -> Option<f64> {let (mut t0, mut t1) = (f64::NEG_INFINITY, f64::INFINITY);for (oo, dd, lo, hi) in [(o.x, d.x, b.min.x, b.max.x), (o.y, d.y, b.min.y, b.max.y), (o.z, d.z, b.min.z, b.max.z)] {let (lo, hi) = (lo - pad, hi + pad);@@ -1411,7 +1411,7 @@ fn ray_box(o: Vec3, d: Vec3, b: &solidcraft_engine::geom::Aabb3, pad: f64) -> Op}/// The point of segment `a`–`b` nearest the ray from `o` along `d`.-fn ray_segment_closest(o: Vec3, d: Vec3, a: Vec3, b: Vec3) -> Vec3 {+pub(crate) fn ray_segment_closest(o: Vec3, d: Vec3, a: Vec3, b: Vec3) -> Vec3 {let u = b - a;let w = a - o;let (dd, du, uu) = (d.dot(d), d.dot(u), u.dot(u));@@ -1428,7 +1428,7 @@ fn ray_segment_closest(o: Vec3, d: Vec3, a: Vec3, b: Vec3) -> Vec3 {}/// A box's outline on screen, grown by `px` (None: behind the eye).-fn screen_box(proj: &Proj, b: &solidcraft_engine::geom::Aabb3, px: f32) -> Option<Rect> {+pub(crate) fn screen_box(proj: &Proj, b: &solidcraft_engine::geom::Aabb3, px: f32) -> Option<Rect> {let mut r = Rect::NOTHING;for i in 0..8 {let p = Vec3::new(@@ -2887,7 +2887,7 @@ fn body_rgb(app: &SolveApp, name: &str) -> (u8, u8, u8) {/// The ray `o` + t `d` in the frame of a mesh that `place` (a rigid move) puts in the world, with/// the same t along it; `None` for a move with no inverse.-fn local_ray(place: &solidcraft_engine::doc::Mat, o: Vec3, d: Vec3) -> Option<(Vec3, Vec3)> {+pub(crate) fn local_ray(place: &solidcraft_engine::doc::Mat, o: Vec3, d: Vec3) -> Option<(Vec3, Vec3)> {if solidcraft_engine::doc::is_identity(place) {return Some((o, d));}@@ -2908,7 +2908,7 @@ pub(crate) fn polyline_mid(pts: &[Vec3]) -> Vec3 {pts.first().copied().unwrap_or_default()}-fn seg_dist(p: Pos2, a: Pos2, b: Pos2) -> f32 {+pub(crate) fn seg_dist(p: Pos2, a: Pos2, b: Pos2) -> f32 {let ab = b - a;let l2 = ab.length_sq();let t = if l2 > 0.0 { ((p - a).dot(ab) / l2).clamp(0.0, 1.0) } else { 0.0 };diff --git a/docs/control-protocol.md b/docs/control-protocol.mdindex 4773f65e6f..2a16e4e293 100644--- a/docs/control-protocol.md+++ b/docs/control-protocol.md@@ -230,6 +230,7 @@ labels; `prefs:measure_axis_distances` identifies the checkbox.| `ui.sketchToScreen` | `points`: `[[x, y]…]` in active-sketch coordinates | their screen points (null when behind the camera) || `ui.dialogInput` | `index` | what clicking an input's Select chip does: picks go to that input || `ui.at` | `world: [x,y,z]` \| `sketch: [x,y]` \| `plane: "XY"` \| `axis: "Z"` \| `dimension: "d1"` (its value text) \| `handle: "arrow"` (a manipulator handle, see `scenario::publish_handle`) | the screen point (a scenario step's target; see `crates/ui-egui/src/scenario.rs`) |+| `ui.pick` | `pixels?: [[x,y]…]`, `rays?: [{origin, dir}…]`, `frame?: "window"\|"model"`, `width?`, `height?`, `edges?` (default true), `edge_px?` (default 6), `all?`, `include_ghosted?` | read-only geometry under each pixel or ray (#6304): one entry per input, `null` where nothing is hit — body, face index, exact world `point` and outward `normal`, `depth`, `occurrence`/`component` for placed parts, `edge`/`vertex` within `edge_px` of a pixel, `section_cap: true` on a cut face. `window` pixels are physical pixels of a window screenshot; `model` pixels are pixels of a `ui.render` of `width`×`height` (default 960×600), so it works headless. As `view.pick` through `engine.execute`/`engine.batch`. See `crates/ui-egui/src/view_pick.rs` || `ui.handles` | `prefix?` | the handles drawn last frame, `[{name, x, y}]`: toolbar buttons and drop-down items `toolbar:<command id>` (the button alone: `button:<command id>`; a drop-down row: `dropdown:<command id>`, highlighted `dropdown_hot:<command id>`, its pin `dropdown_pin:<command id>`; Customize Toolbar's check boxes `customize:<command id>`), ribbon tabs `tab:<name>` (MODEL, SURFACE, ASSEMBLE, INSERT, INSPECT, MANAGE, VIEW; contextual SKETCH and SHEET METAL; workspace RENDER), panel labels `panel:<tab>:<panel>` (also `ribbon:section:<TAB>:<SECTION>`), document overflow list `tab:overflow`, workspace column `ribbon:workspace`, command-search launcher `ribbon:search`, browser rows `row:<label>` and fold arrows `fold:<label>`, manipulator handles; signed-axis tips `axis_view:z`, `axis_view:-y`; optional cube facets `navigation_cube:top`, `navigation_cube:top_front_right`, controls `navigation_cube:control:home` (and other actions), plus signed-axis aliases for visible cube faces; Preferences navigator picker `prefs:navigator`, options `prefs:navigator:Signed axes` and `prefs:navigator:Cube`; in the Motion Study dialog, where the chart puts its first and last step `motion:chart_start` and `motion:chart_end`, the Check collisions button `motion:check` and each collision found `motion:collision:<n>` || `ui.editFeature` | `feature` (id or name) | what double-clicking a timeline item does: rolls back to it and opens its dialog filled in (sketches open in sketch mode) || `ui.scroll` | `x`, `y`, `delta?` | wheel zoom at the cursor |