Skip to content

SolidCraft: an open-source parametric 3D CAD application built in pure Rust. Desktop, web, and automation through MCP.

Identify commands beside the animated agent cursor (#622) (#6026)

Colin Faulkingham692265980b3d
diff --git a/crates/ui-egui/src/agent_cursor.rs b/crates/ui-egui/src/agent_cursor.rs
index 4163308d11..028284f12e 100644
--- a/crates/ui-egui/src/agent_cursor.rs
+++ b/crates/ui-egui/src/agent_cursor.rs
@@ -1,6 +1,7 @@
//! The agent cursor (#35): a second, clearly marked pointer that acts out what an agent does
//! over the control channel or MCP. Before an `engine.execute` runs, the pointer glides to the
//! command's toolbar button (or its panel when the button sits in a drop-down) and clicks it,
+//! with the command's name beside the pointer so a closed group still identifies the tool,
//! then glides to each pick in the parameters (faces, edges and points projected to the screen,
//! sketch points, the plane of a new sketch), and only then does the command run. The reply
//! waits for the animation, so an agent's timing stays honest. With "follow camera" on, a new
@@ -21,9 +22,7 @@
use std::collections::VecDeque;
use std::sync::mpsc::Sender;
-#[cfg(test)]
-use egui::vec2;
-use egui::{Pos2, Stroke, pos2};
+use egui::{Pos2, Stroke, pos2, vec2};
use serde::{Deserialize, Serialize};
use serde_json::{Value, json};
use solidcraft_engine::geom::{Vec2, Vec3};
@@ -133,7 +132,8 @@ pub struct Job {
method: String,
params: Value,
reply: Sender<ControlResponse>,
- stops: VecDeque<Pos2>,
+ /// Screen position and registry label, kept together even when several commands share a panel.
+ stops: VecDeque<(Pos2, &'static str)>,
/// Seconds per glide and per click for this request (within [`TIME_BUDGET`] in all).
glide: f64,
click: f64,
@@ -253,13 +253,17 @@ pub fn stops(app: &SolveApp, command: &str, params: &Value) -> Vec<Pos2> {
/// Where the pointer goes for an `engine.batch`: the toolbar buttons of its distinct commands, in
/// the order they first come, at most [`MAX_BATCH_STOPS`] of them.
pub fn batch_stops(steps: &[Value]) -> Vec<Pos2> {
+ batch_targets(steps).into_iter().map(|(at, _)| at).collect()
+}
+
+fn batch_targets(steps: &[Value]) -> Vec<(Pos2, &'static str)> {
let mut seen: Vec<&str> = Vec::new();
for c in steps.iter().filter_map(crate::control::batch_step_command) {
if !seen.contains(&c) {
seen.push(c);
}
}
- let buttons: Vec<Pos2> = seen.into_iter().filter_map(button).collect();
+ let buttons: Vec<_> = seen.into_iter().filter_map(|c| Some((button(c)?, solidcraft_engine::find_command(c)?.label))).collect();
summarise(&buttons, MAX_BATCH_STOPS)
}
@@ -330,22 +334,23 @@ pub fn intercept(app: &mut SolveApp, method: &str, params: &Value, reply: &Sende
if !s.show || params.get("animate").and_then(Value::as_bool) == Some(false) {
return false;
}
- let stops: VecDeque<Pos2> = match method {
+ let stops: VecDeque<_> = match method {
"engine.execute" | "command" => {
let Some(cmd) = params.get("command").or(params.get("id")).and_then(Value::as_str) else { return false };
let inner = params.get("params").cloned().unwrap_or(json!({}));
- stops(app, cmd, &inner).into()
+ let label = solidcraft_engine::find_command(cmd).map_or("", |c| c.label);
+ stops(app, cmd, &inner).into_iter().map(|at| (at, label)).collect()
}
// A batch is acted out as one request; one the app refuses is answered at once.
"engine.batch" => match params.get("commands").and_then(Value::as_array) {
- Some(steps) if crate::control::batch_refusal(steps).is_none() => batch_stops(steps).into(),
+ Some(steps) if crate::control::batch_refusal(steps).is_none() => batch_targets(steps).into(),
_ => return false,
},
_ => return false,
};
if s.speed == Speed::Instant {
// No animation: the command runs now; the pointer jumps to where it acted.
- if let Some(last) = stops.back() {
+ if let Some((last, _)) = stops.back() {
app.agent.pos = Some(*last);
app.agent.ripples.push((*last, app.now));
}
@@ -395,7 +400,7 @@ pub fn step(app: &mut SolveApp, ctx: &egui::Context) {
let Some(job) = app.agent.job.as_mut() else { return };
let (glide, click) = (job.glide, job.click);
ctx.request_repaint();
- let Some(&to) = job.stops.front() else {
+ let Some(&(to, _)) = job.stops.front() else {
if job.ahead.as_ref().is_none_or(crate::control_job::Ahead::finished) {
finish(app, ctx);
}
@@ -468,6 +473,22 @@ pub fn paint(app: &SolveApp, ctx: &egui::Context) {
painter.circle_stroke(*at, 6.0 + 22.0 * k, Stroke::new(3.0 * (1.0 - k) + 0.5, agent().gamma_multiply(1.0 - k)));
}
crate::icons::pointer::paint(&painter, p);
+ if let Some((_, label)) = app.agent.job.as_ref().and_then(|job| job.stops.front()).filter(|(_, label)| !label.is_empty()) {
+ use crate::ui_kit::{Container, Surface};
+ let t = crate::theme::Tokens::get();
+ Container::new(ctx, Surface::Popup)
+ .area(egui::Id::new("sc_agent_command"))
+ .order(egui::Order::Debug)
+ .fixed_pos(p + vec2(15.0, 42.0))
+ .constrain(true)
+ .interactable(false)
+ .fade_in(false)
+ .show(ctx, |ui| {
+ Container::popup_frame(ui.style()).fill(t.overlay).show(ui, |ui| {
+ ui.label(egui::RichText::new(crate::i18n::tr_label(label)).size(t.font_label).color(t.text));
+ });
+ });
+ }
}
#[cfg(test)]
@@ -579,6 +600,55 @@ mod tests {
assert!(frames <= 3, "{frames}");
}
+ #[test]
+ fn a_command_in_a_closed_group_is_named_while_the_pointer_acts_it_out() {
+ let mut h = crate::scenario::Harness::new();
+ h.call("ui.agent_cursor", json!({"show": true, "speed": 1}));
+ assert!(crate::scenario::handle_at("toolbar:solid.box").is_none());
+ assert!(button("solid.box").is_some(), "the closed group is the pointer's target");
+ let (tx, rx) = std::sync::mpsc::channel();
+ assert!(intercept(&mut h.app, "engine.execute", &json!({"command":"solid.box", "params":{"length":10,"width":10,"height":10}}), &tx));
+ h.frames(3);
+ assert!(h.widget("Box").is_some(), "the command must be identified before it executes");
+ assert!(rx.try_recv().is_err());
+ assert!(h.app.dialog.is_none(), "programmatic commands do not open interactive tools");
+ h.frames(120);
+ assert_eq!(rx.try_recv().unwrap()["ok"], true);
+ assert_eq!(h.app.session.world_state().bodies.len(), 1);
+ assert!(!h.app.agent.busy());
+ assert!(h.widget("Box").is_none(), "the command label ends with the animation");
+ }
+
+ #[test]
+ fn a_batch_names_each_command_even_when_they_share_a_closed_group() {
+ let mut h = crate::scenario::Harness::new();
+ h.call("ui.agent_cursor", json!({"show": true, "speed": 1}));
+ assert_eq!(button("solid.box"), button("solid.sphere"));
+ let (tx, rx) = std::sync::mpsc::channel();
+ let params = json!({"commands":[
+ {"command":"solid.box","params":{"length":10,"width":10,"height":10}},
+ {"command":"solid.sphere","params":{"center":[30,0,0],"radius":3}}
+ ]});
+ assert!(intercept(&mut h.app, "engine.batch", &params, &tx));
+ let mut seen = Vec::new();
+ for _ in 0..120 {
+ h.frame();
+ for label in ["Box", "Sphere"] {
+ if h.widget(label).is_some() && seen.last() != Some(&label) {
+ seen.push(label);
+ }
+ }
+ if !h.app.agent.busy() {
+ break;
+ }
+ }
+ assert_eq!(seen, ["Box", "Sphere"]);
+ let reply = rx.try_recv().unwrap();
+ assert_eq!(reply["result"]["completed"], 2, "{reply}");
+ assert!(h.app.dialog.is_none());
+ assert!(h.widget("Box").is_none() && h.widget("Sphere").is_none());
+ }
+
#[test]
fn the_pointer_follows_the_agents_own_mouse_input() {
let mut h = crate::scenario::Harness::new();
diff --git a/docs/control-protocol.md b/docs/control-protocol.md
index 2a16e4e293..2e789cd515 100644
--- a/docs/control-protocol.md
+++ b/docs/control-protocol.md
@@ -248,7 +248,7 @@ labels; `prefs:measure_axis_distances` identifies the checkbox.
| `ui.prefs` | any of `default_units`, `nav` (`fusion`, `solidworks`, `inventor`, `alias`, `blender`, `tinkercad`, `openscad`, `touchpad`), `zoom_reverse`, `pan_reverse_x` (default false: horizontal scrolling pans the other way), `zoom_to_cursor`, `smooth_wheel_zoom` (default true), `trackpad_pan` (default true: two-finger scroll pans, except Touchpad always zooms; pinch zooms), `orbit_around` (`view_centre` (default: what Pan moves and Fit frames), `origin` (the world origin stays still on screen), `cursor` (the model point under the pointer when the drag began); the older `orbit_cursor` boolean still sets `cursor` when true), `look_at_anim`, `look_at_speed`, `middle_double_click` (`home` (default), `fit`, `off`), `msaa`, `show_frame_rate` (default false: frames per second in the view's bottom-right corner, counting the frames the view draws anyway; it reads 0 FPS once the view is idle), `window_system` (Linux: `auto` (default: XWayland whenever there is an X display, native Wayland on niri), `wayland`, `x11`; saved in `graphics.json`, applies at the next start; `SOLIDCRAFT_WAYLAND` overrides it), `frame_rate_details` (default false: with the frame rate, a frame-time graph of the frames drawn with the triangles over it, a frame-time histogram and what the GPU drew; clicking the readout switches it), `length_decimals`, `angle_decimals`, `theme` (`dark`, `light`, `system` or a theme file's id: a built-in one such as `gruvbox-dark`, or an imported one; the reply's `theme_ids` lists them), `theme_import` (a `solidcraft-theme/1` file's JSON or text: adds the theme and turns it on; the reply's `theme_imported` is its id), `theme_remove` (an imported theme's id), `theme_export: true` (the reply's `theme_file` is the theme on now as a complete file; see [colour themes](book/src/guide/ui-tour.md#colour-themes)), `language` (`en`, `ja`, `de`, `fr`, `ko`, `ms`, `ru`, `uk`, `zh`; see [localization](localization.md)), `ui_scale`, `ui_font` (an installed font family or a font file for the interface; empty for the built-in font), `up_axis` (`z`, `y`), `offer_stand_up`, `extrude_auto_to_object` (default false: in the Extrude dialog, with a profile picked, a face, vertex or parallel plane picked off its plane switches the extent to To Object; it applies to Extrude dialogs opened afterwards), `hole_snaps` (default false: a Single hole snaps to its face's middle and round edges' centres), `hole_moves` (default false: a placed Single hole has a drag handle (`hole:<n>`), and a pick on a face that has a hole moves it there), `color_vision` (`off` (default), `protan`, `deutan`, `tritan`: colour vision modes, #778), `check_updates` (default true; [updates](book/src/guide/updates.md)); `open` (a page: General, Interface, Design, Navigation, Keyboard Shortcuts, Sketch, Graphics, AI, Plugins; Shortcuts, Assistant and Sketch & selection are aliases, and the former Selection page opens Sketch) opens the Preferences dialog there; `button`: `ok`, `cancel`, `apply`, `defaults`; `stand_up: true` stands up a just-opened Y-up file, `stand_up: false` leaves it as it is ("Leave it": the offer closes and `offer_stand_up` stays on); the offer's buttons are the handles `stand_up:stand`, `stand_up:leave` and `stand_up:never` | sets preferences (kept between runs), returns them all plus `dialog` (the open page or null), `dark` and `stand_up_offer` |
| `ui.capture` | `action?`: `capture`, `revert`, `cancel` | answers the Capture Position question (asked when a command starts or the design is saved while components were moved without capturing); without `action`, what waits |
| `ui.window` | `action`: minimize, maximize, restore, toggle, close | what the title bar's caption buttons do (on Windows and Linux the application bar is the title bar); close asks about unsaved designs first |
-| `ui.agent_cursor` | `show` (bool), `speed`: a number from 1 to 5 (times the normal speed) or `"instant"` (no animation: the command runs at once and the pointer jumps to where it acted), `follow_camera` (bool); all optional | the settings. While the cursor is shown, `engine.execute` first glides to the command's toolbar button and to each pick, then runs and replies; later requests wait their turn. It also follows the pointer input sent with `ui.move`, `ui.click`, `ui.drag` and `ui.scroll`, as each event reaches the app, and a press leaves a ripple. `"animate": false` on a call skips it |
+| `ui.agent_cursor` | `show` (bool), `speed`: a number from 1 to 5 (times the normal speed) or `"instant"` (no animation: the command runs at once and the pointer jumps to where it acted), `follow_camera` (bool); all optional | the settings. While the cursor is shown, `engine.execute` first glides to the command's toolbar button and to each pick, then runs and replies; later requests wait their turn. The pointer shows the command's localized name, including commands inside a closed toolbar group; an animated batch names each command it visits. It also follows the pointer input sent with `ui.move`, `ui.click`, `ui.drag` and `ui.scroll`, as each event reaches the app, and a press leaves a ripple. `"animate": false` on a call skips it |
| `engine.execute` → `view.layout` | `mode`: `floating` or `compact` | Apply and save the workspace layout without opening a dialog, changing the design, resetting the camera or cancelling its active tool. General preferences expose the same choice; `ui.prefs {"layout":"compact"}` also saves it. |
| `engine.execute` → `browser.expand_all` | `path?`: occurrence ids; `component?`: id; omit both for the whole tree | Open every Browser row under that scope, including folders that start closed (Origin, Named Views, Features). Display only: not saved in the design, not an undo step. |
| `engine.execute` → `browser.collapse_all` | same | Close those rows. The design row stays open when the whole tree is collapsed. |