Skip to content

SolidCraft: an open-source parametric 3D CAD application built in pure Rust. Desktop, web, and automation through MCP.

solidcraft-cli bench-assistant: the standard Assistant model test (#4851) (#4930)

raykholodad28abecf29
diff --git a/apps/solidcraft-cli/src/bench_assistant.rs b/apps/solidcraft-cli/src/bench_assistant.rs
new file mode 100644
index 0000000000..d6f0740b93
--- /dev/null
+++ b/apps/solidcraft-cli/src/bench_assistant.rs
@@ -0,0 +1,700 @@
+//! `bench-assistant`: the standard Assistant model test (v1, #4851), run against a desktop app.
+//!
+//! The app is already running with `--control PORT`; everything goes through its control channel
+//! (`docs/control-protocol.md`), so the command needs no window code of its own. For each task it
+//! starts a new design, runs the task's prescribed setup (a box, for some), adds the endpoint as an
+//! API agent with All permissions, starts a new chat, sends the prompt once and waits for the turn
+//! to end. The result is checked on the design (`document.inspect` with measurements), never on
+//! what the model says, except for the one question whose answer is the reply. A failed task is
+//! put in one of the issue's failure categories from the conversation's items.
+//!
+//! ```text
+//! solidcraft-cli bench-assistant --connect 127.0.0.1:PORT --base-url URL --model NAME
+//! [--token T] [--tasks 1,2,5] [--repeat N] [--timeout-s 300]
+//! [--json out.json] [--markdown out.md]
+//! ```
+//! prints a table of the results; `--markdown` writes the issue's comment template filled in,
+//! `--json` every task's rules, conversation and design before and after.
+
+use std::io::{BufRead, BufReader, Write};
+use std::net::{SocketAddr, TcpStream};
+use std::time::{Duration, Instant};
+
+use serde_json::{Value, json};
+
+/// Standard test v1, as #4851 states it. Changing a prompt or a rule makes it another test.
+const TASKS: &str = r#"[
+ {"id": "01-cube", "prompt": "Make a 20 mm cube.", "check": {"bodies": 1, "volume": 8000, "bbox_size": [20, 20, 20]}},
+ {"id": "02-cylinder", "prompt": "Make a cylinder 30 mm in diameter and 50 mm tall.", "check": {"bodies": 1, "volume": 35342.92, "bbox_size": [30, 30, 50]}},
+ {"id": "03-block", "prompt": "Create a block 60 mm long, 40 mm wide and 10 mm thick.", "check": {"bodies": 1, "volume": 24000, "bbox_size": [10, 40, 60]}},
+ {"id": "04-sketch-extrude", "prompt": "Sketch a 50 by 30 mm rectangle on the XY plane and extrude it 15 mm.", "check": {"bodies": 1, "volume": 22500, "bbox_size": [15, 30, 50], "feature_kinds": ["Extrude"]}},
+ {"id": "05-plate-hole", "prompt": "Make a plate 80 x 60 x 10 mm with a 20 mm diameter hole straight through the middle.", "check": {"bodies": 1, "volume": 44858.41, "bbox_size": [10, 60, 80], "note": "80*60*10 - pi*10^2*10"}},
+ {"id": "06-fillet-box", "prompt": "Round all the edges of the box with a 3 mm radius.", "setup": [{"command": "solid.box", "params": {"length": 40, "width": 40, "height": 40}}], "check": {"bodies": 1, "volume": 63108.53, "tol": 0.003, "feature_kinds": ["Fillet"], "note": "all 12 edges of a 40 mm cube rounded r=3"}},
+ {"id": "07-move-body", "prompt": "Move the box 50 mm along the X axis.", "setup": [{"command": "solid.box", "params": {"length": 20, "width": 20, "height": 20}}], "check": {"bodies": 1, "volume": 8000, "bbox_min": [50, 0, 0]}},
+ {"id": "08-rename-body", "prompt": "Rename the body to Bracket.", "setup": [{"command": "solid.box", "params": {"length": 30, "width": 20, "height": 10}}], "check": {"bodies": 1, "volume": 6000, "body_name": "Bracket"}},
+ {"id": "09-volume-question", "prompt": "What is the volume of the part?", "setup": [{"command": "solid.box", "params": {"length": 30, "width": 20, "height": 10}}], "check": {"bodies": 1, "volume": 6000, "unchanged": true, "reply_number": "6[ ,. ]?000(?![0-9])|6(\\.0+)? ?cm"}},
+ {"id": "10-two-holes", "prompt": "Make a 100 x 50 x 5 mm plate and drill two 10 mm holes through it, each 15 mm in from the short ends and centred across the width.", "check": {"bodies": 1, "volume": 24214.6, "bbox_size": [5, 50, 100], "note": "100*50*5 - 2*pi*5^2*5"}},
+ {"id": "11-any-change", "prompt": "can you use the software to make a change to the current shape, any change at all", "setup": [{"command": "solid.box", "params": {"length": 30, "width": 20, "height": 10}}], "check": {"changed": true, "note": "verbatim user prompt from #4846; pass = the design changed"}}
+]"#;
+
+/// The agent the test adds. A fixed id replaces it on every run instead of adding another.
+const AGENT_ID: &str = "api-bench-assistant";
+const AGENT_NAME: &str = "bench";
+/// Longest control reply read (the channel's own request limit).
+const MAX_REPLY: usize = 16 << 20;
+/// Most result data kept for `--json`.
+const MAX_RESULTS: usize = 64 << 20;
+/// Deadline of the requests that only set up or read (each is answered at once by the app).
+const REQUEST_TIMEOUT: Duration = Duration::from_secs(120);
+/// How often a running turn is polled.
+const POLL: Duration = Duration::from_millis(250);
+/// After this long, an idle chat with no reply yet counts as a turn that ended without one.
+const START_GRACE: Duration = Duration::from_secs(30);
+/// Absolute tolerance of the lowest corner, in millimetres.
+const MIN_CORNER_TOL: f64 = 0.05;
+
+/// A JSON-lines connection to the app. Requests are sent once; a lost reply is never retried,
+/// so nothing runs twice.
+struct Control {
+ addr: SocketAddr,
+ token: String,
+ reader: BufReader<TcpStream>,
+ id: u64,
+}
+
+impl Control {
+ fn connect(addr: &str, token: Option<&str>) -> Result<Self, String> {
+ let addr: SocketAddr = addr.parse().map_err(|_| format!("--connect: `{addr}` is not an IP address and port"))?;
+ if !addr.ip().is_loopback() || addr.port() == 0 {
+ return Err("--connect requires a loopback address and a nonzero port (the app listens on 127.0.0.1)".into());
+ }
+ let token = solidcraft_engine::control_auth::client_token(addr.port(), token)
+ .ok_or("no control token: pass --token, set SOLIDCRAFT_CONTROL_TOKEN, or run on the app's computer (its token file)")?;
+ if token.len() > 4096 {
+ return Err("the control token is longer than 4096 bytes".into());
+ }
+ Self::open(addr, token)
+ }
+
+ fn open(addr: SocketAddr, token: String) -> Result<Self, String> {
+ let stream = TcpStream::connect_timeout(&addr, Duration::from_secs(5)).map_err(|e| format!("connecting to {addr}: {e}"))?;
+ Ok(Self { addr, token, reader: BufReader::new(stream), id: 0 })
+ }
+
+ /// A new connection to the same app: a request that outlived its deadline keeps the old one
+ /// busy until it is answered.
+ fn reopen(&self) -> Result<Self, String> {
+ Self::open(self.addr, self.token.clone())
+ }
+
+ fn call(&mut self, method: &str, params: Value, deadline: Instant) -> Result<Value, String> {
+ let left = || deadline.checked_duration_since(Instant::now()).filter(|d| !d.is_zero()).ok_or_else(|| "timeout".to_string());
+ let io = |e: std::io::Error| match e.kind() {
+ std::io::ErrorKind::TimedOut | std::io::ErrorKind::WouldBlock => "timeout".to_string(),
+ _ => format!("control channel: {e}"),
+ };
+ self.reader.get_ref().set_write_timeout(Some(left()?)).map_err(io)?;
+ self.id += 1;
+ // The first request on a connection carries the token; sending it on every one is allowed.
+ let request = json!({"id": self.id, "method": method, "params": params, "token": self.token});
+ writeln!(self.reader.get_mut(), "{request}").map_err(io)?;
+ let mut line = Vec::new();
+ loop {
+ self.reader.get_ref().set_read_timeout(Some(left()?)).map_err(io)?;
+ let bytes = self.reader.fill_buf().map_err(io)?;
+ if bytes.is_empty() {
+ return Err("the app closed the control connection".into());
+ }
+ let newline = bytes.iter().position(|b| *b == b'\n');
+ let used = newline.map_or(bytes.len(), |n| n + 1);
+ if line.len() + used > MAX_REPLY {
+ return Err("a control reply is larger than 16 MiB".into());
+ }
+ line.extend_from_slice(bytes.get(..used).unwrap_or_default());
+ self.reader.consume(used);
+ if newline.is_some() {
+ break;
+ }
+ }
+ let reply: Value = serde_json::from_slice(&line).map_err(|e| format!("control reply: {e}"))?;
+ if reply["id"].as_u64() != Some(self.id) {
+ return Err("control reply for another request".into());
+ }
+ if reply["ok"] != true {
+ return Err(format!("{method}: {}", reply["error"].as_str().unwrap_or("failed")));
+ }
+ reply.get("result").cloned().ok_or_else(|| format!("{method}: reply without a result"))
+ }
+
+ fn execute(&mut self, command: &str, params: Value) -> Result<Value, String> {
+ self.call("engine.execute", json!({"command": command, "params": params, "animate": false}), Instant::now() + REQUEST_TIMEOUT)
+ }
+
+ fn chat(&mut self, params: Value) -> Result<Value, String> {
+ self.call("ui.agent_chat", params, Instant::now() + REQUEST_TIMEOUT)
+ }
+
+ fn inspect(&mut self) -> Result<Value, String> {
+ let design = self.call("document.inspect", json!({"measure": true}), Instant::now() + REQUEST_TIMEOUT)?;
+ if design.is_object() { Ok(design) } else { Err("document.inspect returned no design".into()) }
+ }
+}
+
+fn tasks() -> Result<Vec<Value>, String> {
+ serde_json::from_str(TASKS).map_err(|e| format!("embedded tasks: {e}"))
+}
+
+fn triple(value: &Value) -> Option<[f64; 3]> {
+ match value.as_array()?.as_slice() {
+ [x, y, z] => {
+ let n = |v: &Value| v.as_f64().filter(|n| n.is_finite());
+ Some([n(x)?, n(y)?, n(z)?])
+ }
+ _ => None,
+ }
+}
+
+/// What the model said (thoughts left out).
+fn reply(items: &[Value]) -> String {
+ items.iter().filter(|i| i["thought"] != true).filter_map(|i| i["agent"].as_str()).collect::<Vec<_>>().join("\n")
+}
+
+/// v1's reply pattern `6[ ,. ]?000(?![0-9])|6(\.0+)? ?cm`: "6000", "6 000", "6,000", "6.000" not
+/// followed by a digit, or "6 cm", "6.0cm", … (a narrow no-break space also separates thousands).
+fn volume_reply(text: &str) -> bool {
+ text.match_indices('6').any(|(at, _)| {
+ let tail = text.get(at + 1..).unwrap_or_default();
+ let thousands = |t: &str| t.strip_prefix("000").is_some_and(|rest| !rest.starts_with(|c: char| c.is_ascii_digit()));
+ let separated = tail.strip_prefix([' ', ',', '.', '\u{202f}']).is_some_and(thousands);
+ let cm = {
+ let t = tail.strip_prefix(".0").map_or(tail, |t| t.trim_start_matches('0'));
+ t.strip_prefix(' ').unwrap_or(t).starts_with("cm")
+ };
+ thousands(tail) || separated || cm
+ })
+}
+
+/// The design as the user sees it, without what reading or saving changes (path, saved state,
+/// revision, selection, undo depth, evaluation timings).
+fn content(design: &Value) -> Value {
+ let mut v = design.clone();
+ if let Some(o) = v.as_object_mut() {
+ for key in ["path", "dirty", "revision", "selection", "undo", "redo"] {
+ o.remove(key);
+ }
+ }
+ for feature in v.get_mut("timeline").and_then(Value::as_array_mut).into_iter().flatten() {
+ if let Some(o) = feature.as_object_mut() {
+ o.remove("ms");
+ o.remove("boolean_diagnostics");
+ }
+ }
+ v
+}
+
+/// Whether the design changed: its content, or a step on the undo stack (a colour, say, that the
+/// inspection does not list).
+fn changed(before: &Value, after: &Value) -> bool {
+ content(before) != content(after) || after["undo"].as_u64() > before["undo"].as_u64()
+}
+
+/// The rules a task's result breaks (none: it passed). `before` and `after` are
+/// `document.inspect {measure: true}` before the prompt and after the turn.
+pub(super) fn check(rule: &Value, before: &Value, after: &Value, items: &[Value]) -> Vec<String> {
+ let Some(all) = after["bodies"].as_array() else { return vec!["the design could not be read".into()] };
+ // Bodies the user sees; a hidden one (a tool body, a hidden original) is not the result.
+ let bodies: Vec<&Value> = all.iter().filter(|b| b["visible"] != false).collect();
+ let mut errors = Vec::new();
+ if let Some(n) = rule["bodies"].as_u64()
+ && bodies.len() as u64 != n
+ {
+ errors.push(format!("{} bodies, expected {n}", bodies.len()));
+ }
+ if let Some(expected) = rule["volume"].as_f64() {
+ let tol = rule["tol"].as_f64().unwrap_or(0.01);
+ let volumes: Vec<f64> = bodies.iter().filter_map(|b| b["volume_mm3"].as_f64()).filter(|v| v.is_finite()).collect();
+ let total: f64 = volumes.iter().sum();
+ if volumes.len() != bodies.len() || (total - expected).abs() > expected * tol {
+ errors.push(format!("volume {total:.1} mm³, expected {expected} mm³ ±{}%", tol * 100.0));
+ }
+ }
+ let first = bodies.first().copied().unwrap_or(&Value::Null);
+ let min = triple(&first["bbox"]["min"]);
+ if let Some(mut expected) = triple(&rule["bbox_size"]) {
+ expected.sort_by(f64::total_cmp);
+ let size = min.zip(triple(&first["bbox"]["max"])).map(|(lo, hi)| {
+ let mut size = [hi[0] - lo[0], hi[1] - lo[1], hi[2] - lo[2]];
+ size.sort_by(f64::total_cmp);
+ size
+ });
+ if !size.is_some_and(|s| s.iter().zip(expected).all(|(a, e)| (a - e).abs() <= 0.01 * e.max(1.0))) {
+ errors.push(format!("size {size:?}, expected {expected:?} in any orientation"));
+ }
+ }
+ if let Some(expected) = triple(&rule["bbox_min"])
+ && !min.is_some_and(|m| m.iter().zip(expected).all(|(a, e)| (a - e).abs() <= MIN_CORNER_TOL))
+ {
+ errors.push(format!("lowest corner {min:?}, expected {expected:?}"));
+ }
+ if let Some(name) = rule["body_name"].as_str()
+ && !bodies.iter().any(|b| b["name"] == name)
+ {
+ errors.push(format!("no body named {name}"));
+ }
+ for kind in rule["feature_kinds"].as_array().into_iter().flatten().filter_map(Value::as_str) {
+ let live = |f: &&Value| f["suppressed"] != true && f["rolled_back"] != true && f["error"].is_null();
+ if !after["timeline"].as_array().into_iter().flatten().filter(live).any(|f| f["type"].as_str().is_some_and(|t| t.contains(kind))) {
+ errors.push(format!("no {kind} in the timeline"));
+ }
+ }
+ if rule["changed"] == true || rule["unchanged"] == true {
+ let changed = before.is_object() && changed(before, after);
+ if rule["changed"] == true && !changed {
+ errors.push("the design did not change".into());
+ }
+ if rule["unchanged"] == true && (changed || !before.is_object()) {
+ errors.push("the design changed".into());
+ }
+ }
+ if rule["reply_number"].is_string() && !volume_reply(&reply(items)) {
+ errors.push("the reply does not give the volume".into());
+ }
+ errors
+}
+
+/// A tool call written into the reply as JSON: `{"name": …`, `{"command": …`, `{"function": …`,
+/// `{"tool": …`.
+fn call_as_text(text: &str) -> bool {
+ text.match_indices('{').any(|(at, _)| {
+ let rest = text.get(at + 1..).unwrap_or_default().trim_start();
+ ["\"name\"", "\"command\"", "\"function\"", "\"tool\""]
+ .iter()
+ .any(|key| rest.strip_prefix(key).is_some_and(|r| r.trim_start().starts_with(':')))
+ })
+}
+
+/// How a failed task failed, in the issue's words, from the conversation's items.
+pub(super) fn classify(items: &[Value], timed_out: bool) -> &'static str {
+ if timed_out {
+ return "timeout";
+ }
+ let tools: Vec<&Value> = items.iter().filter(|i| i.get("tool").is_some()).collect();
+ // A failed call, or a batch whose answer reports a failed step (`"ok": false`).
+ let failed: Vec<String> = tools
+ .iter()
+ .map(|t| t["output"].as_str().unwrap_or_default())
+ .zip(&tools)
+ .filter(|(output, t)| t["status"] == "Failed" || serde_json::from_str::<Value>(output).is_ok_and(|v| v["ok"] == false))
+ .map(|(output, _)| output.to_lowercase())
+ .collect();
+ let notices: Vec<String> = items.iter().filter_map(|i| i["notice"].as_str()).map(str::to_lowercase).collect();
+ let answer = reply(items);
+ if notices.iter().any(|n| n.contains("http 400")) {
+ return "the endpoint refused tools (HTTP 400)";
+ }
+ if notices.iter().any(|n| n.contains("max_turn_requests")) || answer.contains("requests for one message") {
+ return "hit the request limit";
+ }
+ if call_as_text(&answer) {
+ return "wrote the call as text";
+ }
+ if failed.iter().any(|o| o.contains("unknown command") || o.starts_with("unknown tool")) {
+ return "wrong or unknown command";
+ }
+ if failed.iter().any(|o| o.contains("invalid parameters") || o.starts_with("not run:") || o.contains("not valid json")) {
+ return "wrong arguments";
+ }
+ if !failed.is_empty() || notices.iter().any(|n| n.starts_with("http ")) || items.iter().any(|i| i["error"] == true) {
+ return "gave up after errors";
+ }
+ if tools.is_empty() {
+ return if answer.trim().is_empty() { "an empty reply" } else { "only talked" };
+ }
+ "wrong result"
+}
+
+/// One run's results in the issue's comment template (`run`: which repetition).
+pub(super) fn markdown(model: &str, version: &str, results: &[Value], run: u64) -> String {
+ let clean = |s: &str| s.replace(['\n', '\r', '|'], " ");
+ let rows: Vec<&Value> = results.iter().filter(|r| r["repeat"] == run).collect();
+ let passed = rows.iter().filter(|r| r["passed"] == true).count();
+ let mut out = format!(
+ "### {} (<quant>, <download size>)\nRuntime: <Ollama/LM Studio/... version>, context <num_ctx if changed>\nSolidCraft: {}\nResult: {passed}/11\n| # | pass/fail | how it failed |\n|---|---|---|\n",
+ clean(model),
+ clean(version)
+ );
+ for n in 1..=11 {
+ match rows.iter().find(|r| r["task"] == n) {
+ Some(r) if r["passed"] == true => out.push_str(&format!("| {n} | pass | |\n")),
+ Some(r) => out.push_str(&format!("| {n} | fail | {} |\n", clean(r["failure"].as_str().unwrap_or_default()))),
+ None => out.push_str(&format!("| {n} | | not run |\n")),
+ }
+ }
+ out.push_str(if rows.len() == 11 {
+ "Notes: <anything odd, e.g. very slow, needed a retry>\n"
+ } else {
+ "Notes: partial run; the tasks marked \"not run\" were left out.\n"
+ });
+ out
+}
+
+fn write(path: &str, text: &str) -> Result<(), String> {
+ solidcraft_engine::io::vfs::write(path, text.as_bytes()).map_err(|e| format!("{path}: {e}"))
+}
+
+struct Options<'a> {
+ connect: &'a str,
+ token: Option<&'a str>,
+ base_url: &'a str,
+ model: &'a str,
+ tasks: Vec<usize>,
+ repeat: u64,
+ timeout: Duration,
+ json: Option<&'a str>,
+ markdown: Option<&'a str>,
+}
+
+fn options(args: &[String]) -> Result<Options<'_>, String> {
+ const FLAGS: [&str; 9] = ["--connect", "--token", "--base-url", "--model", "--tasks", "--repeat", "--timeout-s", "--json", "--markdown"];
+ let mut given = std::collections::BTreeMap::new();
+ let mut args = args.iter();
+ while let Some(flag) = args.next() {
+ let Some(flag) = FLAGS.iter().find(|f| **f == flag.as_str()) else { return Err(format!("unknown bench-assistant option: {flag}")) };
+ let value = args
+ .next()
+ .filter(|v| !v.starts_with("--") && !v.is_empty() && v.len() <= 4096)
+ .ok_or_else(|| format!("{flag} needs a value (at most 4096 bytes)"))?;
+ if given.insert(*flag, value.as_str()).is_some() {
+ return Err(format!("{flag} given twice"));
+ }
+ }
+ let required = |key: &str| given.get(key).copied().ok_or_else(|| format!("bench-assistant needs {key}"));
+ let number = |key: &str, default: u64, max: u64| -> Result<u64, String> {
+ given
+ .get(key)
+ .map_or(Ok(default), |s| s.parse::<u64>().ok().filter(|n| (1..=max).contains(n)).ok_or_else(|| format!("{key} must be 1 to {max}")))
+ };
+ let tasks = match given.get("--tasks") {
+ None => (1..=11).collect(),
+ Some(list) => {
+ let mut tasks = Vec::new();
+ for n in list.split(',') {
+ let n = n
+ .trim()
+ .parse::<usize>()
+ .ok()
+ .filter(|n| (1..=11).contains(n))
+ .ok_or("--tasks takes task numbers 1 to 11, separated by commas")?;
+ if tasks.contains(&n) {
+ return Err(format!("--tasks lists task {n} twice"));
+ }
+ tasks.push(n);
+ }
+ tasks
+ }
+ };
+ Ok(Options {
+ connect: required("--connect")?,
+ token: given.get("--token").copied(),
+ base_url: required("--base-url")?,
+ model: required("--model")?,
+ tasks,
+ repeat: number("--repeat", 1, 100)?,
+ timeout: Duration::from_secs(number("--timeout-s", 300, 3600)?),
+ json: given.get("--json").copied(),
+ markdown: given.get("--markdown").copied(),
+ })
+}
+
+/// Waits for the turn the prompt started to end. Returns the chat's last state and whether the
+/// deadline passed first.
+fn wait_for_turn(control: &mut Control, mut chat: Value, began: Instant, end: Instant) -> Result<(Value, bool), String> {
+ let mut seen_busy = false;
+ loop {
+ let state = chat["state"].as_str().unwrap_or_default();
+ let turn = !chat["turn"].is_null();
+ let queued = chat["pending"].as_array().is_some_and(|p| !p.is_empty());
+ seen_busy |= state == "busy" || turn;
+ let idle = !matches!(state, "busy" | "starting") && !turn && !queued;
+ // A turn can start and end between two polls: then the conversation has its answer.
+ let answered = chat["items"].as_array().into_iter().flatten().any(|i| i.get("user").is_none());
+ if idle && (seen_busy || answered || matches!(state, "exited" | "auth" | "off") || began.elapsed() >= START_GRACE) {
+ return Ok((chat, false));
+ }
+ let Some(left) = end.checked_duration_since(Instant::now()).filter(|d| !d.is_zero()) else { return Ok((chat, true)) };
+ std::thread::sleep(POLL.min(left));
+ match control.call("ui.agent_chat", json!({}), end) {
+ Ok(next) => chat = next,
+ Err(e) if e == "timeout" => return Ok((chat, true)),
+ Err(e) => return Err(e),
+ }
+ }
+}
+
+pub fn run(args: &[String]) -> Result<(), String> {
+ let o = options(args)?;
+ let tasks = tasks()?;
+ let mut control = Control::connect(o.connect, o.token)?;
+ let help = control.call("ui.help", json!({}), Instant::now() + REQUEST_TIMEOUT)?;
+ let version = help["version"].as_str().filter(|s| !s.is_empty() && s.len() <= 256).ok_or("the app reported no version")?.to_string();
+ let config = control.chat(json!({
+ "features": true,
+ "add_agent": {"id": AGENT_ID, "name": AGENT_NAME, "kind": "api", "base_url": o.base_url, "model": o.model},
+ }))?;
+ if !config["agents"].as_array().into_iter().flatten().any(|a| a["id"] == AGENT_ID) {
+ return Err("the app did not add the benchmark's agent".into());
+ }
+ // Put the user's permission setting back afterwards.
+ let permissions = config["permissions"].as_str().map(str::to_string);
+ let result = run_tasks(&mut control, &o, &tasks, &version);
+ let _ = control.chat(json!({"stop": true}));
+ if let Some(p) = permissions {
+ let _ = control.chat(json!({"permissions": p}));
+ }
+ result
+}
+
+fn run_tasks(control: &mut Control, o: &Options, tasks: &[Value], version: &str) -> Result<(), String> {
+ let mut results: Vec<Value> = Vec::new();
+ let mut kept = 0;
+ let mut stdout = std::io::stdout().lock();
+ let mut say = |line: String| writeln!(stdout, "{line}").map_err(|e| format!("stdout: {e}"));
+ say(format!("bench-assistant v1: {} on SolidCraft {version}", o.model))?;
+ say("| run | # | task | pass/fail | how it failed | seconds |\n|---|---|---|---|---|---|".into())?;
+ for run in 1..=o.repeat {
+ for &n in &o.tasks {
+ let task = tasks.get(n - 1).ok_or("missing embedded task")?;
+ // A fresh design and a fresh chat, with nothing of the last task still running.
+ control.chat(json!({"stop": true}))?;
+ control.execute("file.new", json!({"name": "Benchmark"}))?;
+ for step in task["setup"].as_array().into_iter().flatten() {
+ control.execute(step["command"].as_str().ok_or("embedded setup without a command")?, step["params"].clone())?;
+ }
+ control.chat(json!({"agent": AGENT_ID, "permissions": "all", "new_chat": true, "attach_view": false}))?;
+ let before = control.inspect()?;
+ let began = Instant::now();
+ let end = began + o.timeout;
+ let (chat, timed_out) = match control.call("ui.agent_chat", json!({"send": task["prompt"]}), end) {
+ Ok(chat) => wait_for_turn(control, chat, began, end)?,
+ Err(e) if e == "timeout" => (json!({}), true),
+ Err(e) => return Err(e),
+ };
+ if timed_out {
+ // The late reply would arrive on the old connection; stop the model on a new one
+ // before reading the design.
+ *control = control.reopen()?;
+ }
+ // Stop the agent before reading the result, so nothing changes the design meanwhile.
+ let stopped = control.chat(json!({"stop": true}))?;
+ let items = stopped["items"].as_array().or(chat["items"].as_array()).cloned().unwrap_or_default();
+ let after = control.inspect()?;
+ let failed_rules = check(&task["check"], &before, &after, &items);
+ let passed = failed_rules.is_empty();
+ let failure = if passed { "" } else { classify(&items, timed_out) };
+ let seconds = began.elapsed().as_secs_f64();
+ say(format!(
+ "| {run} | {n} | {} | {} | {failure} | {seconds:.0} |",
+ task["id"].as_str().unwrap_or_default(),
+ if passed { "pass" } else { "fail" }
+ ))?;
+ let result = json!({
+ "task": n, "id": task["id"], "prompt": task["prompt"], "repeat": run, "passed": passed, "failure": failure,
+ "failed_rules": failed_rules, "timed_out": timed_out, "seconds": seconds, "check": task["check"],
+ "items": items, "before": before, "after": after,
+ });
+ kept += result.to_string().len();
+ if kept > MAX_RESULTS {
+ return Err("the results passed 64 MiB; run fewer tasks or repetitions".into());
+ }
+ results.push(result);
+ // Written after every task, so an interrupted run keeps what it measured.
+ if let Some(path) = o.json {
+ let all = json!({"test": "v1", "model": o.model, "base_url": o.base_url, "solidcraft": version, "results": results});
+ write(path, &serde_json::to_string_pretty(&all).map_err(|e| e.to_string())?)?;
+ }
+ if let Some(path) = o.markdown {
+ write(path, &(1..=run).map(|r| markdown(o.model, version, &results, r)).collect::<Vec<_>>().join("\n"))?;
+ }
+ }
+ }
+ for run in 1..=o.repeat {
+ let rows: Vec<&Value> = results.iter().filter(|r| r["repeat"] == run).collect();
+ say(format!("run {run}: {}/{} passed", rows.iter().filter(|r| r["passed"] == true).count(), rows.len()))?;
+ }
+ Ok(())
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ fn design(bodies: Value, timeline: Value) -> Value {
+ json!({"name": "Benchmark", "params": [], "timeline": timeline, "bodies": bodies, "sketches": [], "undo": 1, "revision": 3, "dirty": true})
+ }
+
+ fn body(name: &str, volume: f64, min: [f64; 3], max: [f64; 3]) -> Value {
+ json!({"name": name, "volume_mm3": volume, "bbox": {"min": min, "max": max}, "visible": true})
+ }
+
+ #[test]
+ fn the_embedded_tasks_are_v1() {
+ let tasks = tasks().unwrap();
+ assert_eq!(tasks.len(), 11);
+ assert_eq!(tasks[0]["prompt"], "Make a 20 mm cube.");
+ assert_eq!(tasks[10]["prompt"], "can you use the software to make a change to the current shape, any change at all");
+ assert_eq!(tasks[5]["check"]["tol"], 0.003);
+ assert_eq!(tasks[6]["check"]["bbox_min"], json!([50, 0, 0]));
+ let boxes: Vec<_> = tasks.iter().filter(|t| t["setup"].is_array()).map(|t| t["id"].as_str().unwrap()).collect();
+ assert_eq!(boxes, ["06-fillet-box", "07-move-body", "08-rename-body", "09-volume-question", "11-any-change"]);
+ }
+
+ #[test]
+ fn pass_rules() {
+ let tasks = tasks().unwrap();
+ let rule = |n: usize| tasks[n - 1]["check"].clone();
+ let empty = design(json!([]), json!([]));
+ let pass = |n: usize, before: &Value, after: &Value, items: &[Value]| check(&rule(n), before, after, items);
+ // 3: any orientation, within 1%.
+ let block = design(json!([body("Body1", 24_100.0, [0.0; 3], [40.0, 10.0, 60.2])]), json!([{"type": "Box"}]));
+ assert!(pass(3, &empty, &block, &[]).is_empty());
+ let thick = design(json!([body("Body1", 24_000.0, [0.0; 3], [40.0, 12.0, 60.0])]), json!([]));
+ assert_eq!(pass(3, &empty, &thick, &[]).len(), 1);
+ let two = design(json!([body("A", 12_000.0, [0.0; 3], [60.0, 40.0, 5.0]), body("B", 12_000.0, [0.0; 3], [60.0, 40.0, 5.0])]), json!([]));
+ assert!(!pass(3, &empty, &two, &[]).is_empty());
+ // A hidden body is not part of the result.
+ let mut hidden = two.clone();
+ hidden["bodies"][1]["visible"] = json!(false);
+ hidden["bodies"][0]["volume_mm3"] = json!(24_000.0);
+ hidden["bodies"][0]["bbox"]["max"] = json!([60.0, 40.0, 10.0]);
+ assert!(pass(3, &empty, &hidden, &[]).is_empty());
+ // 4: the timeline needs a working Extrude.
+ let plate = |kind: Value| design(json!([body("Body1", 22_500.0, [0.0; 3], [50.0, 30.0, 15.0])]), kind);
+ assert!(pass(4, &empty, &plate(json!([{"type": "Sketch"}, {"type": "Extrude"}])), &[]).is_empty());
+ assert!(!pass(4, &empty, &plate(json!([{"type": "Box"}])), &[]).is_empty());
+ for flag in [json!({"suppressed": true}), json!({"rolled_back": true}), json!({"error": "failed"})] {
+ let mut feature = json!({"type": "Extrude"});
+ feature.as_object_mut().unwrap().extend(flag.as_object().unwrap().clone());
+ assert!(!pass(4, &empty, &plate(json!([feature])), &[]).is_empty(), "{flag}");
+ }
+ // 6: the fillet's tolerance is 0.3%.
+ let start = design(json!([body("Body1", 64_000.0, [0.0; 3], [40.0; 3])]), json!([{"type": "Box"}]));
+ let filleted = |v: f64| design(json!([body("Body1", v, [0.0; 3], [40.0; 3])]), json!([{"type": "Box"}, {"type": "Fillet"}]));
+ assert!(pass(6, &start, &filleted(63_108.53 * 1.002), &[]).is_empty());
+ assert!(!pass(6, &start, &filleted(63_108.53 * 1.004), &[]).is_empty());
+ // 7: the lowest corner.
+ let moved = |x: f64| design(json!([body("Body1", 8000.0, [x, 0.0, 0.0], [x + 20.0, 20.0, 20.0])]), json!([]));
+ assert!(pass(7, &empty, &moved(50.0), &[]).is_empty());
+ assert!(!pass(7, &empty, &moved(0.0), &[]).is_empty());
+ // 8: the name.
+ let named = |name: &str| design(json!([body(name, 6000.0, [0.0; 3], [30.0, 20.0, 10.0])]), json!([]));
+ assert!(pass(8, &empty, &named("Bracket"), &[]).is_empty());
+ assert!(!pass(8, &empty, &named("Body1"), &[]).is_empty());
+ // 9: unchanged, and the reply gives the volume.
+ let start = named("Body1");
+ let said = |text: &str| [json!({"user": "What is the volume of the part?"}), json!({"agent": text, "thought": false})];
+ let mut read = start.clone();
+ read["revision"] = json!(7);
+ read["dirty"] = json!(false);
+ read["selection"] = json!([{"body": "Body1"}]);
+ assert!(pass(9, &start, &read, &said("The volume is 6000 mm³.")).is_empty());
+ assert!(!pass(9, &start, &read, &said("About 60 cm³.")).is_empty());
+ assert!(!pass(9, &start, &named("Bracket"), &said("6 000 mm³")).is_empty());
+ let mut undone = start.clone();
+ undone["undo"] = json!(2);
+ assert!(!pass(9, &start, &undone, &said("6000 mm³")).is_empty());
+ // 11: any change counts; reading does not.
+ assert!(pass(11, &start, &named("Bracket"), &[]).is_empty());
+ assert!(pass(11, &start, &undone, &[]).is_empty());
+ assert!(!pass(11, &start, &read, &[]).is_empty());
+ // An unreadable design passes nothing.
+ assert!(!pass(11, &start, &json!({}), &[]).is_empty());
+ }
+
+ #[test]
+ fn the_volume_reply() {
+ for yes in ["6000", "6 000 mm³", "6,000", "6.000", "6\u{202f}000", "6 cm³", "6.00cm3", "is 6000."] {
+ assert!(volume_reply(yes), "{yes}");
+ }
+ for no in ["60000", "6,0001", "60 cm³", "600", "6.01 cm³", ""] {
+ assert!(!volume_reply(no), "{no}");
+ }
+ }
+
+ #[test]
+ fn failure_categories() {
+ for (items, expected) in [
+ (
+ json!([{"user": "Make a 20 mm cube."}, {"agent": "{\"name\": \"execute\", \"arguments\": {}}", "thought": false}]),
+ "wrote the call as text",
+ ),
+ (
+ json!([{"notice": "HTTP 400: registry.ollama.ai/library/gemma3:4b does not support tools", "error": true}]),
+ "the endpoint refused tools (HTTP 400)",
+ ),
+ (json!([{"user": "Make a 20 mm cube."}, {"agent": "", "thought": false}]), "an empty reply"),
+ (json!([{"agent": "Let me think", "thought": true}]), "an empty reply"),
+ (json!([{"agent": "Sure, here is how to make a cube.", "thought": false}]), "only talked"),
+ (
+ json!([{"tool": "1", "status": "Failed", "output": "unknown command `solid.cube` (did you mean `solid.box`?)"}]),
+ "wrong or unknown command",
+ ),
+ (json!([{"tool": "1", "status": "Failed", "output": "unknown tool `make_cube`"}]), "wrong or unknown command"),
+ (json!([{"tool": "1", "status": "Failed", "output": "invalid parameters for `solid.box`: `length` is required"}]), "wrong arguments"),
+ (json!([{"tool": "1", "status": "Failed", "output": "Not run: the tool arguments were not valid JSON"}]), "wrong arguments"),
+ (json!([{"tool": "1", "status": "Failed", "output": "no body `Box`"}]), "gave up after errors"),
+ (
+ json!([{"tool": "1", "status": "Completed", "output": "{\"ok\": false, \"failed\": {\"error\": \"unknown command `solid.delete`\"}}"}]),
+ "wrong or unknown command",
+ ),
+ (
+ json!([{"tool": "1", "status": "Completed"}, {"agent": "\n(stopped after 24 requests for one message; send another message to go on)"}]),
+ "hit the request limit",
+ ),
+ (json!([{"tool": "1", "status": "Completed", "output": "{}"}]), "wrong result"),
+ ] {
+ let items = items.as_array().unwrap();
+ assert_eq!(classify(items, false), expected, "{items:?}");
+ assert_eq!(classify(items, true), "timeout");
+ }
+ }
+
+ #[test]
+ fn markdown_is_the_comment_template() {
+ let rows: Vec<Value> =
+ (1..=11).map(|n| json!({"task": n, "repeat": 1, "passed": n != 2, "failure": if n == 2 { "an empty reply" } else { "" }})).collect();
+ let report = markdown("qwen2.5:7b", "0.19.0", &rows, 1);
+ assert!(report.starts_with(
+ "### qwen2.5:7b (<quant>, <download size>)\nRuntime: <Ollama/LM Studio/... version>, context <num_ctx if changed>\nSolidCraft: 0.19.0\nResult: 10/11\n| # | pass/fail | how it failed |\n|---|---|---|\n| 1 | pass | |\n| 2 | fail | an empty reply |\n"
+ ));
+ assert!(report.ends_with("| 11 | pass | |\nNotes: <anything odd, e.g. very slow, needed a retry>\n"));
+ let partial = markdown("m|x", "v", &rows[..1], 1);
+ assert!(partial.starts_with("### m x ("));
+ assert!(partial.contains("Result: 1/11") && partial.contains("| 2 | | not run |") && partial.contains("partial run"));
+ }
+
+ #[test]
+ fn options_are_checked_before_connecting() {
+ for extra in [
+ &["--repeat", "0"][..],
+ &["--repeat", "101"],
+ &["--timeout-s", "3601"],
+ &["--tasks", "0"],
+ &["--tasks", "12"],
+ &["--tasks", "1,1"],
+ &["--model", "again"],
+ &["--unexpected", "1"],
+ &["--token"],
+ ] {
+ let mut args: Vec<String> =
+ ["--connect", "127.0.0.1:9", "--model", "m", "--base-url", "http://127.0.0.1:11434/v1"].iter().map(|s| s.to_string()).collect();
+ args.extend(extra.iter().map(|s| s.to_string()));
+ assert!(options(&args).is_err(), "{extra:?}");
+ }
+ let args: Vec<String> =
+ ["--connect", "127.0.0.1:9", "--model", "m", "--base-url", "u", "--tasks", "2, 5"].iter().map(|s| s.to_string()).collect();
+ let o = options(&args).unwrap();
+ assert_eq!((o.tasks, o.repeat, o.timeout), (vec![2, 5], 1, Duration::from_secs(300)));
+ assert!(Control::connect("10.0.0.1:9271", Some("t")).is_err());
+ assert!(Control::connect("localhost", Some("t")).is_err());
+ }
+}
diff --git a/apps/solidcraft-cli/src/main.rs b/apps/solidcraft-cli/src/main.rs
index e122683e48..9b4b06e90b 100644
--- a/apps/solidcraft-cli/src/main.rs
+++ b/apps/solidcraft-cli/src/main.rs
@@ -6,6 +6,9 @@
//! solidcraft-cli script <design.solidcraft> [--out script.json] the design's history as a command script
//! solidcraft-cli bench-open <design.solidcraft | part.step> [--parse-only] [--save OUT.solidcraft] [--display LEVEL [--edit NAME EXPR]] [--ops script.json] loading timings as JSON lines
//! solidcraft-cli bench-agent [--scale F] [--save F] [--script-out F] a ~1,300-body design built command by command, timed
+//! solidcraft-cli bench-assistant --connect 127.0.0.1:PORT [--token T] --base-url URL --model NAME [--tasks 1,2,5] [--repeat N]
+//! [--timeout-s 300] [--json out.json] [--markdown out.md]
+//! the standard Assistant model test (#4851) on a running app
//! solidcraft-cli snapshot <script|design> [--in design|part.step] --out shot.png [--width W] [--height H] [--view iso|front|top|…|NAME | --camera JSON] [--no-grid] [--no-axes]
//! [--body NAME:opacity=0.3,color=#rrggbb]... [--tool NAME]... [--section z=27]
//! solidcraft-cli exec <command> [json-params] run one command on an empty design
@@ -24,6 +27,7 @@
#![forbid(unsafe_code)]
mod bench_agent;
+mod bench_assistant;
mod crashes;
mod fuzz;
mod logger;
@@ -57,6 +61,7 @@ const USAGE: &str = "usage:
(--ops: after opening, run each command of the script and undo it, timing the command, the
evaluations it caused and the view it leaves: placing and meshing what changed)
solidcraft-cli bench-agent [--scale F] [--save design.solidcraft] [--script-out script.json]
+ solidcraft-cli bench-assistant --connect 127.0.0.1:PORT [--token T] --base-url URL --model NAME [--tasks 1,2,5] [--repeat N] [--timeout-s 300] [--json out.json] [--markdown out.md]
solidcraft-cli snapshot <script|design> [--in design.solidcraft|part.step] --out shot.png [--width W] [--height H] [--view NAME | --camera JSON] [--no-grid] [--no-axes]
[--body NAME:opacity=0.3,color=#rrggbb]... [--tool NAME]... [--section z=27]
solidcraft-cli snapshot --in <design.solidcraft|part.step> --out shot.png ...
@@ -136,6 +141,8 @@ fn main() -> ExitCode {
Some("bench-open") => cmd_bench_open(&args[1..]),
// A ~1,300-body design built command by command, as an agent builds one over MCP.
Some("bench-agent") => bench_agent::run(&args[1..]),
+ // The standard Assistant model test (#4851), run against a desktop app over its control channel.
+ Some("bench-assistant") => bench_assistant::run(&args[1..]),
Some("--version" | "-V") => {
println!("solidcraft-cli {}", env!("CARGO_PKG_VERSION"));
Ok(())
diff --git a/docs/agent-chat.md b/docs/agent-chat.md
index e17831886c..df2955285d 100644
--- a/docs/agent-chat.md
+++ b/docs/agent-chat.md
@@ -319,3 +319,36 @@ cargo build --release -p solidcraft-ui-egui --example acp_test_agent
Then add it under Preferences → AI → **Add agent… → Custom command** with the path of
`target/release/examples/acp_test_agent`. Put "save" in your message to see an Allow / Deny card,
or "ask" to see the agent ask its own permission.
+
+## Benchmarking the Assistant
+
+`solidcraft-cli bench-assistant` runs the eleven tasks of the
+[standard model test (v1, #4851)](https://github.com/tangency-labs/solidcraft/issues/4851) against a
+desktop app that is already running with a control port, and checks each result on the design
+itself. Use it to see how well a local model drives SolidCraft before you post its results.
+
+```sh
+solidcraft --control 9271
+# In another terminal:
+solidcraft-cli bench-assistant --connect 127.0.0.1:9271 \
+ --base-url http://localhost:11434/v1 --model qwen2.5:7b \
+ --json results.json --markdown results.md
+```
+
+Each task replaces the open design with a new one, so save your work first. The command adds the
+endpoint as an API agent named "bench", starts a new chat with All permissions for every task,
+sends the prompt once, and waits for the turn to end. It runs only the setup the test prescribes
+(the box some tasks start from) and does none of the modelling itself. When it finishes it stops
+the agent and puts your permission setting back.
+
+- The control token comes from `--token`, `SOLIDCRAFT_CONTROL_TOKEN`, or the app's
+ [token file](control-protocol.md#token), in that order.
+- `--tasks 1,2,5` runs only those tasks, `--repeat N` runs the test N times, and `--timeout-s`
+ (default 300) is each turn's deadline; a turn that runs past it is stopped and fails as "timeout".
+- A failed task gets one of the issue's failure categories ("wrote the call as text", "wrong or
+ unknown command", "wrong arguments", "the endpoint refused tools", "an empty reply", "only
+ talked", "gave up after errors", or "hit the request limit"), or "wrong result" when the calls
+ worked but the design is not what the task asks for.
+- `--markdown` writes the issue's comment template filled in; add the quantization, download
+ size, and runtime before you post it. `--json` keeps each task's failed rules, conversation, and
+ design before and after. Both files are rewritten after every task.