Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

fix(cua-driver): accept scroll dx/dy and handle get_window_state in run_actions (#4831)

Francesco Bonacci5ac4ec763b04
diff --git a/libs/cua-driver/rust/Skills/cua-driver/WORKFLOW.md b/libs/cua-driver/rust/Skills/cua-driver/WORKFLOW.md
index 543c7baae8..2babd4ebed 100644
--- a/libs/cua-driver/rust/Skills/cua-driver/WORKFLOW.md
+++ b/libs/cua-driver/rust/Skills/cua-driver/WORKFLOW.md
@@ -69,7 +69,7 @@ cua-driver click '{"target":{"kind":"window","pid":844,"window_id":10725},"eleme
| Focus a visible field and type | `type_text` with `x,y,text`, when that form is advertised |
| Replace an exposed native value | `set_value` with `pid,element_token,value` (its own schema, no `target`) |
| Send a key or combination | `press_key` with `key`, or `hotkey` with `keys` |
-| Scroll / drag | Inspect `describe scroll` / `describe drag`; units and supported shapes matter |
+| Scroll / drag | `scroll` takes `direction` and `amount` (wheel notches); signed `dx`/`dy` is converted (positive `dy` is down; up to 50 is notches, more is pixels at 100 px per notch). Inspect `describe drag` |
Text insertion and value replacement are different intents. Setting a field does not prove a form submission, navigation, or rename committed. Open a collapsed search/input control and reobserve before typing into it; one focus-click may not both open and focus it. Inspect the existing value/selection before replacing content.
@@ -90,7 +90,7 @@ cua-driver run_actions '{"session":"run-1","steps":[
],"delay_ms":100,"observe":true}'
```
-- `tool` is one of `click`, `double_click`, `right_click`, `set_value`, `type_text`, `press_key`, `hotkey`, `scroll`, `drag` (`press` and `type` are accepted for `press_key` and `type_text`); `args` are exactly that tool's arguments. Observation tools cannot run inside a batch: use `observe`. Run `describe run_actions` and `describe <tool>` for schemas. Up to 32 steps.
+- `tool` is one of `click`, `double_click`, `right_click`, `set_value`, `type_text`, `press_key`, `hotkey`, `scroll`, `drag` (`press` and `type` are accepted for `press_key` and `type_text`); `args` are exactly that tool's arguments. Observation tools cannot run inside a batch: use `observe`. A `get_window_state` as the last step, with no `observe`, is taken as the observation. Run `describe run_actions` and `describe <tool>` for schemas. Up to 32 steps.
- Every step is validated before the first runs, so a malformed step changes nothing. Each step then passes the same session, permission, capability-manifest and approval checks as a direct call; a batch grants nothing a single call lacks, and a refused step ends the batch like any other failure.
- A batch has one session. Set `session` on `run_actions`; a step may repeat it but not name another.
- `observe` is optional and reads once, after the last executed step (also after a failure). Pass `true` or `get_window_state` arguments; `pid`/`window_id` come from the last step that names both. Defaults: `since:"latest"` (only what changed since your last read of that window with the same view; a full read if there is none), `include_screenshot:false`, `max_elements:250`. Pass `include_screenshot:true` to see the window, or `since:null` for a full read. Omit `observe` to read nothing.
diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/batch_tools.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/batch_tools.rs
index 4ad16c1453..368b164a17 100644
--- a/libs/cua-driver/rust/crates/cua-driver-core/src/batch_tools.rs
+++ b/libs/cua-driver/rust/crates/cua-driver-core/src/batch_tools.rs
@@ -37,6 +37,17 @@ pub const BATCHABLE_TOOLS: &[&str] = &[
"drag",
];
+/// Read-only tools models put in a batch; refused with a pointer to `observe`.
+const OBSERVATION_TOOLS: &[&str] = &[
+ OBSERVE_TOOL,
+ "get_desktop_state",
+ "get_accessibility_tree",
+ "get_browser_state",
+ "list_windows",
+ "verify_state",
+ "zoom",
+];
+
/// Most steps one call may carry. Keeps a batch inside client call timeouts.
pub const MAX_STEPS: usize = 32;
/// Longest pause between two steps.
@@ -233,14 +244,40 @@ impl Plan {
.and_then(Value::as_str)
.map(str::to_owned);
+ // A trailing get_window_state step is what `observe` is for: take it as
+ // the observation when the batch has none.
+ let mut raw_steps = raw_steps.as_slice();
+ let mut trailing_observe = None;
+ if let [actions @ .., last] = raw_steps {
+ let observe_given = !matches!(
+ object.get("observe"),
+ None | Some(Value::Null) | Some(Value::Bool(false))
+ );
+ if !actions.is_empty()
+ && !observe_given
+ && last.get("tool").and_then(Value::as_str) == Some(OBSERVE_TOOL)
+ {
+ trailing_observe = Some(
+ last.get("args")
+ .cloned()
+ .filter(Value::is_object)
+ .unwrap_or_else(|| Value::Object(Map::new())),
+ );
+ raw_steps = actions;
+ }
+ }
+
let mut steps = Vec::with_capacity(raw_steps.len());
for (index, raw) in raw_steps.iter().enumerate() {
steps.push(parse_step(registry, index, raw, session.as_deref())?);
}
- let observe = match object.get("observe") {
- None | Some(Value::Null) | Some(Value::Bool(false)) => None,
- Some(value) => Some(parse_observe(registry, value, &steps, session.as_deref())?),
+ let observe = match (trailing_observe.as_ref(), object.get("observe")) {
+ (Some(value), _) => Some(parse_observe(registry, value, &steps, session.as_deref())?),
+ (None, None | Some(Value::Null) | Some(Value::Bool(false))) => None,
+ (None, Some(value)) => {
+ Some(parse_observe(registry, value, &steps, session.as_deref())?)
+ }
};
Ok(Self {
steps,
@@ -378,13 +415,21 @@ fn parse_step(
.copied()
.find(|candidate| *candidate == name)
.ok_or_else(|| {
- PlanError::step(
- index,
+ let message = if OBSERVATION_TOOLS.contains(&name) {
+ format!(
+ "`{name}` reads state and a batch only acts. Put get_window_state arguments \
+ in `observe` (one read after the last step; a get_window_state as the \
+ last step is taken as `observe`), or split the batch where you need to look. \
+ Batchable tools: {}",
+ BATCHABLE_TOOLS.join(", ")
+ )
+ } else {
format!(
"`{name}` cannot run in a batch; allowed tools: {}",
BATCHABLE_TOOLS.join(", ")
- ),
- )
+ )
+ };
+ PlanError::step(index, message)
})?;
let mut args = match object.get("args") {
None | Some(Value::Null) => Value::Object(Map::new()),
@@ -392,6 +437,8 @@ fn parse_step(
Some(_) => return Err(PlanError::step(index, "`args` must be an object")),
};
prepare_args(&mut args, session).map_err(|message| PlanError::step(index, message))?;
+ crate::tool::normalize_argument_aliases(tool, &mut args)
+ .map_err(|message| PlanError::step(index, format!("{tool}: {message}")))?;
validate_against_schema(registry, tool, &args)
.map_err(|message| PlanError::step(index, format!("{tool}: {message}")))?;
Ok(Step { tool, args })
diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/batch_tools/tests.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/batch_tools/tests.rs
index 7ac21d3b81..962fd4c3a6 100644
--- a/libs/cua-driver/rust/crates/cua-driver-core/src/batch_tools/tests.rs
+++ b/libs/cua-driver/rust/crates/cua-driver-core/src/batch_tools/tests.rs
@@ -66,6 +66,10 @@ impl Harness {
"fail": {"type": "boolean"},
"include_screenshot": {"type": "boolean"},
"since": {"type": "string"},
+ "direction": {"type": "string", "enum": ["up", "down", "left", "right"]},
+ "amount": {"type": "integer", "minimum": 1, "maximum": 50},
+ "x": {"type": "number"},
+ "y": {"type": "number"},
"full_output": {"type": "boolean"},
"max_elements": {"type": "integer", "minimum": 1}
},
@@ -202,8 +206,8 @@ async fn validates_every_step_before_running_any() {
let harness = Harness::new();
let cases = [
(
- json!({"tool": "get_window_state", "args": {}}),
- "cannot run in a batch",
+ json!({"tool": "zoom", "args": {}}),
+ "reads state and a batch only acts",
),
(
json!({"tool": "run_actions", "args": {}}),
@@ -456,3 +460,86 @@ async fn common_tool_name_slips_map_to_the_batchable_tool() {
.await;
assert_eq!(refused.is_error, Some(true));
}
+
+#[tokio::test]
+async fn scroll_steps_accept_dx_dy_like_a_direct_call() {
+ let harness = Harness::new();
+ // The live-check shape: {"dy": 500} in a batch step.
+ let result = harness
+ .run(json!({"steps": [
+ {"tool": "scroll", "args": {"pid": 1, "window_id": 2, "x": 600, "y": 500, "dy": 500}},
+ {"tool": "scroll", "args": {"pid": 1, "window_id": 2, "x": 600, "y": 500, "dx": "-4"}},
+ ]}))
+ .await;
+ assert_ne!(result.is_error, Some(true), "{}", text(&result));
+ let calls = harness.calls["scroll"].lock().unwrap().clone();
+ assert_eq!(calls.len(), 2);
+ assert_eq!(
+ (calls[0]["direction"].clone(), calls[0]["amount"].clone()),
+ (json!("down"), json!(5))
+ );
+ assert_eq!(
+ (calls[1]["direction"].clone(), calls[1]["amount"].clone()),
+ (json!("left"), json!(4))
+ );
+ assert!(calls
+ .iter()
+ .all(|c| c.get("dy").is_none() && c.get("dx").is_none()));
+
+ let refused = harness
+ .run(json!({"steps": [{"tool": "scroll", "args": {"pid": 1, "dx": 1, "dy": 1}}]}))
+ .await;
+ assert_eq!(refused.is_error, Some(true));
+ assert!(
+ text(&refused).contains("one axis per call"),
+ "{}",
+ text(&refused)
+ );
+ assert_eq!(harness.hits("scroll"), 2, "a refused batch runs nothing");
+}
+
+#[tokio::test]
+async fn a_trailing_read_becomes_the_observation_and_a_middle_one_points_to_observe() {
+ let harness = Harness::new();
+ // The v035 / live-check shape: act, act, then get_window_state as a step.
+ let result = harness
+ .run(json!({"steps": [
+ {"tool": "click", "args": {"pid": 7, "window_id": 3}},
+ {"tool": "click", "args": {"pid": 7, "window_id": 3, "text": "x"}},
+ {"tool": "get_window_state", "args": {"pid": 7, "window_id": 3, "max_elements": 70}},
+ ]}))
+ .await;
+ assert_ne!(result.is_error, Some(true), "{}", text(&result));
+ let structured = result.structured_content.unwrap();
+ assert_eq!(structured["total"], 2, "the read is not counted as a step");
+ assert_eq!(structured["observation"]["ok"], true);
+ let observed = harness.last("get_window_state");
+ assert_eq!(observed["max_elements"], 70);
+ assert_eq!(observed["since"], "latest");
+ assert_eq!(harness.hits("click"), 2);
+
+ let refused = harness
+ .run(json!({"steps": [
+ {"tool": "click", "args": {"pid": 7, "window_id": 3}},
+ {"tool": "get_window_state", "args": {"pid": 7, "window_id": 3}},
+ {"tool": "click", "args": {"pid": 7, "window_id": 3}},
+ ]}))
+ .await;
+ assert_eq!(refused.is_error, Some(true));
+ let message = text(&refused);
+ assert!(message.contains("step 2 of the batch"), "{message}");
+ assert!(
+ message.contains("Put get_window_state arguments in `observe`"),
+ "{message}"
+ );
+
+ // With an explicit observe, a trailing read is not silently dropped.
+ let both = harness
+ .run(json!({"steps": [
+ {"tool": "click", "args": {"pid": 7, "window_id": 3}},
+ {"tool": "get_window_state", "args": {"pid": 7, "window_id": 3}},
+ ], "observe": true}))
+ .await;
+ assert_eq!(both.is_error, Some(true));
+ assert_eq!(harness.hits("get_window_state"), 1);
+}
diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/tool.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/tool.rs
index 87021e4025..5f68877d2c 100644
--- a/libs/cua-driver/rust/crates/cua-driver-core/src/tool.rs
+++ b/libs/cua-driver/rust/crates/cua-driver-core/src/tool.rs
@@ -235,6 +235,79 @@ fn unknown_argument(def: &ToolDef, args: &Value) -> Option<String> {
))
}
+/// Rewrite argument spellings models guess into the advertised ones, before
+/// the closed schema is checked. Shared by dispatch and `run_actions` step
+/// validation, so a batch step accepts what a direct call accepts. `Err` is a
+/// refusal detail for an alias that cannot be translated.
+pub(crate) fn normalize_argument_aliases(tool_name: &str, args: &mut Value) -> Result<(), String> {
+ normalize_zoom_args(tool_name, args);
+ normalize_scroll_args(tool_name, args)
+}
+
+/// A larger `dx`/`dy` magnitude than this many notches is read as pixels.
+const SCROLL_MAX_NOTCHES: f64 = 50.0;
+/// Pixels per wheel notch when `dx`/`dy` is given in pixels (a Chromium
+/// wheel notch scrolls about 100 px).
+const SCROLL_PIXELS_PER_NOTCH: f64 = 100.0;
+
+/// `scroll` takes `direction` plus `amount` (wheel notches) on every platform.
+/// Models reach for signed `dx`/`dy` deltas (sometimes as strings), so
+/// translate them: positive `dy` scrolls down and positive `dx` right, as in
+/// DOM `scrollBy` and wheel `deltaY`. A magnitude up to 50 counts notches; a
+/// larger one is pixels at 100 px per notch, clamped to 50 notches. An
+/// explicit `direction` (and `amount`) wins over `dx`/`dy`.
+fn normalize_scroll_args(tool_name: &str, args: &mut Value) -> Result<(), String> {
+ if tool_name != "scroll" {
+ return Ok(());
+ }
+ let Some(arguments) = args.as_object_mut() else {
+ return Ok(());
+ };
+ let read = |value: Option<Value>| -> Result<f64, String> {
+ match value {
+ None | Some(Value::Null) => Ok(0.0),
+ Some(value) => value
+ .as_f64()
+ .or_else(|| value.as_str().and_then(|text| text.trim().parse().ok()))
+ .filter(|number: &f64| number.is_finite())
+ .ok_or_else(|| format!("dx/dy must be numbers, got {value}")),
+ }
+ };
+ if !arguments.contains_key("dx") && !arguments.contains_key("dy") {
+ return Ok(());
+ }
+ let dx = read(arguments.remove("dx"))?;
+ let dy = read(arguments.remove("dy"))?;
+ if arguments.contains_key("direction") {
+ return Ok(());
+ }
+ let (direction, delta) = match (dx != 0.0, dy != 0.0) {
+ (true, true) => {
+ return Err(format!(
+ "scroll moves along one axis per call, got dx={dx} and dy={dy}. Pass only dx or dy (or direction and amount), or use two scroll steps in run_actions."
+ ))
+ }
+ (false, false) => {
+ return Err("dx and dy are both 0, so there is nothing to scroll. Pass direction (up, down, left or right) and amount (wheel notches)."
+ .to_owned())
+ }
+ (false, true) => (if dy > 0.0 { "down" } else { "up" }, dy),
+ (true, false) => (if dx > 0.0 { "right" } else { "left" }, dx),
+ };
+ let magnitude = delta.abs();
+ let notches = if magnitude <= SCROLL_MAX_NOTCHES {
+ magnitude
+ } else {
+ magnitude / SCROLL_PIXELS_PER_NOTCH
+ };
+ let notches = notches.round().clamp(1.0, SCROLL_MAX_NOTCHES) as u64;
+ arguments.insert("direction".to_owned(), Value::String(direction.to_owned()));
+ arguments
+ .entry("amount")
+ .or_insert_with(|| serde_json::json!(notches));
+ Ok(())
+}
+
/// `zoom` takes a region as two corners (`x1,y1`-`x2,y2`). Models reach for
/// `x, y, width, height` (and pass numbers as strings), so rewrite that shape
/// into corners before the closed schema is checked. Explicit corners win.
@@ -1358,7 +1431,15 @@ impl ToolRegistry {
// Normalize deprecated public argument spellings before any policy,
// consent, recording, or implementation layer interprets the call.
normalize_delivery_mode_args(tool.def(), &mut args);
- normalize_zoom_args(resolved_name, &mut args);
+ if let Err(detail) = normalize_argument_aliases(resolved_name, &mut args) {
+ return ToolResult::error(format!("{resolved_name}: {detail}")).with_structured(
+ serde_json::json!({
+ "code": "invalid_arguments",
+ "tool": resolved_name,
+ "detail": detail,
+ }),
+ );
+ }
let unknown_argument = unknown_argument(tool.def(), &args);
if let Err(result) = crate::action_target::normalize_action_target(resolved_name, &mut args)
{
@@ -6541,7 +6622,7 @@ mod capability_tests {
#[cfg(test)]
mod argument_shape_tests {
- use super::{normalize_zoom_args, unknown_argument, ToolDef};
+ use super::{normalize_argument_aliases, normalize_zoom_args, unknown_argument, ToolDef};
use serde_json::json;
fn zoom_def() -> ToolDef {
@@ -6594,6 +6675,64 @@ mod argument_shape_tests {
assert_eq!(other_tool, json!({"x": 1, "width": 2}));
}
+ fn scroll(args: serde_json::Value) -> Result<serde_json::Value, String> {
+ let mut args = args;
+ normalize_argument_aliases("scroll", &mut args).map(|()| args)
+ }
+
+ #[test]
+ fn scroll_translates_signed_deltas_into_direction_and_notches() {
+ // Shapes models sent in the v035 bench and the live check of #4812.
+ let base = json!({"pid": 1, "window_id": 2, "x": 450, "y": 500});
+ let with = |extra: serde_json::Value| {
+ let mut args = base.clone();
+ args.as_object_mut()
+ .unwrap()
+ .extend(extra.as_object().unwrap().clone());
+ args
+ };
+ for (delta, direction, amount) in [
+ (json!({"dy": "10"}), "down", 10),
+ (json!({"dy": 20}), "down", 20),
+ (json!({"dy": -3}), "up", 3),
+ (json!({"dy": 500}), "down", 5),
+ (json!({"dy": "1500"}), "down", 15),
+ (json!({"dy": "2000"}), "down", 20),
+ (json!({"dy": 99999}), "down", 50),
+ (json!({"dy": 0.2}), "down", 1),
+ (json!({"dx": 5, "dy": 0}), "right", 5),
+ (json!({"dx": "-120"}), "left", 1),
+ ] {
+ let out = scroll(with(delta.clone())).unwrap();
+ assert_eq!(out["direction"], direction, "{delta}");
+ assert_eq!(out["amount"], amount, "{delta}");
+ assert!(
+ out.get("dx").is_none() && out.get("dy").is_none(),
+ "{delta}"
+ );
+ assert_eq!(out["x"], 450);
+ }
+ }
+
+ #[test]
+ fn scroll_keeps_explicit_direction_and_amount_and_refuses_what_it_cannot_translate() {
+ let out = scroll(json!({"direction": "up", "amount": 2, "dy": 900})).unwrap();
+ assert_eq!(out, json!({"direction": "up", "amount": 2}));
+ let out = scroll(json!({"dy": 400, "amount": 7})).unwrap();
+ assert_eq!(out, json!({"direction": "down", "amount": 7}));
+ let both = scroll(json!({"dx": 3, "dy": 4})).unwrap_err();
+ assert!(both.contains("one axis per call"), "{both}");
+ let zero = scroll(json!({"dx": 0, "dy": "0"})).unwrap_err();
+ assert!(zero.contains("nothing to scroll"), "{zero}");
+ let junk = scroll(json!({"dy": "lots"})).unwrap_err();
+ assert!(junk.contains("must be numbers"), "{junk}");
+ let untouched = json!({"direction": "down", "amount": 3});
+ assert_eq!(scroll(untouched.clone()).unwrap(), untouched);
+ let mut click = json!({"dy": 4});
+ normalize_argument_aliases("click", &mut click).unwrap();
+ assert_eq!(click, json!({"dy": 4}));
+ }
+
#[test]
fn unknown_arguments_are_all_named_with_the_accepted_ones() {
let detail = unknown_argument(
diff --git a/scripts/ci/cua-driver-live-smoke/smoke.py b/scripts/ci/cua-driver-live-smoke/smoke.py
index 4f5cfb060f..9e7b7d34bd 100644
--- a/scripts/ci/cua-driver-live-smoke/smoke.py
+++ b/scripts/ci/cua-driver-live-smoke/smoke.py
@@ -481,7 +481,7 @@ def token_without_pid_check(smoke):
def lean_read_ergonomics_check(smoke):
- check = "since:latest, bare row token, run_actions observe:true, zoom x/y/width/height"
+ check = "since:latest, bare row token, run_actions observe:true, zoom x/y/width/height, scroll dy"
problems = []
smoke.read()
latest = smoke.read(since="latest")
@@ -517,11 +517,16 @@ def lean_read_ergonomics_check(smoke):
"x": "0", "y": "0", "width": "120", "height": "60"})
if zoom.is_error and "unknown argument" in zoom.text:
problems.append(f"zoom x/y/width/height -> {zoom.first_line()}")
+ scrolled = smoke.call("scroll", {"pid": smoke.pid, "window_id": smoke.window_id,
+ "x": 40, "y": 40, "dy": "3"})
+ if scrolled.is_error and ("unknown argument" in scrolled.text or "dx/dy" in scrolled.text):
+ problems.append(f"scroll dy -> {scrolled.first_line()}")
smoke.record(
check,
"fail" if problems else "pass",
"; ".join(problems)
- or f"since=latest -> {latest.structured.get('since_status')}; click element_token "
+ or f"scroll dy:'3' -> {'ok' if not scrolled.is_error else scrolled.first_line()}; "
+ f"since=latest -> {latest.structured.get('since_status')}; click element_token "
f"{str(increment)!r} + window_id -> Count moved; observe:true -> "
f"{observed.get('since_status')}; zoom x/y/width/height -> "
f"{'ok' if not zoom.is_error else zoom.first_line()}",