Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

feat(cua-driver): experimental run_script tool (JavaScript, opt-in) (#4822)

Francesco Bonacci45469a59c931
diff --git a/libs/cua-driver/rust/Cargo.lock b/libs/cua-driver/rust/Cargo.lock
index 805f0331e4..707ed352b3 100644
--- a/libs/cua-driver/rust/Cargo.lock
+++ b/libs/cua-driver/rust/Cargo.lock
@@ -1293,6 +1293,7 @@ dependencies = [
"libc",
"regex",
"regorus",
+ "rquickjs",
"serde",
"serde_json",
"serde_yaml_ng",
@@ -2124,6 +2125,11 @@ name = "hashbrown"
version = "0.17.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a"
+dependencies = [
+ "allocator-api2",
+ "equivalent",
+ "foldhash 0.2.0",
+]
[[package]]
name = "heapless"
@@ -4014,6 +4020,15 @@ dependencies = [
"tokio",
]
+[[package]]
+name = "relative-path"
+version = "2.0.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f2a073568cd8b9c4825429d3e9fc5f52851b4f97cc1348800689408921e6b8f8"
+dependencies = [
+ "serde",
+]
+
[[package]]
name = "ring"
version = "0.17.14"
@@ -4028,6 +4043,35 @@ dependencies = [
"windows-sys 0.52.0",
]
+[[package]]
+name = "rquickjs"
+version = "0.14.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a0a22bf72f515cc4189dd7c63d016f6de92a11eee98ddf7dd64794198f6eb34e"
+dependencies = [
+ "rquickjs-core",
+]
+
+[[package]]
+name = "rquickjs-core"
+version = "0.14.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6657979a24d543e5fc66f32cef2f47ca82ce8eb55cec5f7d59a12b7b570bbf24"
+dependencies = [
+ "hashbrown 0.17.1",
+ "relative-path",
+ "rquickjs-sys",
+]
+
+[[package]]
+name = "rquickjs-sys"
+version = "0.14.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cee271d0eeba64f0915b846cb7ae02e16faf3dfdffdca91731101d9d30fe3423"
+dependencies = [
+ "cc",
+]
+
[[package]]
name = "rustc-hash"
version = "1.1.0"
@@ -4562,7 +4606,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd"
dependencies = [
"fastrand",
- "getrandom 0.3.4",
+ "getrandom 0.4.2",
"once_cell",
"rustix",
"windows-sys 0.61.2",
diff --git a/libs/cua-driver/rust/THIRD_PARTY_NOTICES.md b/libs/cua-driver/rust/THIRD_PARTY_NOTICES.md
index 297cd37053..729023e17a 100644
--- a/libs/cua-driver/rust/THIRD_PARTY_NOTICES.md
+++ b/libs/cua-driver/rust/THIRD_PARTY_NOTICES.md
@@ -1,7 +1,7 @@
# Third-party notices
-Parts of the Cua Driver Rust workspace are derived from these MIT-licensed
-projects. Each project's copyright notice is below, followed by the MIT
+Parts of the Cua Driver Rust workspace are derived from, or embed, these
+MIT-licensed projects. Each project's copyright notice is below, followed by the MIT
License text they share.
trope-cua's cursor motion is itself a port of Cua Driver's Swift cursor
@@ -11,11 +11,13 @@ In `cursor-overlay`, the motion knobs, their defaults, the arc formula and the
Dubins glide originate in that Swift code; trope-cua is credited below only for
the items listed.
-| Project | Copyright | Used in | What |
-| --------------------------------------------------------------- | ------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| [trope-cua](https://github.com/voctory/trope-cua) | Copyright (c) 2026 Victor Vannara | `crates/platform-windows` (`src/overlay.rs`, `src/input/`, `src/tools/impl_.rs`); `crates/cursor-overlay/src/motion.rs`, `crates/cursor-overlay/src/bezier.rs` (specific items only) | Windows cursor overlay, background input and layered pixel-click dispatch; in `cursor-overlay`, the `press_duration_ms` knob, the 80 ms dwell default, the former 20 s idle default, the `with_overrides` method shape and the cached 32-point arc-length helper |
-| [Interface-Agent](https://github.com/francedot/Interface-Agent) | Copyright (c) 2024 Francesco Bonacci | `crates/platform-windows` (`src/uia/`, `src/win32/`, `src/tools/impl_.rs`) | Windows UI Automation tree walk, app and window enumeration, InvokePattern click and ValuePattern set-value |
-| [yabai](https://github.com/koekeishiya/yabai) | Copyright (c) 2019 Åsmund Vikane | `crates/platform-macos/src/input/skylight.rs` (`activate_without_raise`) | The focus-without-raise SkyLight event-record sequence |
+| Project | Copyright | Used in | What |
+| --------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| [trope-cua](https://github.com/voctory/trope-cua) | Copyright (c) 2026 Victor Vannara | `crates/platform-windows` (`src/overlay.rs`, `src/input/`, `src/tools/impl_.rs`); `crates/cursor-overlay/src/motion.rs`, `crates/cursor-overlay/src/bezier.rs` (specific items only) | Windows cursor overlay, background input and layered pixel-click dispatch; in `cursor-overlay`, the `press_duration_ms` knob, the 80 ms dwell default, the former 20 s idle default, the `with_overrides` method shape and the cached 32-point arc-length helper |
+| [Interface-Agent](https://github.com/francedot/Interface-Agent) | Copyright (c) 2024 Francesco Bonacci | `crates/platform-windows` (`src/uia/`, `src/win32/`, `src/tools/impl_.rs`) | Windows UI Automation tree walk, app and window enumeration, InvokePattern click and ValuePattern set-value |
+| [yabai](https://github.com/koekeishiya/yabai) | Copyright (c) 2019 Åsmund Vikane | `crates/platform-macos/src/input/skylight.rs` (`activate_without_raise`) | The focus-without-raise SkyLight event-record sequence |
+| [QuickJS-ng](https://github.com/quickjs-ng/quickjs) | Copyright (c) 2017-2026 Fabrice Bellard; (c) 2017-2024 Charlie Gordon; (c) 2023-2026 Ben Noordhuis; (c) 2023-2026 Saúl Ibarra Corretgé | `crates/cua-driver-core/src/script_tool/` (compiled in through rquickjs-sys) | The JavaScript engine behind the experimental `run_script` tool |
+| [rquickjs](https://github.com/DelSkayn/rquickjs) | Copyright (c) 2020 Mees Delzenne; (c) 2025 Rquickjs Contributors | `crates/cua-driver-core/src/script_tool/` | Rust bindings to QuickJS-ng |
## MIT License
diff --git a/libs/cua-driver/rust/crates/cua-driver-core/Cargo.toml b/libs/cua-driver/rust/crates/cua-driver-core/Cargo.toml
index db609d624a..55cfbc9b26 100644
--- a/libs/cua-driver/rust/crates/cua-driver-core/Cargo.toml
+++ b/libs/cua-driver/rust/crates/cua-driver-core/Cargo.toml
@@ -6,9 +6,13 @@ license.workspace = true
publish = false
[features]
-default = ["yaml", "rego"]
+default = ["yaml", "rego", "script"]
yaml = ["dep:regex"]
rego = ["dep:regorus"]
+# Experimental `run_script` tool: an embedded QuickJS engine (rquickjs and
+# quickjs-ng, both MIT). Compiled in by default; the tool itself is only
+# registered when the operator opts in at runtime.
+script = ["dep:rquickjs"]
[dependencies]
cua-driver-contract = { path = "../cua-driver-contract" }
@@ -53,6 +57,7 @@ hmac = { workspace = true }
jsonschema = { version = "0.46", default-features = false }
# Private per-request working directory for the optional perception worker.
tempfile = "3"
+rquickjs = { version = "0.14", default-features = false, features = ["std"], optional = true }
[target.'cfg(unix)'.dependencies]
libc = "0.2"
diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/authorization.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/authorization.rs
index 6007d49ae9..3ed93fa70f 100644
--- a/libs/cua-driver/rust/crates/cua-driver-core/src/authorization.rs
+++ b/libs/cua-driver/rust/crates/cua-driver-core/src/authorization.rs
@@ -893,6 +893,7 @@ pub fn advertised_risk_for(tool: &str) -> RiskAssessment {
| "hotkey"
| "set_value"
| "run_actions"
+ | "run_script"
| "invoke_menu"
| "launch_app"
| "bring_to_front"
diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/batch_tools.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/batch_tools.rs
index dc86c03745..32ad113414 100644
--- a/libs/cua-driver/rust/crates/cua-driver-core/src/batch_tools.rs
+++ b/libs/cua-driver/rust/crates/cua-driver-core/src/batch_tools.rs
@@ -20,7 +20,7 @@ use crate::{
tool::{Tool, ToolDef, ToolRegistry},
};
-pub(crate) mod locate;
+pub mod locate;
use locate::{Check, ElementSpec, Locator, Window, WindowSpec};
diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/batch_tools/locate.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/batch_tools/locate.rs
index 1c2287fbb6..bcd2e2ede2 100644
--- a/libs/cua-driver/rust/crates/cua-driver-core/src/batch_tools/locate.rs
+++ b/libs/cua-driver/rust/crates/cua-driver-core/src/batch_tools/locate.rs
@@ -1163,6 +1163,57 @@ fn selected_state(element: &Value) -> Option<bool> {
element.get("selected").and_then(Value::as_bool).or(checked)
}
+/// Display-only rows (static text and other rows without an element index)
+/// that match `spec`, parsed into {role, label, value, text}. Scripts read
+/// values such as "Ticket: 4F2K" from these; they cannot be acted on.
+pub(crate) fn display_rows(read: &Read, spec: &ElementSpec) -> Vec<Value> {
+ if spec.role.is_none() && spec.name.is_none() && spec.text.is_none() {
+ return Vec::new();
+ }
+ read.markdown
+ .lines()
+ .map(str::trim)
+ .filter_map(|line| line.strip_prefix("- "))
+ .filter(|rest| !rest.starts_with('['))
+ .filter_map(|rest| {
+ let role = rest.split_whitespace().next()?.to_owned();
+ let after = rest[role.len()..].trim_start();
+ let quoted = |text: &str| -> Option<String> {
+ let inner = text.strip_prefix('"')?;
+ inner.find('"').map(|end| inner[..end].to_owned())
+ };
+ let label = quoted(after).or_else(|| {
+ let open = after.rfind('(')?;
+ after[open + 1..].strip_suffix(')').map(str::to_owned)
+ });
+ let value = after.find("= ").and_then(|at| quoted(&after[at + 2..]));
+ let folded = fold(rest);
+ if let Some(role_wanted) = &spec.role {
+ if !role_matches(role_wanted, &role) {
+ return None;
+ }
+ }
+ if let Some(name) = &spec.name {
+ if !has_word_start(&folded, &fold(name)) {
+ return None;
+ }
+ }
+ if let Some(text) = &spec.text {
+ if !folded.contains(&fold(text)) {
+ return None;
+ }
+ }
+ Some(json!({
+ "role": role,
+ "label": label,
+ "value": value,
+ "text": rest,
+ "display_only": true,
+ }))
+ })
+ .collect()
+}
+
fn row_matches(row: &str, spec: &ElementSpec) -> bool {
if !row.starts_with("- ") {
return false;
diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/lib.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/lib.rs
index eabf86392b..5fd18d2196 100644
--- a/libs/cua-driver/rust/crates/cua-driver-core/src/lib.rs
+++ b/libs/cua-driver/rust/crates/cua-driver-core/src/lib.rs
@@ -94,6 +94,8 @@ pub mod recording_loader;
pub mod recording_render;
pub mod recording_tools;
pub mod recording_zoom;
+#[cfg(feature = "script")]
+pub mod script_tool;
pub mod server;
pub mod session;
pub mod session_authorization;
diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/script_tool.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/script_tool.rs
new file mode 100644
index 0000000000..a1ed8b1c36
--- /dev/null
+++ b/libs/cua-driver/rust/crates/cua-driver-core/src/script_tool.rs
@@ -0,0 +1,852 @@
+//! `run_script` (experimental): run a model-written JavaScript program that
+//! drives the computer through Cua Driver's own actions and reads.
+//!
+//! Off by default. It is registered only when the person running the driver
+//! opts in, with `CUA_DRIVER_EXPERIMENTAL_SCRIPT=1` or
+//! `"experimental_script": true` in `~/.cua-driver/config.json`.
+//! `set_config` cannot write that key, so a model cannot turn it on.
+//!
+//! The script runs in an embedded QuickJS engine ([`engine`]) on its own
+//! thread, with no filesystem, network, process or environment access. Its
+//! only reach outside the engine is the `cua` API, and every call that API
+//! makes is dispatched here, inside this tool call's task, through the same
+//! [`ToolRegistry`] entry point a direct call uses. Session, policy,
+//! capability manifest and approvals therefore apply per call, and a script
+//! can do nothing the same sequence of direct calls could not.
+//!
+//! Limits: wall time, number of driver calls, heap size and stack size, and
+//! the script stops when the request is cancelled (the tool future is
+//! dropped).
+
+use std::sync::{
+ atomic::{AtomicBool, Ordering},
+ Arc, OnceLock,
+};
+use std::time::{Duration, Instant};
+
+use async_trait::async_trait;
+use serde_json::{json, Map, Value};
+
+use crate::{
+ batch_tools::{
+ locate::{self, Check, ElementSpec, Locator, WindowSpec},
+ BATCHABLE_TOOLS,
+ },
+ protocol::{Content, ToolResult},
+ recording_tools::ReplayRegistrySlot,
+ tool::{Tool, ToolDef, ToolRegistry},
+};
+
+mod engine;
+
+use engine::{FailureKind, HostCall, HostError, Limits};
+
+pub const RUN_SCRIPT_TOOL: &str = "run_script";
+/// Environment switch that registers the tool.
+pub const ENABLE_ENV: &str = "CUA_DRIVER_EXPERIMENTAL_SCRIPT";
+/// `~/.cua-driver/config.json` key that registers the tool.
+pub const ENABLE_CONFIG_KEY: &str = "experimental_script";
+
+pub const DEFAULT_TIMEOUT_MS: u64 = 30_000;
+pub const MAX_TIMEOUT_MS: u64 = 120_000;
+pub const DEFAULT_MAX_CALLS: u32 = 100;
+pub const MAX_MAX_CALLS: u32 = 500;
+pub const MAX_SCRIPT_BYTES: usize = 64 * 1024;
+const MEMORY_BYTES: usize = 32 * 1024 * 1024;
+const STACK_BYTES: usize = 1024 * 1024;
+const THREAD_STACK_BYTES: usize = 16 * 1024 * 1024;
+const RETURN_TEXT_LIMIT: usize = 8_000;
+const CALL_TEXT_LIMIT: usize = 12_000;
+const IMAGES_KEPT: usize = 2;
+/// Extra time the engine thread gets to notice a deadline or cancel.
+const SHUTDOWN_GRACE: Duration = Duration::from_millis(1_500);
+
+/// Driver tools a script may call, besides the batchable actions.
+const READ_TOOLS: &[&str] = &[
+ "launch_app",
+ "list_apps",
+ "list_windows",
+ "get_window_state",
+ "verify_state",
+ "zoom",
+];
+
+/// Whether the person running the driver opted in.
+pub fn enabled() -> bool {
+ let config = std::env::var_os("HOME")
+ .or_else(|| std::env::var_os("USERPROFILE"))
+ .map(|home| {
+ std::path::PathBuf::from(home)
+ .join(".cua-driver")
+ .join("config.json")
+ });
+ enabled_from(std::env::var(ENABLE_ENV).ok().as_deref(), config.as_deref())
+}
+
+/// The environment variable wins; otherwise the config file decides.
+fn enabled_from(env: Option<&str>, config: Option<&std::path::Path>) -> bool {
+ if let Some(value) = env {
+ return matches!(
+ value.trim().to_ascii_lowercase().as_str(),
+ "1" | "true" | "yes" | "on"
+ );
+ }
+ config
+ .and_then(|path| std::fs::read_to_string(path).ok())
+ .and_then(|text| serde_json::from_str::<Value>(&text).ok())
+ .and_then(|config| config.get(ENABLE_CONFIG_KEY).and_then(Value::as_bool))
+ .unwrap_or(false)
+}
+
+pub struct RunScriptTool {
+ registry: ReplayRegistrySlot,
+}
+
+impl RunScriptTool {
+ pub fn new(registry: ReplayRegistrySlot) -> Self {
+ Self { registry }
+ }
+}
+
+static DEF: OnceLock<ToolDef> = OnceLock::new();
+
+fn platform() -> &'static str {
+ if cfg!(target_os = "macos") {
+ "mac"
+ } else if cfg!(target_os = "windows") {
+ "windows"
+ } else {
+ "linux"
+ }
+}
+
+#[async_trait]
+impl Tool for RunScriptTool {
+ fn def(&self) -> &ToolDef {
+ DEF.get_or_init(|| ToolDef {
+ name: RUN_SCRIPT_TOOL.into(),
+ description: "EXPERIMENTAL. Run a JavaScript program that drives apps through Cua \
+ Driver, in ONE call: loops, conditions, reads and checks between actions, and \
+ only the result comes back. Use it for a multi-step flow you can write down; \
+ use run_actions for a fixed list of steps.\n\n\
+ The script is the body of an async function: use `await`, and `return` a JSON \
+ value. It runs in a sandbox with no filesystem, network, process, timers or \
+ environment; the only API is `cua`:\n\
+ - `await cua.getApp(\"TextEdit\" | \"com.apple.TextEdit\" | {pid, windowId})` → \
+ app (follows the app's frontmost window unless a windowId is given)\n\
+ - `await cua.launch(nameOrBundleId)` → app; `cua.listApps()`, \
+ `cua.listWindows({pid?})`, `cua.sleep(ms)`, `cua.computer.target`\n\
+ - app.getState(opts?) → get_window_state result (`tree_markdown` or \
+ `tree_diff`, `snapshot_id`; pass get_window_state options such as \
+ {since:\"latest\"})\n\
+ - app.query({role?, name?, text?}) → matching elements across the app's \
+ windows {element_token, role, label, value, enabled, selected, frame}, then \
+ matching static-text rows {role, label, value, text, display_only} (read text \
+ such as a status line from their `value`)\n\
+ - app.waitFor({role?, name?, text?, gone?, value?, timeout_ms?}) and \
+ app.verify(predicate | predicates) (verify_state)\n\
+ - app.click(t), doubleClick(t), rightClick(t), typeText(text, t?), \
+ pressKey(\"return\" | \"cmd+s\", t?), hotkey([\"cmd\",\"s\"]), scroll(t, \
+ direction, amount?), setValue(t, value), drag([x,y], [x,y]), zoom([x1,y1,x2,y2])\n\
+ where a target `t` is {role, name, nth?} (found fresh, waits up to 3 s), an \
+ element_token string, a row number from this app's last getState(), or [x, y] \
+ window pixels.\n\
+ - `cua.call(tool, args)` for any other allowed tool: click, double_click, \
+ right_click, set_value, type_text, press_key, hotkey, scroll, drag, \
+ launch_app, list_apps, list_windows, get_window_state, verify_state, zoom.\n\
+ A failed driver call throws a CuaError (catch it to recover). Every call goes \
+ through the same session, permission and approval checks as a direct call.\n\n\
+ Limits: `timeout_ms` wall time (default 30000, max 120000), `max_calls` driver \
+ calls (default 100, max 500), 32 MiB heap. Returns the return value, console \
+ output, and a per-call log; an error names the script line, the call and the \
+ reason.\n\n\
+ Example:\n\
+ const app = await cua.getApp(\"Calculator\");\n\
+ for (const key of [\"7\", \"multiply\", \"6\", \"equals\"]) await app.click({role: \"button\", name: key});\n\
+ const [display] = await app.query({role: \"statictext\"});\n\
+ return display.value;"
+ .into(),
+ input_schema: json!({
+ "type": "object",
+ "required": ["script"],
+ "properties": {
+ "script": {
+ "type": "string",
+ "minLength": 1,
+ "maxLength": MAX_SCRIPT_BYTES,
+ "description": "JavaScript: the body of an async function. `return` the result."
+ },
+ "timeout_ms": {
+ "type": "integer",
+ "minimum": 100,
+ "maximum": MAX_TIMEOUT_MS,
+ "description": "Wall-time limit. Default 30000."
+ },
+ "max_calls": {
+ "type": "integer",
+ "minimum": 1,
+ "maximum": MAX_MAX_CALLS,
+ "description": "Most driver calls the script may make. Default 100."
+ }
+ },
+ "additionalProperties": false
+ }),
+ read_only: false,
+ destructive: true,
+ idempotent: false,
+ open_world: true,
+ })
+ }
+
+ async fn invoke(&self, args: Value) -> ToolResult {
+ let registry = match self.registry.lock().unwrap().upgrade() {
+ Some(registry) => registry,
+ None => {
+ return ToolResult::error(
+ "run_script is not available: registry not initialised yet.",
+ )
+ }
+ };
+ let request = match Request::parse(&args) {
+ Ok(request) => request,
+ Err(message) => {
+ return ToolResult::error(format!("run_script rejected: {message}"))
+ .with_structured(
+ json!({"ok": false, "code": "invalid_script", "detail": message}),
+ )
+ }
+ };
+ run(&registry, request).await
+ }
+}
+
+struct Request {
+ script: String,
+ timeout: Duration,
+ max_calls: u32,
+ session: Option<String>,
+}
+
+impl Request {
+ fn parse(args: &Value) -> Result<Self, String> {
+ let object = args.as_object().ok_or("arguments must be a JSON object")?;
+ if let Some(key) = object.keys().find(|key| {
+ !key.starts_with('_')
+ && !matches!(
+ key.as_str(),
+ "script" | "timeout_ms" | "max_calls" | "session"
+ )
+ }) {
+ return Err(format!("unknown field `{key}`"));
+ }
+ let script = object
+ .get("script")
+ .and_then(Value::as_str)
+ .filter(|script| !script.trim().is_empty())
+ .ok_or("`script` must be a non-empty string")?;
+ if script.len() > MAX_SCRIPT_BYTES {
+ return Err(format!("`script` is longer than {MAX_SCRIPT_BYTES} bytes"));
+ }
+ let timeout_ms = match object.get("timeout_ms") {
+ None | Some(Value::Null) => DEFAULT_TIMEOUT_MS,
+ Some(value) => value
+ .as_u64()
+ .filter(|ms| (100..=MAX_TIMEOUT_MS).contains(ms))
+ .ok_or_else(|| {
+ format!("`timeout_ms` must be an integer 100 to {MAX_TIMEOUT_MS}")
+ })?,
+ };
+ let max_calls = match object.get("max_calls") {
+ None | Some(Value::Null) => DEFAULT_MAX_CALLS,
+ Some(value) => value
+ .as_u64()
+ .filter(|calls| (1..=u64::from(MAX_MAX_CALLS)).contains(calls))
+ .ok_or_else(|| format!("`max_calls` must be an integer 1 to {MAX_MAX_CALLS}"))?
+ as u32,
+ };
+ Ok(Self {
+ script: script.to_owned(),
+ timeout: Duration::from_millis(timeout_ms),
+ max_calls,
+ session: object
+ .get("_public_session_label")
+ .and_then(Value::as_str)
+ .map(str::to_owned),
+ })
+ }
+}
+
+/// Sets the cancel flag when the tool call ends for any reason, including
+/// the request being dropped.
+struct CancelOnDrop(Arc<AtomicBool>);
+
+impl Drop for CancelOnDrop {
+ fn drop(&mut self) {
+ self.0.store(true, Ordering::SeqCst);
+ }
+}
+
+/// One entry of the per-call log.
+struct CallRecord {
+ number: u32,
+ op: String,
+ line: u32,
+ ok: bool,
+ elapsed: Duration,
+ detail: String,
+}
+
+async fn run(registry: &ToolRegistry, request: Request) -> ToolResult {
+ let started = Instant::now();
+ let deadline = started + request.timeout;
+ let cancel = Arc::new(AtomicBool::new(false));
+ let _cancel_guard = CancelOnDrop(cancel.clone());
+ let (call_tx, mut call_rx) = tokio::sync::mpsc::channel::<HostCall>(1);
+ let (done_tx, mut done_rx) = tokio::sync::oneshot::channel();
+
+ let limits = Limits {
+ deadline,
+ max_calls: request.max_calls,
+ memory_bytes: MEMORY_BYTES,
+ stack_bytes: STACK_BYTES,
+ };
+ let script = request.script.clone();
+ let engine_cancel = cancel.clone();
+ let spawned = std::thread::Builder::new()
+ .name("cua-run-script".into())
+ .stack_size(THREAD_STACK_BYTES)
+ .spawn(move || {
+ let outcome = engine::run(&script, platform(), limits, engine_cancel, call_tx);
+ let _ = done_tx.send(outcome);
+ });
+ if let Err(error) = spawned {
+ return ToolResult::error(format!("run_script could not start its engine: {error}"));
+ }
+
+ let mut api = Api::new(registry, request.session.clone(), deadline);
+ let mut log: Vec<CallRecord> = Vec::new();
+ let hard_stop = deadline + SHUTDOWN_GRACE;
+ let outcome = loop {
+ tokio::select! {
+ Some(call) = call_rx.recv() => {
+ let call_started = Instant::now();
+ let result = match tokio::time::timeout_at(deadline.into(), api.dispatch(&call.op, call.args)).await {
+ Ok(result) => result,
+ Err(_) => Err(HostError::new("the script's wall-time limit was reached", "timeout")),
+ };
+ if call.number > 0 {
+ log.push(CallRecord {
+ number: call.number,
+ op: call.op.clone(),
+ line: call.line,
+ ok: result.is_ok(),
+ elapsed: call_started.elapsed(),
+ detail: match &result {
+ Ok(value) => summarize(value),
+ Err(error) => error.message.clone(),
+ },
+ });
+ }
+ let _ = call.reply.send(result);
+ }
+ outcome = &mut done_rx => break outcome.ok(),
+ _ = tokio::time::sleep_until(hard_stop.into()) => {
+ cancel.store(true, Ordering::SeqCst);
+ break tokio::time::timeout(SHUTDOWN_GRACE, &mut done_rx).await.ok().and_then(Result::ok);
+ }
+ }
+ };
+ render(
+ outcome,
+ log,
+ api.images,
+ started.elapsed(),
+ request.max_calls,
+ )
+}
+
+/// Executes the script's operations through the registry.
+struct Api<'a> {
+ registry: &'a ToolRegistry,
+ locator: Locator<'a>,
+ session: Option<String>,
+ deadline: Instant,
+ images: Vec<Content>,
+}
+
+impl<'a> Api<'a> {
+ fn new(registry: &'a ToolRegistry, session: Option<String>, deadline: Instant) -> Self {
+ Self {
+ registry,
+ locator: Locator::new(registry, session.clone(), deadline),
+ session,
+ deadline,
+ images: Vec::new(),
+ }
+ }
+
+ async fn dispatch(&mut self, op: &str, args: Value) -> Result<Value, HostError> {
+ let Value::Object(mut args) = args else {
+ return Err(HostError::new(
+ "arguments must be an object",
+ "invalid_args",
+ ));
+ };
+ // Runtime-private fields are never reachable from a script.
+ if let Some(key) = args.keys().find(|key| key.starts_with('_')) {
+ return Err(HostError::new(
+ format!("`{key}` is reserved and cannot be set"),
+ "invalid_args",
+ ));
+ }
+ match args.get("session").and_then(Value::as_str) {
+ Some(own)
+ if Some(own) != self.session.as_deref()
+ && !(self.session.is_none() && own == "default") =>
+ {
+ return Err(HostError::new(
+ "a script runs in its run_script call's session; drop `session`",
+ "invalid_args",
+ ))
+ }
+ _ => {}
+ }
+ args.remove("session");
+ match op {
+ "sleep" => {
+ let ms = args.get("ms").and_then(Value::as_u64).unwrap_or(0);
+ let until = Instant::now() + Duration::from_millis(ms);
+ if until > self.deadline {
+ tokio::time::sleep_until(self.deadline.into()).await;
+ return Err(HostError::new(
+ "the script's wall-time limit was reached during sleep",
+ "timeout",
+ ));
+ }
+ tokio::time::sleep_until(until.into()).await;
+ Ok(Value::Null)
+ }
+ "get_app" => self.get_app(args).await,
+ "find" => self.find(args).await,
+ "wait_for" => self.wait_for(args).await,
+ tool if BATCHABLE_TOOLS.contains(&tool) => self.act(tool, args).await,
+ tool if READ_TOOLS.contains(&tool) => self.invoke(tool, args).await,
+ other => Err(HostError::new(
+ format!(
+ "`{other}` is not available to scripts; allowed: {}, {}",
+ BATCHABLE_TOOLS.join(", "),
+ READ_TOOLS.join(", ")
+ ),
+ "not_allowed",
+ )),
+ }
+ }
+
+ async fn invoke(
+ &mut self,
+ tool: &str,
+ mut args: Map<String, Value>,
+ ) -> Result<Value, HostError> {
+ if let Some(session) = &self.session {
+ args.insert("session".into(), json!(session));
+ }
+ let result = self.registry.invoke(tool, Value::Object(args)).await;
+ let mut text = String::new();
+ for content in &result.content {
+ match content {
+ Content::Text { text: part, .. } => {
+ if !text.is_empty() {
+ text.push('\n');
+ }
+ text.push_str(part);
+ }
+ Content::Image { .. } => {
+ self.images.push(content.clone());
+ if self.images.len() > IMAGES_KEPT {
+ self.images.remove(0);
+ }
+ }
+ }
+ }
+ if result.is_error == Some(true) {
+ let code = result
+ .structured_content
+ .as_ref()
+ .and_then(|value| value.get("code"))
+ .and_then(Value::as_str)
+ .unwrap_or("tool_error");
+ return Err(HostError::new(
+ clip(text.lines().next().unwrap_or("failed"), 500),
+ code,
+ ));
+ }
+ let mut value = match result.structured_content {
+ Some(Value::Object(object)) => Value::Object(object),
+ Some(other) => json!({ "result": other }),
+ None => json!({}),
+ };
+ if !text.is_empty() && value.get("text").is_none() {
+ value["text"] = json!(clip(&text, CALL_TEXT_LIMIT));
+ }
+ Ok(value)
+ }
+
+ async fn get_app(&mut self, args: Map<String, Value>) -> Result<Value, HostError> {
+ let spec = WindowSpec {
+ pid: args.get("pid").and_then(Value::as_i64),
+ window_id: args.get("window_id").and_then(Value::as_u64),
+ app: args.get("app").and_then(Value::as_str).map(str::to_owned),
+ title: args
+ .get("window")
+ .and_then(Value::as_str)
+ .map(str::to_owned),
+ };
+ if spec.is_empty() {
+ return Err(HostError::new(
+ "getApp needs an app name, bundle id, pid or windowId",
+ "invalid_args",
+ ));
+ }
+ // Always listed, so an exact pid/window_id is confirmed and named.
+ // A launch may need a moment before its window shows.
+ let wait = Duration::from_millis(
+ args.get("timeout_ms")
+ .and_then(Value::as_u64)
+ .unwrap_or(0)
+ .min(locate::MAX_CHECK_TIMEOUT_MS),
+ );
+ let until = Instant::now() + wait;
+ loop {
+ match self.locator.listed_window(&spec).await {
+ Ok(window) => return Ok(window.to_json()),
+ Err(miss) if Instant::now() < until && miss.code != "lookup_failed" => {
+ tokio::time::sleep(Duration::from_millis(250)).await;
+ }
+ Err(miss) => {
+ let hint = if miss.code == "app_not_found" {
+ "; launch it with cua.launch()"
+ } else {
+ ""
+ };
+ return Err(HostError::new(format!("{}{hint}", miss.message), miss.code));
+ }
+ }
+ }
+ }
+
+ /// Elements matching {role, name, text} across the app's windows
+ /// (topmost first): addressable ones with their element_token, then
+ /// matching display-only text rows, whose `value` holds the text.
+ async fn find(&mut self, mut args: Map<String, Value>) -> Result<Value, HostError> {
+ let text = args
+ .remove("text")
+ .and_then(|value| value.as_str().map(str::to_owned));
+ let (window, element) = locate::take_selector(&mut args)
+ .map_err(|message| HostError::new(message, "invalid_args"))?;
+ let spec = ElementSpec {
+ nth: None,
+ text,
+ ..element
+ };
+ let windows = self
+ .locator
+ .search_windows(&window)
+ .await
+ .map_err(|miss| HostError::new(miss.message, miss.code))?;
+ let mut found: Vec<Value> = Vec::new();
+ for resolved in &windows {
+ let read = self
+ .locator
+ .read(resolved)
+ .await
+ .map_err(|miss| HostError::new(miss.message, miss.code))?;
+ let window_json = json!({"window_id": resolved.window_id, "title": resolved.title});
+ if spec.text.is_none() {
+ for element in locate::matches(&read, &spec) {
+ let mut out = Map::new();
+ for key in [
+ "element_token",
+ "role",
+ "label",
+ "value",
+ "enabled",
+ "selected",
+ "frame",
+ ] {
+ if let Some(value) = element.get(key) {
+ out.insert(key.to_owned(), value.clone());
+ }
+ }
+ out.insert("window".into(), window_json.clone());
+ found.push(Value::Object(out));
+ }
+ }
+ for mut row in locate::display_rows(&read, &spec) {
+ row["window"] = window_json.clone();
+ found.push(row);
+ }
+ if found.len() >= 50 {
+ break;
+ }
+ }
+ found.truncate(50);
+ Ok(Value::Array(found))
+ }
+
+ async fn wait_for(&mut self, args: Map<String, Value>) -> Result<Value, HostError> {
+ let check = Check::parse(&Value::Object(args), locate::DEFAULT_WAIT_TIMEOUT_MS)
+ .map_err(|message| HostError::new(message, "invalid_args"))?;
+ match self.locator.check(&check, &WindowSpec::default()).await {
+ Ok(checked) => Ok(json!({
+ "ok": true,
+ "observed": checked.observed,
+ "waited_ms": checked.waited.as_millis() as u64,
+ })),
+ Err(miss) => Err(HostError::new(
+ format!("{} did not hold: {}", check.describe(), miss.message),
+ miss.code,
+ )),
+ }
+ }
+
+ async fn act(&mut self, tool: &str, mut args: Map<String, Value>) -> Result<Value, HostError> {
+ let find_timeout = match args.remove("timeout_ms") {
+ None | Some(Value::Null) => locate::DEFAULT_FIND_TIMEOUT_MS,
+ Some(value) => value
+ .as_u64()
+ .filter(|ms| *ms <= locate::MAX_CHECK_TIMEOUT_MS)
+ .ok_or_else(|| {
+ HostError::new("`timeout_ms` must be an integer 0 to 10000", "invalid_args")
+ })?,
+ };
+ let (window, element) = locate::take_selector(&mut args)
+ .map_err(|message| HostError::new(message, "invalid_args"))?;
+ let mut target = None;
+ if !element.is_empty() {
+ if tool == "drag" {
+ return Err(HostError::new(
+ "drag takes coordinates, not a role/name target",
+ "invalid_args",
+ ));
+ }
+ if let Some(conflict) = ["element_token", "x", "y"]
+ .iter()
+ .find(|key| args.contains_key(**key))
+ {
+ return Err(HostError::new(
+ format!("target by role/name OR by `{conflict}`, not both"),
+ "invalid_args",
+ ));
+ }
+ let found = self
+ .locator
+ .find(&window, &element, Duration::from_millis(find_timeout))
+ .await
+ .map_err(|miss| HostError::new(miss.message, miss.code))?;
+ args.insert("pid".into(), json!(found.window.pid));
+ args.insert("window_id".into(), json!(found.window.window_id));
+ args.insert(
+ "element_token".into(),
+ json!(found.token().unwrap_or_default()),
+ );
+ target = Some(json!({"window": found.window.to_json(), "element": found.describe()}));
+ } else if window.window_id.is_none()
+ && (window.pid.is_some() || window.app.is_some() || window.title.is_some())
+ && !args.contains_key("element_token")
+ {
+ // Pixel and focus targets act on the app's frontmost window.
+ let resolved = self
+ .locator
+ .window(&window)
+ .await
+ .map_err(|miss| HostError::new(miss.message, miss.code))?;
+ args.insert("pid".into(), json!(resolved.pid));
+ args.insert("window_id".into(), json!(resolved.window_id));
+ } else if let Some(pid) = window.pid {
+ args.insert("pid".into(), json!(pid));
+ }
+ let mut value = self.invoke(tool, args).await?;
+ if let (Some(target), Value::Object(object)) = (target, &mut value) {
+ object.insert("target".into(), target);
+ }
+ Ok(value)
+ }
+}
+
+/// Short, model-facing summary of a successful call.
+fn summarize(value: &Value) -> String {
+ if let Some(target) = value
+ .get("target")
+ .and_then(|target| target.get("element"))
+ .and_then(Value::as_str)
+ {
+ return format!("on {target}");
+ }
+ if let Some(text) = value.get("text").and_then(Value::as_str) {
+ return clip(text.lines().next().unwrap_or(""), 120);
+ }
+ match value {
+ Value::Array(items) => format!("{} item(s)", items.len()),
+ Value::Object(object) if object.contains_key("window_id") && object.contains_key("pid") => {
+ format!("window {} of pid {}", object["window_id"], object["pid"])
+ }
+ Value::Null => String::new(),
+ other => clip(&other.to_string(), 120),
+ }
+}
+
+fn render(
+ outcome: Option<engine::Outcome>,
+ log: Vec<CallRecord>,
+ images: Vec<Content>,
+ elapsed: Duration,
+ max_calls: u32,
+) -> ToolResult {
+ let elapsed_ms = elapsed.as_millis() as u64;
+ let outcome = outcome.unwrap_or_else(|| engine::Outcome {
+ result: Err(engine::Failure {
+ kind: FailureKind::Timeout,
+ message: "the script did not stop after its wall-time limit; its engine was abandoned"
+ .into(),
+ line: None,
+ op: None,
+ code: None,
+ call: None,
+ }),
+ console: Vec::new(),
+ console_truncated: false,
+ calls: log.len() as u32,
+ });
+ let calls_json: Vec<Value> = log
+ .iter()
+ .map(|record| {
+ json!({
+ "call": record.number,
+ "op": record.op,
+ "line": (record.line > 0).then_some(record.line),
+ "ok": record.ok,
+ "ms": record.elapsed.as_millis() as u64,
+ "detail": record.detail,
+ })
+ })
+ .collect();
+ let mut lines = Vec::new();
+ let mut structured = json!({
+ "elapsed_ms": elapsed_ms,
+ "driver_calls": outcome.calls,
+ "max_calls": max_calls,
+ "console": outcome.console,
+ "console_truncated": outcome.console_truncated,
+ "calls": calls_json,
+ });
+ let is_error = match &outcome.result {
+ Ok(value) => {
+ structured["ok"] = json!(true);
+ structured["value"] = value.clone();
+ lines.push(format!(
+ "run_script: ok in {elapsed_ms} ms, {} driver call(s)",
+ outcome.calls
+ ));
+ lines.push(format!(
+ "return value: {}",
+ clip(&value.to_string(), RETURN_TEXT_LIMIT)
+ ));
+ false
+ }
+ Err(failure) => {
+ structured["ok"] = json!(false);
+ // A failed driver call the script did not catch: say which call.
+ let failed_call = failure
+ .call
+ .and_then(|number| log.iter().find(|record| record.number == number));
+ let line = failure.line.or(failed_call
+ .map(|record| record.line)
+ .filter(|line| *line > 0));
+ let mut location = Vec::new();
+ if let Some(line) = line {
+ location.push(format!("line {line}"));
+ }
+ if let Some(record) = failed_call {
+ location.push(format!("call {} ({})", record.number, record.op));
+ } else if let Some(op) = &failure.op {
+ location.push(format!("call to {op}"));
+ }
+ structured["error"] = json!({
+ "kind": failure.kind.as_str(),
+ "message": failure.message,
+ "line": line,
+ "call": failure.call,
+ "op": failure.op,
+ "code": failure.code,
+ });
+ lines.push(format!(
+ "run_script: ERROR ({}){}: {}",
+ failure.kind.as_str(),
+ if location.is_empty() {
+ String::new()
+ } else {
+ format!(" at {}", location.join(", "))
+ },
+ failure.message
+ ));
+ lines.push(format!(
+ "after {elapsed_ms} ms and {} driver call(s); calls before the error took effect",
+ outcome.calls
+ ));
+ true
+ }
+ };
+ if !outcome.console.is_empty() {
+ lines.push(format!(
+ "console ({} line(s){}):",
+ outcome.console.len(),
+ if outcome.console_truncated {
+ ", truncated"
+ } else {
+ ""
+ }
+ ));
+ lines.extend(outcome.console.iter().map(|line| format!(" {line}")));
+ }
+ if !log.is_empty() {
+ lines.push("calls:".to_owned());
+ for record in &log {
+ lines.push(format!(
+ " {}. {}{} {} ({} ms){}",
+ record.number,
+ if record.line > 0 {
+ format!("line {} ", record.line)
+ } else {
+ String::new()
+ },
+ record.op,
+ if record.ok { "ok" } else { "ERROR" },
+ record.elapsed.as_millis(),
+ if record.detail.is_empty() {
+ String::new()
+ } else {
+ format!(": {}", clip(&record.detail, 160))
+ }
+ ));
+ }
+ }
+ let mut content = vec![Content::text(lines.join("\n"))];
+ content.extend(images);
+ ToolResult {
+ content,
+ is_error: is_error.then_some(true),
+ structured_content: Some(structured),
+ ..Default::default()
+ }
+}
+
+fn clip(text: &str, max: usize) -> String {
+ if text.chars().count() <= max {
+ text.to_owned()
+ } else {
+ text.chars().take(max).collect::<String>() + "…"
+ }
+}
+
+#[cfg(test)]
+mod tests;
diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/script_tool/engine.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/script_tool/engine.rs
new file mode 100644
index 0000000000..35fcd807f7
--- /dev/null
+++ b/libs/cua-driver/rust/crates/cua-driver-core/src/script_tool/engine.rs
@@ -0,0 +1,504 @@
+//! The QuickJS side of `run_script`. Runs on its own OS thread.
+//!
+//! The context gets the ECMAScript built-ins and nothing else: QuickJS has no
+//! filesystem, network, process or environment API unless its `std`/`os`
+//! modules are added, and no module loader is installed, so `import()` fails.
+//! The one way out is `__host`, which the prelude captures and hides, and
+//! which only forwards an operation name and JSON arguments to the async
+//! side; that side decides what the operation may do.
+
+use std::sync::{
+ atomic::{AtomicBool, AtomicU32, Ordering},
+ Arc, Mutex,
+};
+use std::time::{Duration, Instant};
+
+use rquickjs::{context::EvalOptions, Context, Function, Runtime, Value as JsValue};
+use serde_json::{json, Value};
+
+/// Ops that do not count as driver calls.
+const FREE_OPS: &[&str] = &["platform", "sleep"];
+/// Longest console output kept.
+const CONSOLE_MAX_LINES: usize = 200;
+const CONSOLE_MAX_BYTES: usize = 32 * 1024;
+const CONSOLE_LINE_MAX: usize = 2_000;
+
+pub(crate) struct Limits {
+ pub deadline: Instant,
+ pub max_calls: u32,
+ pub memory_bytes: usize,
+ pub stack_bytes: usize,
+}
+
+/// One driver operation the script asked for.
+pub(crate) struct HostCall {
+ pub op: String,
+ pub args: Value,
+ pub line: u32,
+ pub number: u32,
+ pub reply: std::sync::mpsc::SyncSender<Result<Value, HostError>>,
+}
+
+#[derive(Debug, Clone)]
+pub(crate) struct HostError {
+ pub message: String,
+ pub code: Option<String>,
+}
+
+impl HostError {
+ pub fn new(message: impl Into<String>, code: &str) -> Self {
+ Self {
+ message: message.into(),
+ code: Some(code.to_owned()),
+ }
+ }
+}
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub(crate) enum FailureKind {
+ /// The script threw (or a driver call it made failed and it did not
+ /// catch that).
+ Exception,
+ Syntax,
+ Timeout,
+ Cancelled,
+ Memory,
+ /// The script's promise never settled.
+ Stalled,
+ Internal,
+}
+
+impl FailureKind {
+ pub fn as_str(&self) -> &'static str {
+ match self {
+ FailureKind::Exception => "exception",
+ FailureKind::Syntax => "syntax",
+ FailureKind::Timeout => "timeout",
+ FailureKind::Cancelled => "cancelled",
+ FailureKind::Memory => "memory",
+ FailureKind::Stalled => "stalled",
+ FailureKind::Internal => "internal",
+ }
+ }
+}
+
+#[derive(Debug, Clone)]
+pub(crate) struct Failure {
+ pub kind: FailureKind,
+ pub message: String,
+ pub line: Option<u32>,
+ pub op: Option<String>,
+ pub code: Option<String>,
+ pub call: Option<u32>,
+}
+
+impl Failure {
+ fn new(kind: FailureKind, message: impl Into<String>) -> Self {
+ Self {
+ kind,
+ message: message.into(),
+ line: None,
+ op: None,
+ code: None,
+ call: None,
+ }
+ }
+}
+
+pub(crate) struct Outcome {
+ pub result: Result<Value, Failure>,
+ pub console: Vec<String>,
+ pub console_truncated: bool,
+ pub calls: u32,
+}
+
+#[derive(Default)]
+struct Console {
+ lines: Vec<String>,
+ bytes: usize,
+ truncated: bool,
+}
+
+impl Console {
+ fn push(&mut self, level: &str, text: &str) {
+ if self.lines.len() >= CONSOLE_MAX_LINES || self.bytes >= CONSOLE_MAX_BYTES {
+ self.truncated = true;
+ return;
+ }
+ let mut line = if level == "log" || level == "info" {
+ text.to_owned()
+ } else {
+ format!("[{level}] {text}")
+ };
+ if line.chars().count() > CONSOLE_LINE_MAX {
+ line = line.chars().take(CONSOLE_LINE_MAX).collect::<String>() + "…";
+ }
+ self.bytes += line.len();
+ self.lines.push(line);
+ }
+}
+
+/// Run `script` to completion. Blocks the calling thread.
+pub(crate) fn run(
+ script: &str,
+ platform: &str,
+ limits: Limits,
+ cancel: Arc<AtomicBool>,
+ calls: tokio::sync::mpsc::Sender<HostCall>,
+) -> Outcome {
+ let console = Arc::new(Mutex::new(Console::default()));
+ let used = Arc::new(AtomicU32::new(0));
+ let result = run_inner(script, platform, &limits, &cancel, calls, &console, &used);
+ let console = std::mem::take(&mut *console.lock().unwrap_or_else(|e| e.into_inner()));
+ Outcome {
+ result,
+ console: console.lines,
+ console_truncated: console.truncated,
+ calls: used.load(Ordering::SeqCst),
+ }
+}
+
+fn run_inner(
+ script: &str,
+ platform: &str,
+ limits: &Limits,
+ cancel: &Arc<AtomicBool>,
+ calls: tokio::sync::mpsc::Sender<HostCall>,
+ console: &Arc<Mutex<Console>>,
+ used: &Arc<AtomicU32>,
+) -> Result<Value, Failure> {
+ let internal = |error: rquickjs::Error| Failure::new(FailureKind::Internal, error.to_string());
+ let runtime = Runtime::new().map_err(internal)?;
+ runtime.set_memory_limit(limits.memory_bytes);
+ runtime.set_max_stack_size(limits.stack_bytes);
+ let deadline = limits.deadline;
+ {
+ let cancel = cancel.clone();
+ runtime.set_interrupt_handler(Some(Box::new(move || {
+ cancel.load(Ordering::SeqCst) || Instant::now() >= deadline
+ })));
+ }
+ let context = Context::full(&runtime).map_err(internal)?;
+ let settled: Arc<Mutex<Option<String>>> = Arc::new(Mutex::new(None));
+
+ let started = context.with(|ctx| -> Result<(), Failure> {
+ let globals = ctx.globals();
+ let host = {
+ let cancel = cancel.clone();
+ let used = used.clone();
+ let max_calls = limits.max_calls;
+ let platform = platform.to_owned();
+ Function::new(
+ ctx.clone(),
+ move |op: String, args: String, line: Option<u32>| -> String {
+ host_call(
+ &op,
+ &args,
+ line.unwrap_or(0),
+ &platform,
+ deadline,
+ max_calls,
+ &cancel,
+ &used,
+ &calls,
+ )
+ },
+ )
+ .map_err(internal)?
+ };
+ globals.set("__host", host).map_err(internal)?;
+ let done = {
+ let settled = settled.clone();
+ Function::new(ctx.clone(), move |encoded: String| {
+ let mut slot = settled.lock().unwrap_or_else(|e| e.into_inner());
+ if slot.is_none() {
+ *slot = Some(encoded);
+ }
+ })
+ .map_err(internal)?
+ };
+ globals.set("__done", done).map_err(internal)?;
+ let log = {
+ let console = console.clone();
+ Function::new(ctx.clone(), move |level: String, text: String| {
+ console
+ .lock()
+ .unwrap_or_else(|e| e.into_inner())
+ .push(&level, &text);
+ })
+ .map_err(internal)?
+ };
+ globals.set("__log", log).map_err(internal)?;
+
+ let mut prelude_options = EvalOptions::default();
+ prelude_options.global = true;
+ prelude_options.strict = true;
+ prelude_options.filename = Some("prelude.js".into());
+ let _: JsValue = ctx
+ .eval_with_options(include_str!("prelude.js"), prelude_options)
+ .map_err(|error| {
+ Failure::new(
+ FailureKind::Internal,
+ format!("prelude failed: {error}: {}", describe_exception(&ctx)),
+ )
+ })?;
+ let settle: Function = globals.get("__settle").map_err(internal)?;
+ globals.remove("__settle").map_err(internal)?;
+
+ // The script is the body of an async function, opened on its first
+ // line so its line numbers are the caller's.
+ let wrapped = format!("(async () => {{{script}\n}})()");
+ let mut options = EvalOptions::default();
+ options.global = true;
+ options.strict = false;
+ options.filename = Some("script.js".into());
+ let promise: JsValue = match ctx.eval_with_options(wrapped, options) {
+ Ok(value) => value,
+ Err(_) => return Err(exception_failure(&ctx, cancel, deadline, true)),
+ };
+ settle
+ .call::<_, ()>((promise,))
+ .map_err(|_| exception_failure(&ctx, cancel, deadline, false))?;
+ Ok(())
+ });
+ started?;
+
+ // Every await resolves synchronously (driver calls block this thread),
+ // so draining the job queue runs the script to its end.
+ loop {
+ match runtime.execute_pending_job() {
+ Ok(true) => continue,
+ Ok(false) => break,
+ Err(_) => {
+ if let Some(failure) = stop_reason(cancel, deadline) {
+ return Err(failure);
+ }
+ // An exception escaping a job is reported through __settle;
+ // keep draining.
+ }
+ }
+ }
+
+ let settled = settled.lock().unwrap_or_else(|e| e.into_inner()).take();
+ let Some(encoded) = settled else {
+ return Err(stop_reason(cancel, deadline).unwrap_or_else(|| {
+ Failure::new(
+ FailureKind::Stalled,
+ "the script awaited something that never settles (a promise nothing resolves); \
+ run_script has no timers or events, only driver calls",
+ )
+ }));
+ };
+ let outcome: Value = serde_json::from_str(&encoded)
+ .map_err(|error| Failure::new(FailureKind::Internal, format!("bad result: {error}")))?;
+ if outcome["ok"] == true {
+ return Ok(outcome["value"].clone());
+ }
+ if let Some(failure) = stop_reason(cancel, deadline) {
+ return Err(failure);
+ }
+ let name = outcome["name"].as_str();
+ let message = outcome["message"].as_str().unwrap_or("error");
+ // QuickJS reports an exhausted heap as a null exception or "out of memory".
+ let out_of_memory = (name.is_none() && message == "null") || message.contains("out of memory");
+ let mut failure = if out_of_memory {
+ Failure::new(
+ FailureKind::Memory,
+ format!(
+ "the script ran out of memory (limit {} MiB)",
+ limits.memory_bytes >> 20
+ ),
+ )
+ } else {
+ Failure::new(
+ FailureKind::Exception,
+ match name {
+ Some(name) if name != "CuaError" => format!("{name}: {message}"),
+ _ => message.to_owned(),
+ },
+ )
+ };
+ failure.line = outcome["line"].as_u64().map(|line| line as u32);
+ failure.op = outcome["op"].as_str().map(str::to_owned);
+ failure.code = outcome["code"].as_str().map(str::to_owned);
+ failure.call = outcome["call"].as_u64().map(|call| call as u32);
+ Err(failure)
+}
+
+#[allow(clippy::too_many_arguments)]
+fn host_call(
+ op: &str,
+ args: &str,
+ line: u32,
+ platform: &str,
+ deadline: Instant,
+ max_calls: u32,
+ cancel: &AtomicBool,
+ used: &AtomicU32,
+ calls: &tokio::sync::mpsc::Sender<HostCall>,
+) -> String {
+ let reply = |result: Result<Value, HostError>, number: Option<u32>| -> String {
+ match result {
+ Ok(value) => json!({"ok": true, "result": value}).to_string(),
+ Err(error) => json!({
+ "ok": false,
+ "error": error.message,
+ "code": error.code,
+ "call": number,
+ })
+ .to_string(),
+ }
+ };
+ if op == "platform" {
+ return reply(Ok(json!(platform)), None);
+ }
+ if cancel.load(Ordering::SeqCst) {
+ return reply(
+ Err(HostError::new("the script was cancelled", "cancelled")),
+ None,
+ );
+ }
+ let free = FREE_OPS.contains(&op);
+ let number = if free {
+ 0
+ } else {
+ let number = used.fetch_add(1, Ordering::SeqCst) + 1;
+ if number > max_calls {
+ used.fetch_sub(1, Ordering::SeqCst);
+ return reply(
+ Err(HostError::new(
+ format!("driver call limit reached (max_calls={max_calls})"),
+ "call_limit",
+ )),
+ None,
+ );
+ }
+ number
+ };
+ let args: Value = match serde_json::from_str(args) {
+ Ok(value @ Value::Object(_)) => value,
+ Ok(_) => {
+ return reply(
+ Err(HostError::new(
+ "arguments must be an object",
+ "invalid_args",
+ )),
+ Some(number),
+ )
+ }
+ Err(error) => {
+ return reply(
+ Err(HostError::new(
+ format!("bad arguments: {error}"),
+ "invalid_args",
+ )),
+ Some(number),
+ )
+ }
+ };
+ let (reply_tx, reply_rx) = std::sync::mpsc::sync_channel(1);
+ let request = HostCall {
+ op: op.to_owned(),
+ args,
+ line,
+ number,
+ reply: reply_tx,
+ };
+ if calls.blocking_send(request).is_err() {
+ cancel.store(true, Ordering::SeqCst);
+ return reply(
+ Err(HostError::new("the script was cancelled", "cancelled")),
+ Some(number),
+ );
+ }
+ let remaining = deadline.saturating_duration_since(Instant::now()) + Duration::from_millis(250);
+ match reply_rx.recv_timeout(remaining) {
+ Ok(result) => reply(result, (!free).then_some(number)),
+ Err(std::sync::mpsc::RecvTimeoutError::Timeout) => reply(
+ Err(HostError::new(
+ "the script's wall-time limit was reached",
+ "timeout",
+ )),
+ Some(number),
+ ),
+ Err(std::sync::mpsc::RecvTimeoutError::Disconnected) => {
+ cancel.store(true, Ordering::SeqCst);
+ reply(
+ Err(HostError::new("the script was cancelled", "cancelled")),
+ Some(number),
+ )
+ }
+ }
+}
+
+fn stop_reason(cancel: &AtomicBool, deadline: Instant) -> Option<Failure> {
+ if cancel.load(Ordering::SeqCst) {
+ Some(Failure::new(
+ FailureKind::Cancelled,
+ "the script was cancelled",
+ ))
+ } else if Instant::now() >= deadline {
+ Some(Failure::new(
+ FailureKind::Timeout,
+ "the script's wall-time limit was reached",
+ ))
+ } else {
+ None
+ }
+}
+
+fn exception_failure(
+ ctx: &rquickjs::Ctx<'_>,
+ cancel: &AtomicBool,
+ deadline: Instant,
+ at_parse: bool,
+) -> Failure {
+ let caught = ctx.catch();
+ if let Some(failure) = stop_reason(cancel, deadline) {
+ return failure;
+ }
+ let (name, message, stack) = match caught.as_exception() {
+ Some(exception) => {
+ let name: Option<String> = exception.get("name").ok();
+ (
+ name,
+ exception.message().unwrap_or_default(),
+ exception.stack().unwrap_or_default(),
+ )
+ }
+ None => (None, format!("{caught:?}"), String::new()),
+ };
+ let kind = if name.as_deref() == Some("SyntaxError") && at_parse {
+ FailureKind::Syntax
+ } else if message.contains("out of memory") {
+ FailureKind::Memory
+ } else {
+ FailureKind::Exception
+ };
+ let mut failure = Failure::new(
+ kind,
+ match name {
+ Some(name) => format!("{name}: {message}"),
+ None => message,
+ },
+ );
+ failure.line = script_line(&stack);
+ failure
+}
+
+fn describe_exception(ctx: &rquickjs::Ctx<'_>) -> String {
+ let caught = ctx.catch();
+ caught
+ .as_exception()
+ .and_then(|exception| exception.message())
+ .unwrap_or_else(|| format!("{caught:?}"))
+}
+
+fn script_line(stack: &str) -> Option<u32> {
+ let at = stack.find("script.js:")?;
+ stack[at + "script.js:".len()..]
+ .split(|c: char| !c.is_ascii_digit())
+ .next()?
+ .parse()
+ .ok()
+}
diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/script_tool/prelude.js b/libs/cua-driver/rust/crates/cua-driver-core/src/script_tool/prelude.js
new file mode 100644
index 0000000000..3a4426ab03
--- /dev/null
+++ b/libs/cua-driver/rust/crates/cua-driver-core/src/script_tool/prelude.js
@@ -0,0 +1,238 @@
+// The `cua` API a run_script script sees. Evaluated once before the script,
+// as prelude.js, so stack frames from script.js are always the caller's.
+//
+// The only bridge to the driver is `__host(op, argsJson, line)`, which this
+// prelude captures and then deletes from the global object.
+(() => {
+ "use strict";
+ const host = globalThis.__host;
+ const finish = globalThis.__done;
+ const emit = globalThis.__log;
+ delete globalThis.__host;
+ delete globalThis.__done;
+ delete globalThis.__log;
+
+ // Line of the innermost script.js frame that led to this call.
+ const scriptLine = (stack) => {
+ const match = /script\.js:(\d+)/.exec(String(stack || ""));
+ return match ? Number(match[1]) : 0;
+ };
+
+ const call = (op, args) => {
+ const line = scriptLine(new Error().stack);
+ const raw = host(op, JSON.stringify(args === undefined ? {} : args), line);
+ const reply = JSON.parse(raw);
+ if (!reply.ok) {
+ const error = new Error(`${op} failed: ${reply.error}`);
+ error.name = "CuaError";
+ error.op = op;
+ error.code = reply.code || null;
+ error.call = reply.call || null;
+ error.line = line || null;
+ throw error;
+ }
+ return reply.result;
+ };
+
+ const show = (value) => {
+ if (typeof value === "string") return value;
+ if (value instanceof Error) return `${value.name}: ${value.message}`;
+ try {
+ const text = JSON.stringify(value);
+ return text === undefined ? String(value) : text;
+ } catch (_) {
+ return String(value);
+ }
+ };
+ const console = {};
+ for (const level of ["log", "info", "warn", "error", "debug"]) {
+ console[level] = (...values) => emit(level, values.map(show).join(" "));
+ }
+ globalThis.console = console;
+
+ // An element target: a token string, an element row number from this
+ // app's last getState(), [x, y] window pixels, or {role, name, nth}.
+ const target = (app, value) => {
+ if (value === undefined || value === null) return {};
+ if (typeof value === "string") return { element_token: value };
+ if (typeof value === "number") {
+ if (!app.snapshotId) {
+ throw new Error(
+ `element ${value} needs a getState() first, or pass a token string or {role, name}`,
+ );
+ }
+ return { element_token: `${app.snapshotId}:${value}` };
+ }
+ if (Array.isArray(value)) return { x: value[0], y: value[1] };
+ if (typeof value === "object") {
+ const out = {};
+ for (const key of ["role", "name", "label", "nth", "element_token", "x", "y", "timeout_ms"]) {
+ if (value[key] !== undefined) out[key] = value[key];
+ }
+ return out;
+ }
+ throw new Error(`cannot target ${show(value)}`);
+ };
+
+ const keysOf = (keys) =>
+ Array.isArray(keys) ? keys : String(keys).split("+").map((key) => key.trim()).filter(Boolean);
+
+ class App {
+ constructor(info) {
+ this.pid = info.pid;
+ this.windowId = info.window_id;
+ this.name = info.app || null;
+ this.title = info.title || null;
+ this.snapshotId = null;
+ // Pinned to one window only when the caller asked for one; otherwise
+ // every call follows the app's frontmost window (dialogs included).
+ this.pinned = Boolean(info.pinned);
+ }
+ where() {
+ return this.pinned ? { pid: this.pid, window_id: this.windowId } : { pid: this.pid };
+ }
+ async window() {
+ const info = call("get_app", this.where());
+ this.windowId = info.window_id;
+ this.title = info.title || this.title;
+ return info;
+ }
+ async getState(options = {}) {
+ const info = await this.window();
+ const state = call("get_window_state", { pid: this.pid, window_id: info.window_id, ...options });
+ if (state && state.snapshot_id) this.snapshotId = state.snapshot_id;
+ return state;
+ }
+ async query(selector = {}) {
+ return call("find", { ...this.where(), ...selector });
+ }
+ async waitFor(selector, options = {}) {
+ return call("wait_for", { ...this.where(), ...selector, ...options });
+ }
+ async verify(expect, options = {}) {
+ const info = await this.window();
+ return call("verify_state", {
+ pid: this.pid,
+ window_id: info.window_id,
+ expect: Array.isArray(expect) ? expect : [expect],
+ ...options,
+ });
+ }
+ async click(on, options = {}) {
+ return call("click", { ...this.where(), ...target(this, on), ...options });
+ }
+ async doubleClick(on, options = {}) {
+ return call("double_click", { ...this.where(), ...target(this, on), ...options });
+ }
+ async rightClick(on, options = {}) {
+ return call("right_click", { ...this.where(), ...target(this, on), ...options });
+ }
+ async typeText(text, on, options = {}) {
+ return call("type_text", { ...this.where(), ...target(this, on), text: String(text), ...options });
+ }
+ async pressKey(key, on, options = {}) {
+ const parts = keysOf(key);
+ if (parts.length > 1) return this.hotkey(parts, on, options);
+ return call("press_key", { ...this.where(), ...target(this, on), key: parts[0], ...options });
+ }
+ async hotkey(keys, on, options = {}) {
+ return call("hotkey", { ...this.where(), ...target(this, on), keys: keysOf(keys), ...options });
+ }
+ async scroll(on, direction = "down", amount, options = {}) {
+ const args = { ...this.where(), ...target(this, on), direction, ...options };
+ if (amount !== undefined) args.amount = amount;
+ return call("scroll", args);
+ }
+ async setValue(on, value, options = {}) {
+ return call("set_value", { ...this.where(), ...target(this, on), value: String(value), ...options });
+ }
+ async drag(from, to, options = {}) {
+ const info = await this.window();
+ return call("drag", {
+ pid: this.pid,
+ window_id: info.window_id,
+ from_x: from[0],
+ from_y: from[1],
+ to_x: to[0],
+ to_y: to[1],
+ ...options,
+ });
+ }
+ async zoom(rect, options = {}) {
+ const info = await this.window();
+ const [x1, y1, x2, y2] = Array.isArray(rect) ? rect : [rect.x1, rect.y1, rect.x2, rect.y2];
+ return call("zoom", { pid: this.pid, window_id: info.window_id, x1, y1, x2, y2, ...options });
+ }
+ }
+
+ const appFrom = (info, pinned) => new App({ ...info, pinned });
+
+ const cua = {
+ computer: { target: JSON.parse(host("platform", "{}", 0)).result },
+ async getApp(which) {
+ if (which === undefined || which === null) throw new Error("getApp needs an app name, bundle id, or {pid, windowId}");
+ if (typeof which === "string") return appFrom(call("get_app", { app: which }), false);
+ const args = {};
+ if (which.pid !== undefined) args.pid = which.pid;
+ if (which.windowId !== undefined) args.window_id = which.windowId;
+ if (which.window_id !== undefined) args.window_id = which.window_id;
+ if (which.app !== undefined) args.app = which.app;
+ if (which.title !== undefined) args.window = which.title;
+ return appFrom(call("get_app", args), args.window_id !== undefined || args.window !== undefined);
+ },
+ async launch(which, options = {}) {
+ const args = typeof which === "string"
+ ? (/^[\w-]+(\.[\w-]+)+$/.test(which) ? { bundle_id: which } : { name: which })
+ : { ...which };
+ const launched = call("launch_app", { ...args, ...options });
+ const pid = launched && launched.pid;
+ if (!pid) return launched;
+ return appFrom(call("get_app", { pid, timeout_ms: 5000 }), false);
+ },
+ async listApps(options = {}) {
+ const listed = call("list_apps", options);
+ return (listed && listed.apps) || listed;
+ },
+ async listWindows(options = {}) {
+ const listed = call("list_windows", options);
+ return (listed && listed.windows) || listed;
+ },
+ async sleep(ms) {
+ return call("sleep", { ms });
+ },
+ /** Any allowed driver tool by name, with its own arguments. */
+ async call(tool, args = {}) {
+ return call(String(tool), args);
+ },
+ };
+ Object.freeze(cua.computer);
+ Object.freeze(cua);
+ Object.defineProperty(globalThis, "cua", { value: cua, writable: false, configurable: false });
+
+ globalThis.__settle = (promise) =>
+ promise.then(
+ (value) => {
+ let encoded;
+ try {
+ encoded = JSON.stringify({ ok: true, value: value === undefined ? null : value });
+ } catch (error) {
+ encoded = JSON.stringify({ ok: true, value: show(value), note: `not JSON: ${error}` });
+ }
+ finish(encoded);
+ },
+ (error) => {
+ const isError = error instanceof Error;
+ finish(
+ JSON.stringify({
+ ok: false,
+ name: isError ? error.name : null,
+ message: isError ? error.message : show(error),
+ line: (isError && (scriptLine(error.stack) || error.line)) || null,
+ op: (isError && error.op) || null,
+ code: (isError && error.code) || null,
+ call: (isError && error.call) || null,
+ }),
+ );
+ },
+ );
+})();
diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/script_tool/tests.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/script_tool/tests.rs
new file mode 100644
index 0000000000..883173c533
--- /dev/null
+++ b/libs/cua-driver/rust/crates/cua-driver-core/src/script_tool/tests.rs
@@ -0,0 +1,475 @@
+//! `run_script` against fake driver tools: results, errors, limits and
+//! sandbox-escape attempts.
+
+use std::sync::{Arc, Mutex};
+use std::time::{Duration, Instant};
+
+use async_trait::async_trait;
+use serde_json::{json, Value};
+
+use crate::{
+ protocol::{Content, ToolResult},
+ tool::{Tool, ToolDef, ToolRegistry},
+};
+
+type Calls = Arc<Mutex<Vec<(String, Value)>>>;
+
+/// Fake app: pid 42, window 7, a "Name" field and a "Save" button.
+struct Fake {
+ def: ToolDef,
+ calls: Calls,
+}
+
+#[async_trait]
+impl Tool for Fake {
+ fn def(&self) -> &ToolDef {
+ &self.def
+ }
+
+ async fn invoke(&self, args: Value) -> ToolResult {
+ let name = self.def.name.clone();
+ self.calls
+ .lock()
+ .unwrap()
+ .push((name.clone(), args.clone()));
+ match name.as_str() {
+ "list_windows" => ToolResult::text("windows").with_structured(json!({"windows": [{
+ "pid": 42, "window_id": 7, "app_name": "Notes", "title": "Draft",
+ "z_index": 1, "is_on_screen": true,
+ "bounds": {"x": 0.0, "y": 0.0, "width": 800.0, "height": 600.0}
+ }]})),
+ "list_apps" => ToolResult::text("apps").with_structured(json!({"apps": [
+ {"pid": 42, "name": "Notes", "bundle_id": "com.example.notes", "running": true, "active": true}
+ ]})),
+ "get_window_state" => ToolResult::text("- [0] AXWindow \"Draft\"").with_structured(json!({
+ "pid": 42, "window_id": 7, "snapshot_id": "s00000009",
+ "tree_markdown": "- [0] AXWindow \"Draft\"\n - [1] AXTextField \"Name\"\n - [2] AXButton \"Save\"\n - AXStaticText \"Saved\"\n",
+ "elements": [
+ {"element_index": 1, "element_token": "s00000009:1", "role": "AXTextField", "label": "Name", "value": "Ada", "depth": 1},
+ {"element_index": 2, "element_token": "s00000009:2", "role": "AXButton", "label": "Save", "depth": 1}
+ ]
+ })),
+ "launch_app" => ToolResult::text("launched").with_structured(json!({"pid": 42})),
+ _ if args["fail"] == true => ToolResult::error(format!("{name} refused: asked to fail")),
+ _ => ToolResult::text(format!("{name} done")),
+ }
+ }
+}
+
+struct Harness {
+ registry: Arc<ToolRegistry>,
+ calls: Calls,
+}
+
+impl Harness {
+ fn new() -> Self {
+ let calls: Calls = Arc::default();
+ let mut registry = ToolRegistry::new();
+ for name in crate::batch_tools::BATCHABLE_TOOLS
+ .iter()
+ .copied()
+ .chain(super::READ_TOOLS.iter().copied())
+ .chain(["kill_app"])
+ {
+ registry.register(Box::new(Fake {
+ def: ToolDef {
+ name: name.into(),
+ description: "fake".into(),
+ input_schema: json!({"type": "object"}),
+ read_only: matches!(name, "get_window_state" | "list_windows" | "list_apps"),
+ destructive: false,
+ idempotent: false,
+ open_world: false,
+ },
+ calls: calls.clone(),
+ }));
+ }
+ registry.register_session_tools();
+ registry.register_script_tool();
+ let registry = Arc::new(registry);
+ registry.init_self_weak();
+ Self { registry, calls }
+ }
+
+ async fn run(&self, script: &str) -> ToolResult {
+ self.run_with(json!({ "script": script })).await
+ }
+
+ async fn run_with(&self, args: Value) -> ToolResult {
+ self.registry.invoke(super::RUN_SCRIPT_TOOL, args).await
+ }
+
+ fn calls(&self, tool: &str) -> Vec<Value> {
+ self.calls
+ .lock()
+ .unwrap()
+ .iter()
+ .filter(|(name, _)| name == tool)
+ .map(|(_, args)| args.clone())
+ .collect()
+ }
+}
+
+fn text(result: &ToolResult) -> String {
+ result
+ .content
+ .iter()
+ .filter_map(|content| match content {
+ Content::Text { text, .. } => Some(text.as_str()),
+ _ => None,
+ })
+ .collect::<Vec<_>>()
+ .join("\n")
+}
+
+fn structured(result: &ToolResult) -> Value {
+ result.structured_content.clone().unwrap_or(Value::Null)
+}
+
+#[tokio::test]
+async fn a_script_acts_by_name_and_returns_value_console_and_call_log() {
+ let harness = Harness::new();
+ let result = harness
+ .run(
+ r#"const app = await cua.getApp("Notes");
+console.log("found", app.name, {window: app.windowId});
+await app.setValue({role: "textfield", name: "Name"}, "Grace");
+await app.click({role: "button", name: "Save"});
+const [field] = await app.query({role: "textbox", name: "name"});
+await app.waitFor({text: "Saved"});
+return {value: field.value, platform: cua.computer.target};"#,
+ )
+ .await;
+ assert_ne!(result.is_error, Some(true), "{}", text(&result));
+ let out = structured(&result);
+ assert_eq!(out["value"]["value"], "Ada");
+ assert_eq!(out["console"][0], "found Notes {\"window\":7}");
+ let click = &harness.calls("click")[0];
+ assert_eq!(click["element_token"], "s00000009:2");
+ assert_eq!(click["pid"], 42);
+ assert_eq!(click["window_id"], 7);
+ assert!(
+ click.get("role").is_none(),
+ "selector keys never reach the tool"
+ );
+ assert_eq!(harness.calls("set_value")[0]["value"], "Grace");
+ let log = out["calls"].as_array().unwrap();
+ assert!(
+ log.iter()
+ .any(|entry| entry["op"] == "click" && entry["line"] == 4),
+ "{log:?}"
+ );
+ assert!(
+ text(&result).contains("3. line 4 click ok"),
+ "{}",
+ text(&result)
+ );
+}
+
+#[tokio::test]
+async fn an_uncaught_driver_error_names_the_line_the_call_and_the_reason() {
+ let harness = Harness::new();
+ let result = harness
+ .run("const app = await cua.getApp('Notes');\n\nawait app.click({role: 'button', name: 'Publish'}, {timeout_ms: 0});\nreturn 1;")
+ .await;
+ assert_eq!(result.is_error, Some(true));
+ let message = text(&result);
+ assert!(
+ message.starts_with("run_script: ERROR (exception) at line 3, call 2 (click): click failed: no button \"Publish\""),
+ "{message}"
+ );
+ assert!(
+ message.contains("nearest: [2] AXButton \"Save\""),
+ "{message}"
+ );
+ let error = &structured(&result)["error"];
+ assert_eq!(error["line"], 3);
+ assert_eq!(error["code"], "not_found");
+ assert!(harness.calls("click").is_empty());
+}
+
+#[tokio::test]
+async fn a_script_can_catch_a_failed_call_and_carry_on() {
+ let harness = Harness::new();
+ let result = harness
+ .run("const app = await cua.getApp('Notes');\ntry { await app.click('s00000009:2', {fail: true}); } catch (e) { console.log(e.code, e.op); }\nawait app.pressKey('cmd+s');\nreturn 'ok';")
+ .await;
+ assert_ne!(result.is_error, Some(true), "{}", text(&result));
+ assert_eq!(structured(&result)["console"][0], "tool_error click");
+ assert_eq!(harness.calls("hotkey")[0]["keys"], json!(["cmd", "s"]));
+}
+
+#[tokio::test]
+async fn the_sandbox_has_no_io_and_no_way_around_the_api() {
+ let harness = Harness::new();
+ let result = harness
+ .run(
+ r#"const probes = {};
+for (const name of ["require", "process", "fetch", "std", "os", "XMLHttpRequest", "WebSocket", "setTimeout", "Deno", "Bun", "__host", "__done", "__log", "__settle"]) {
+ probes[name] = typeof globalThis[name];
+}
+probes.viaFunction = typeof Function("return this")().__host;
+probes.viaConstructor = (() => {}).constructor.constructor("return typeof process")();
+try { await import("fs"); probes.import = "loaded"; } catch (e) { probes.import = String(e); }
+try { cua.call = null; probes.frozen = cua.call === null ? "mutated" : "frozen"; } catch (e) { probes.frozen = "frozen"; }
+return probes;"#,
+ )
+ .await;
+ assert_ne!(result.is_error, Some(true), "{}", text(&result));
+ let probes = &structured(&result)["value"];
+ for name in [
+ "require",
+ "process",
+ "fetch",
+ "std",
+ "os",
+ "XMLHttpRequest",
+ "WebSocket",
+ "setTimeout",
+ "Deno",
+ "Bun",
+ "__host",
+ "__done",
+ "__log",
+ "__settle",
+ "viaFunction",
+ ] {
+ assert_eq!(probes[name], "undefined", "{name}: {probes}");
+ }
+ assert_eq!(probes["viaConstructor"], "undefined");
+ assert!(
+ probes["import"]
+ .as_str()
+ .unwrap()
+ .contains("could not load module"),
+ "{probes}"
+ );
+ assert_eq!(probes["frozen"], "frozen");
+}
+
+#[tokio::test]
+async fn only_driver_tools_on_the_allowlist_are_reachable() {
+ let harness = Harness::new();
+ for (script, needle) in [
+ (
+ "await cua.call('kill_app', {pid: 42})",
+ "not available to scripts",
+ ),
+ (
+ "await cua.call('run_actions', {steps: []})",
+ "not available to scripts",
+ ),
+ (
+ "await cua.call('run_script', {script: '1'})",
+ "not available to scripts",
+ ),
+ (
+ "await cua.call('set_config', {key: 'experimental_script', value: true})",
+ "not available to scripts",
+ ),
+ (
+ "await cua.call('click', {pid: 42, _observation_only: true})",
+ "reserved",
+ ),
+ (
+ "await cua.call('click', {pid: 42, session: 'someone-else'})",
+ "session",
+ ),
+ ] {
+ let result = harness.run(script).await;
+ assert_eq!(result.is_error, Some(true), "{script}");
+ assert!(
+ text(&result).contains(needle),
+ "{script}: {}",
+ text(&result)
+ );
+ }
+ assert!(harness.calls("kill_app").is_empty());
+ assert!(harness.calls("click").is_empty());
+}
+
+#[tokio::test]
+async fn an_endless_loop_stops_at_the_wall_time_limit() {
+ let harness = Harness::new();
+ let started = Instant::now();
+ let result = harness
+ .run_with(json!({"script": "while (true) {}", "timeout_ms": 300}))
+ .await;
+ assert_eq!(result.is_error, Some(true));
+ assert_eq!(structured(&result)["error"]["kind"], "timeout");
+ assert!(
+ started.elapsed() < Duration::from_secs(3),
+ "{:?}",
+ started.elapsed()
+ );
+
+ // A catch block cannot swallow the interrupt.
+ let result = harness
+ .run_with(json!({"script": "try { while (true) {} } catch (e) {} return 'escaped';", "timeout_ms": 300}))
+ .await;
+ assert_eq!(structured(&result)["error"]["kind"], "timeout");
+
+ // Nor can a long sleep outlast the limit.
+ let started = Instant::now();
+ let result = harness
+ .run_with(json!({"script": "await cua.sleep(60000); return 'late';", "timeout_ms": 300}))
+ .await;
+ assert_eq!(
+ structured(&result)["error"]["kind"],
+ "timeout",
+ "{}",
+ text(&result)
+ );
+ assert!(started.elapsed() < Duration::from_secs(3));
+}
+
+#[tokio::test]
+async fn memory_and_stack_are_bounded() {
+ let harness = Harness::new();
+ let result = harness
+ .run("const chunks = []; while (true) chunks.push('x'.repeat(1 << 16));")
+ .await;
+ assert_eq!(result.is_error, Some(true));
+ assert_eq!(
+ structured(&result)["error"]["kind"],
+ "memory",
+ "{}",
+ text(&result)
+ );
+
+ let result = harness
+ .run("function down() { return down(); }\ndown();")
+ .await;
+ assert_eq!(result.is_error, Some(true));
+ assert!(
+ text(&result).contains("Maximum call stack size exceeded"),
+ "{}",
+ text(&result)
+ );
+ assert_eq!(structured(&result)["error"]["line"], 1);
+}
+
+#[tokio::test]
+async fn the_driver_call_limit_holds() {
+ let harness = Harness::new();
+ let result = harness
+ .run_with(json!({
+ "script": "for (let i = 0; i < 10; i++) await cua.call('press_key', {pid: 42, key: 'a'});",
+ "max_calls": 3
+ }))
+ .await;
+ assert_eq!(result.is_error, Some(true));
+ assert!(
+ text(&result).contains("driver call limit reached (max_calls=3)"),
+ "{}",
+ text(&result)
+ );
+ assert_eq!(structured(&result)["error"]["code"], "call_limit");
+ assert_eq!(harness.calls("press_key").len(), 3);
+}
+
+#[tokio::test]
+async fn syntax_errors_and_stalled_promises_are_named() {
+ let harness = Harness::new();
+ let result = harness.run("const a = 1;\nconst b = ;").await;
+ assert_eq!(
+ structured(&result)["error"]["kind"],
+ "syntax",
+ "{}",
+ text(&result)
+ );
+ assert_eq!(structured(&result)["error"]["line"], 2);
+
+ let result = harness.run("await new Promise(() => {});").await;
+ assert_eq!(
+ structured(&result)["error"]["kind"],
+ "stalled",
+ "{}",
+ text(&result)
+ );
+
+ let result = harness.run("throw new TypeError('nope');").await;
+ assert!(
+ text(&result).contains("ERROR (exception) at line 1: TypeError: nope"),
+ "{}",
+ text(&result)
+ );
+}
+
+#[tokio::test]
+async fn dropping_the_request_cancels_the_script() {
+ let harness = Harness::new();
+ let script = "while (true) { await cua.call('list_apps', {}); await cua.sleep(20); }";
+ let call = harness.run_with(json!({"script": script, "max_calls": 500, "timeout_ms": 60000}));
+ let _ = tokio::time::timeout(Duration::from_millis(300), call).await;
+ tokio::time::sleep(Duration::from_millis(200)).await;
+ let after_cancel = harness.calls("list_apps").len();
+ assert!(after_cancel > 0, "the script ran");
+ tokio::time::sleep(Duration::from_millis(400)).await;
+ assert_eq!(
+ harness.calls("list_apps").len(),
+ after_cancel,
+ "no driver call after the request was dropped"
+ );
+}
+
+#[tokio::test]
+async fn bad_requests_are_rejected_before_running() {
+ let harness = Harness::new();
+ for args in [
+ json!({}),
+ json!({"script": ""}),
+ json!({"script": "1", "timeout_ms": 999999}),
+ json!({"script": "1", "max_calls": 0}),
+ json!({"script": "1", "extra": true}),
+ ] {
+ let result = harness.run_with(args.clone()).await;
+ assert_eq!(result.is_error, Some(true), "{args}");
+ }
+}
+
+#[test]
+fn the_tool_is_off_unless_the_operator_opts_in() {
+ let dir = tempfile::tempdir().unwrap();
+ let config = dir.path().join("config.json");
+ assert!(!super::enabled_from(None, None), "off by default");
+ assert!(!super::enabled_from(None, Some(&config)), "no config file");
+ std::fs::write(&config, r#"{"experimental_pip": true}"#).unwrap();
+ assert!(!super::enabled_from(None, Some(&config)));
+ std::fs::write(&config, r#"{"experimental_script": true}"#).unwrap();
+ assert!(super::enabled_from(None, Some(&config)), "config opt-in");
+ assert!(
+ !super::enabled_from(Some("0"), Some(&config)),
+ "the environment wins"
+ );
+ assert!(super::enabled_from(Some("1"), None));
+ assert!(super::enabled_from(Some(" TRUE "), None));
+ assert!(!super::enabled_from(Some("maybe"), None));
+}
+
+#[test]
+fn an_unflagged_registry_does_not_offer_the_tool() {
+ let mut registry = ToolRegistry::new();
+ registry.register_session_tools();
+ // The flag is off in the test environment unless someone set it.
+ if !super::enabled() {
+ assert!(registry.get_def(super::RUN_SCRIPT_TOOL).is_none());
+ }
+ assert!(registry
+ .get_def(crate::batch_tools::RUN_ACTIONS_TOOL)
+ .is_some());
+}
+
+#[tokio::test]
+async fn query_returns_static_text_rows_with_their_value() {
+ let harness = Harness::new();
+ let result = harness
+ .run("const app = await cua.getApp('Notes');\nconst rows = await app.query({role: 'statictext', name: 'Saved'});\nconst both = await app.query({name: 'Save'});\nreturn {rows, kinds: both.map(r => r.display_only ? 'text' : r.role)};")
+ .await;
+ assert_ne!(result.is_error, Some(true), "{}", text(&result));
+ let value = &structured(&result)["value"];
+ assert_eq!(value["rows"][0]["role"], "AXStaticText");
+ assert_eq!(value["rows"][0]["label"], "Saved");
+ assert_eq!(value["rows"][0]["window"]["window_id"], 7);
+ assert_eq!(value["kinds"], json!(["AXButton", "text"]));
+}
diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/session_manifest.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/session_manifest.rs
index 3f799da033..6b399a2db1 100644
--- a/libs/cua-driver/rust/crates/cua-driver-core/src/session_manifest.rs
+++ b/libs/cua-driver/rust/crates/cua-driver-core/src/session_manifest.rs
@@ -986,6 +986,7 @@ pub fn load_manifest(path: &Path) -> Result<SessionManifest, String> {
"hotkey",
"set_value",
"run_actions",
+ "run_script",
"mouse_button_down",
"mouse_button_up",
"mouse_drag",
diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/tool.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/tool.rs
index 5f68877d2c..dd730c93a4 100644
--- a/libs/cua-driver/rust/crates/cua-driver-core/src/tool.rs
+++ b/libs/cua-driver/rust/crates/cua-driver-core/src/tool.rs
@@ -548,6 +548,7 @@ pub fn default_capabilities_for(tool_name: &str) -> Vec<String> {
"get_recording_state" => &["recording.state"],
"replay_trajectory" => &["recording.replay"],
"run_actions" => &["input.batch"],
+ "run_script" => &["input.script"],
"install_ffmpeg" => &["recording.install_dependency"],
"install_extension" => &["extension.install"],
@@ -1088,6 +1089,20 @@ impl ToolRegistry {
self.register(Box::new(crate::batch_tools::RunActionsTool::new(
self.replay_registry.clone(),
)));
+ // Experimental and opt-in: only when the operator turned it on.
+ #[cfg(feature = "script")]
+ if crate::script_tool::enabled() {
+ self.register_script_tool();
+ }
+ }
+
+ /// Register the experimental `run_script` tool, which, like
+ /// `run_actions`, re-enters this registry for every driver call.
+ #[cfg(feature = "script")]
+ pub fn register_script_tool(&mut self) {
+ self.register(Box::new(crate::script_tool::RunScriptTool::new(
+ self.replay_registry.clone(),
+ )));
}
pub fn register_perception_tool(
@@ -1889,6 +1904,7 @@ impl ToolRegistry {
| "get_recording_state"
| "replay_trajectory"
| "run_actions"
+ | "run_script"
| "start_session"
| "end_session"
);
diff --git a/libs/cua-spacesd/Cargo.lock b/libs/cua-spacesd/Cargo.lock
index 50308378e0..aeff0100f2 100644
--- a/libs/cua-spacesd/Cargo.lock
+++ b/libs/cua-spacesd/Cargo.lock
@@ -1390,6 +1390,7 @@ dependencies = [
"libc",
"regex",
"regorus",
+ "rquickjs",
"serde",
"serde_json",
"serde_yaml_ng",
@@ -2732,6 +2733,11 @@ name = "hashbrown"
version = "0.17.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a"
+dependencies = [
+ "allocator-api2",
+ "equivalent",
+ "foldhash 0.2.0",
+]
[[package]]
name = "heapless"
@@ -5068,6 +5074,15 @@ dependencies = [
"uuid",
]
+[[package]]
+name = "relative-path"
+version = "2.0.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f2a073568cd8b9c4825429d3e9fc5f52851b4f97cc1348800689408921e6b8f8"
+dependencies = [
+ "serde",
+]
+
[[package]]
name = "reqwest"
version = "0.12.28"
@@ -5120,6 +5135,35 @@ dependencies = [
"windows-sys 0.52.0",
]
+[[package]]
+name = "rquickjs"
+version = "0.14.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a0a22bf72f515cc4189dd7c63d016f6de92a11eee98ddf7dd64794198f6eb34e"
+dependencies = [
+ "rquickjs-core",
+]
+
+[[package]]
+name = "rquickjs-core"
+version = "0.14.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6657979a24d543e5fc66f32cef2f47ca82ce8eb55cec5f7d59a12b7b570bbf24"
+dependencies = [
+ "hashbrown 0.17.1",
+ "relative-path",
+ "rquickjs-sys",
+]
+
+[[package]]
+name = "rquickjs-sys"
+version = "0.14.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cee271d0eeba64f0915b846cb7ae02e16faf3dfdffdca91731101d9d30fe3423"
+dependencies = [
+ "cc",
+]
+
[[package]]
name = "rusqlite"
version = "0.40.2"
@@ -5185,6 +5229,18 @@ dependencies = [
"zeroize",
]
+[[package]]
+name = "rustls-native-certs"
+version = "0.8.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dab5152771c58876a2146916e53e35057e1a4dfa2b9df0f0305b07f611fdea4d"
+dependencies = [
+ "openssl-probe",
+ "rustls-pki-types",
+ "schannel",
+ "security-framework",
+]
+
[[package]]
name = "rustls-pki-types"
version = "1.15.1"
@@ -7932,14 +7988,3 @@ dependencies = [
"syn 3.0.4",
"winnow 1.0.4",
]
-[[package]]
-name = "rustls-native-certs"
-version = "0.8.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dab5152771c58876a2146916e53e35057e1a4dfa2b9df0f0305b07f611fdea4d"
-dependencies = [
- "openssl-probe",
- "rustls-pki-types",
- "schannel",
- "security-framework 3.7.0",
-]
diff --git a/libs/cua-spacesd/tests/spaces-e2e/Cargo.lock b/libs/cua-spacesd/tests/spaces-e2e/Cargo.lock
index 051f75f186..47fda22ef0 100644
--- a/libs/cua-spacesd/tests/spaces-e2e/Cargo.lock
+++ b/libs/cua-spacesd/tests/spaces-e2e/Cargo.lock
@@ -2113,6 +2113,7 @@ dependencies = [
"libc",
"regex",
"regorus",
+ "rquickjs",
"serde",
"serde_json",
"serde_yaml_ng",
@@ -3906,6 +3907,11 @@ name = "hashbrown"
version = "0.17.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a"
+dependencies = [
+ "allocator-api2",
+ "equivalent",
+ "foldhash 0.2.0",
+]
[[package]]
name = "heapless"
@@ -6588,6 +6594,15 @@ dependencies = [
"uuid",
]
+[[package]]
+name = "relative-path"
+version = "2.0.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f2a073568cd8b9c4825429d3e9fc5f52851b4f97cc1348800689408921e6b8f8"
+dependencies = [
+ "serde",
+]
+
[[package]]
name = "reqwest"
version = "0.12.28"
@@ -6716,6 +6731,35 @@ dependencies = [
"tracing",
]
+[[package]]
+name = "rquickjs"
+version = "0.14.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a0a22bf72f515cc4189dd7c63d016f6de92a11eee98ddf7dd64794198f6eb34e"
+dependencies = [
+ "rquickjs-core",
+]
+
+[[package]]
+name = "rquickjs-core"
+version = "0.14.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6657979a24d543e5fc66f32cef2f47ca82ce8eb55cec5f7d59a12b7b570bbf24"
+dependencies = [
+ "hashbrown 0.17.1",
+ "relative-path",
+ "rquickjs-sys",
+]
+
+[[package]]
+name = "rquickjs-sys"
+version = "0.14.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cee271d0eeba64f0915b846cb7ae02e16faf3dfdffdca91731101d9d30fe3423"
+dependencies = [
+ "cc",
+]
+
[[package]]
name = "rusqlite"
version = "0.40.2"
diff --git a/libs/cua-spacesd/tests/teleport-e2e/Cargo.lock b/libs/cua-spacesd/tests/teleport-e2e/Cargo.lock
index d0ce459d9d..ddbab72219 100644
--- a/libs/cua-spacesd/tests/teleport-e2e/Cargo.lock
+++ b/libs/cua-spacesd/tests/teleport-e2e/Cargo.lock
@@ -1309,6 +1309,7 @@ dependencies = [
"libc",
"regex",
"regorus",
+ "rquickjs",
"serde",
"serde_json",
"serde_yaml_ng",
@@ -2430,6 +2431,11 @@ name = "hashbrown"
version = "0.17.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a"
+dependencies = [
+ "allocator-api2",
+ "equivalent",
+ "foldhash 0.2.0",
+]
[[package]]
name = "heapless"
@@ -4676,6 +4682,15 @@ dependencies = [
"uuid",
]
+[[package]]
+name = "relative-path"
+version = "2.0.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f2a073568cd8b9c4825429d3e9fc5f52851b4f97cc1348800689408921e6b8f8"
+dependencies = [
+ "serde",
+]
+
[[package]]
name = "reqwest"
version = "0.12.28"
@@ -4728,6 +4743,35 @@ dependencies = [
"windows-sys 0.52.0",
]
+[[package]]
+name = "rquickjs"
+version = "0.14.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a0a22bf72f515cc4189dd7c63d016f6de92a11eee98ddf7dd64794198f6eb34e"
+dependencies = [
+ "rquickjs-core",
+]
+
+[[package]]
+name = "rquickjs-core"
+version = "0.14.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6657979a24d543e5fc66f32cef2f47ca82ce8eb55cec5f7d59a12b7b570bbf24"
+dependencies = [
+ "hashbrown 0.17.1",
+ "relative-path",
+ "rquickjs-sys",
+]
+
+[[package]]
+name = "rquickjs-sys"
+version = "0.14.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cee271d0eeba64f0915b846cb7ae02e16faf3dfdffdca91731101d9d30fe3423"
+dependencies = [
+ "cc",
+]
+
[[package]]
name = "rusqlite"
version = "0.40.2"
@@ -4784,6 +4828,18 @@ dependencies = [
"zeroize",
]
+[[package]]
+name = "rustls-native-certs"
+version = "0.8.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dab5152771c58876a2146916e53e35057e1a4dfa2b9df0f0305b07f611fdea4d"
+dependencies = [
+ "openssl-probe",
+ "rustls-pki-types",
+ "schannel",
+ "security-framework",
+]
+
[[package]]
name = "rustls-pki-types"
version = "1.15.1"
@@ -7376,14 +7432,3 @@ dependencies = [
"syn 3.0.4",
"winnow 1.0.4",
]
-[[package]]
-name = "rustls-native-certs"
-version = "0.8.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dab5152771c58876a2146916e53e35057e1a4dfa2b9df0f0305b07f611fdea4d"
-dependencies = [
- "openssl-probe",
- "rustls-pki-types",
- "schannel",
- "security-framework 3.7.0",
-]