Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
feat(cua-driver): add browser dialogs and uploads
trycua-release[bot]21ca68928e54
diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/browser/cdp_ws.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/browser/cdp_ws.rsindex 122438242c..c36d2ae2a6 100644--- a/libs/cua-driver/rust/crates/cua-driver-core/src/browser/cdp_ws.rs+++ b/libs/cua-driver/rust/crates/cua-driver-core/src/browser/cdp_ws.rs@@ -61,6 +61,14 @@ pub struct CdpEvent {pub params: Value,}+/// Bounded state for one page-owned JavaScript dialog. Message text and URLs+/// are deliberately not retained.+#[derive(Debug, Clone, PartialEq, Eq)]+pub struct CdpDialogState {+ pub generation: u64,+ pub kind: String,+}+/// What the reader routed back for one in-flight call.enum CallOutcome {Result(Value),@@ -71,6 +79,8 @@ enum CallOutcome {struct Demux {pending: StdMutex<HashMap<u64, oneshot::Sender<CallOutcome>>>,subscribers: StdMutex<Vec<mpsc::UnboundedSender<CdpEvent>>>,+ dialogs: StdMutex<HashMap<String, CdpDialogState>>,+ next_dialog_generation: AtomicU64,closed: AtomicBool,}@@ -81,6 +91,7 @@ impl Demux {// error, which `call` reports as a closed socket.self.pending.lock().unwrap().clear();self.subscribers.lock().unwrap().clear();+ self.dialogs.lock().unwrap().clear();}}@@ -126,6 +137,32 @@ async fn read_loop(mut read: SplitStream<WsStream>, demux: Arc<Demux>) {.map(str::to_owned),params: v.get("params").cloned().unwrap_or(Value::Null),};+ if let Some(session_id) = event.session_id.as_deref() {+ match event.method.as_str() {+ "Page.javascriptDialogOpening" => {+ let kind = match event.params.get("type").and_then(Value::as_str) {+ Some("alert") => "alert",+ Some("confirm") => "confirm",+ Some("prompt") => "prompt",+ Some("beforeunload") => "beforeunload",+ _ => "other",+ };+ let generation =+ demux.next_dialog_generation.fetch_add(1, Ordering::Relaxed);+ demux.dialogs.lock().unwrap().insert(+ session_id.to_owned(),+ CdpDialogState {+ generation,+ kind: kind.to_owned(),+ },+ );+ }+ "Page.javascriptDialogClosed" => {+ demux.dialogs.lock().unwrap().remove(session_id);+ }+ _ => {}+ }+ }demux.subscribers.lock()@@ -162,6 +199,8 @@ impl CdpConnection {let demux = Arc::new(Demux {pending: StdMutex::new(HashMap::new()),subscribers: StdMutex::new(Vec::new()),+ dialogs: StdMutex::new(HashMap::new()),+ next_dialog_generation: AtomicU64::new(1),closed: AtomicBool::new(false),});let reader = tokio::spawn(read_loop(read, demux.clone()));@@ -190,6 +229,17 @@ impl CdpConnection {rx}+ pub fn dialog_state(&self, session_id: &str) -> Option<CdpDialogState> {+ self.demux.dialogs.lock().unwrap().get(session_id).cloned()+ }++ pub fn clear_dialog_state(&self, session_id: &str, generation: u64) {+ let mut dialogs = self.demux.dialogs.lock().unwrap();+ if dialogs.get(session_id).map(|state| state.generation) == Some(generation) {+ dialogs.remove(session_id);+ }+ }+/// Issue one CDP command and await its `id`-matched reply./// `session_id` targets an attached (flattened) target session./// Returns the `result` object, or an error carrying the CDP@@ -589,6 +639,39 @@ mod tests {assert!(events.try_recv().is_err(), "no phantom events");}+ #[tokio::test]+ async fn javascript_dialog_journal_is_bounded_and_generation_checked() {+ let server = MockCdpServer::start(StdArc::new(|call| match call.method.as_str() {+ "Dialog.open" => MockReply::ok(json!({})).with_events(vec![MockEvent {+ method: "Page.javascriptDialogOpening".into(),+ session_id: Some("tab-session".into()),+ params: json!({+ "type": "prompt",+ "message": "private dialog text",+ "url": "https://private.example"+ }),+ }]),+ "Dialog.close" => MockReply::ok(json!({})).with_events(vec![MockEvent {+ method: "Page.javascriptDialogClosed".into(),+ session_id: Some("tab-session".into()),+ params: json!({}),+ }]),+ _ => MockReply::ok(json!({})),+ }))+ .await;+ let conn = CdpConnection::connect(&server.ws_url()).await.unwrap();+ conn.call(Some("tab-session"), "Dialog.open", json!({}))+ .await+ .unwrap();+ let dialog = conn.dialog_state("tab-session").expect("dialog journaled");+ assert_eq!(dialog.kind, "prompt");+ assert!(!format!("{dialog:?}").contains("private"));+ conn.call(Some("tab-session"), "Dialog.close", json!({}))+ .await+ .unwrap();+ assert!(conn.dialog_state("tab-session").is_none());+ }+#[tokio::test]async fn out_of_order_replies_route_to_the_right_callers() {// Raw server: read two requests, reply to the SECOND first.diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/browser/semantic.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/browser/semantic.rsindex 956dc47a75..b2348ef831 100644--- a/libs/cua-driver/rust/crates/cua-driver-core/src/browser/semantic.rs+++ b/libs/cua-driver/rust/crates/cua-driver-core/src/browser/semantic.rs@@ -773,7 +773,18 @@ fn action_kinds(.get("contenteditable").is_some_and(|value| value.is_empty() || value.eq_ignore_ascii_case("true"))});- if matches!(role, "textbox" | "searchbox") || matches!(tag, "input" | "textarea") || editable {+ let file_input = tag == "input"+ && dom.is_some_and(|meta| {+ meta.attrs+ .get("type")+ .is_some_and(|value| value.eq_ignore_ascii_case("file"))+ });+ if file_input {+ actions.push(BrowserActionKind::Upload);+ } else if matches!(role, "textbox" | "searchbox")+ || matches!(tag, "input" | "textarea")+ || editable+ {actions.push(BrowserActionKind::Type);}if actions.is_empty()@@ -1073,6 +1084,22 @@ mod tests {use serde_json::json;use super::*;++ #[test]+ fn file_inputs_expose_upload_instead_of_text_typing() {+ let dom = DomMeta {+ tag: "input".into(),+ attrs: HashMap::from([("type".into(), "file".into())]),+ order: 0,+ css_hidden: false,+ parent_backend_node_id: None,+ frame_id: None,+ };+ assert_eq!(+ action_kinds("textbox", Some(&dom), &BTreeMap::new(), None),+ vec![BrowserActionKind::Upload]+ );+ }use crate::browser::store::FrameRef;fn frame() -> FrameRef {diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/browser/store.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/browser/store.rsindex cb4939e4b6..7bc9740510 100644--- a/libs/cua-driver/rust/crates/cua-driver-core/src/browser/store.rs+++ b/libs/cua-driver/rust/crates/cua-driver-core/src/browser/store.rs@@ -32,6 +32,7 @@ use super::types::{BindingQuality, ProcessFingerprint, Rect};pub enum BrowserActionKind {Click,Type,+ Upload,}impl BrowserActionKind {@@ -39,6 +40,7 @@ impl BrowserActionKind {match self {Self::Click => "click",Self::Type => "type",+ Self::Upload => "upload",}}}diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/browser/tools.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/browser/tools.rsindex c0d2072ea0..1906a18113 100644--- a/libs/cua-driver/rust/crates/cua-driver-core/src/browser/tools.rs+++ b/libs/cua-driver/rust/crates/cua-driver-core/src/browser/tools.rs@@ -1,5 +1,4 @@-//! The five browser tools: get_browser_state, browser_prepare,-//! browser_navigate, browser_click, browser_type.+//! Browser state, preparation, navigation, input, and page-owned dialog tools.//!//! Schemas, annotations, and exact-or-refused semantics live here; OS//! identity comes from the [`BrowserPlatform`] adapter; CDP mechanics@@ -31,6 +30,8 @@ pub fn register_browser_tools(engine: &Arc<BrowserEngine>, registry: &mut ToolReregistry.register(Box::new(BrowserNavigateTool::new(engine.clone())));registry.register(Box::new(BrowserClickTool::new(engine.clone())));registry.register(Box::new(BrowserTypeTool::new(engine.clone())));+ registry.register(Box::new(BrowserDialogTool::new(engine.clone())));+ registry.register(Box::new(BrowserSetInputFilesTool::new(engine.clone())));}// ── Shared helpers ───────────────────────────────────────────────────────────@@ -1405,6 +1406,353 @@ impl Tool for BrowserTypeTool {}}+// ── browser_dialog ──────────────────────────────────────────────────────────++pub struct BrowserDialogTool {+ def: ToolDef,+ engine: Arc<BrowserEngine>,+}++impl BrowserDialogTool {+ pub fn new(engine: Arc<BrowserEngine>) -> Self {+ Self {+ def: ToolDef {+ name: "browser_dialog".into(),+ description: "Inspect or resolve a page-owned JavaScript alert, confirm, prompt, or beforeunload dialog on one exactly-bound tab. This never handles browser permission UI, extension UI, native dialogs, or file pickers. Inspect returns an opaque dialog_id; accept/dismiss require that exact current id.".into(),+ input_schema: json!({+ "type": "object",+ "properties": {+ "target_id": schema_target_id(),+ "tab_id": schema_tab_id(),+ "session": schema_session(),+ "action": { "type": "string", "enum": ["inspect", "accept", "dismiss"] },+ "dialog_id": { "type": "string", "description": "Opaque current dialog generation returned by action=inspect." },+ "prompt_text": { "type": "string", "description": "Sensitive response text, valid only when accepting a prompt dialog." }+ },+ "required": ["target_id", "tab_id", "action"],+ "additionalProperties": true+ }),+ read_only: false,+ destructive: false,+ idempotent: false,+ open_world: false,+ },+ engine,+ }+ }+}++#[async_trait]+impl Tool for BrowserDialogTool {+ fn def(&self) -> &ToolDef {+ &self.def+ }++ async fn invoke(&self, args: Value) -> ToolResult {+ let (target_id, tab_id, action) = match (+ args.require_str("target_id"),+ args.require_str("tab_id"),+ args.require_str("action"),+ ) {+ (Ok(target), Ok(tab), Ok(action)) => (target, tab, action),+ (Err(error), _, _) | (_, Err(error), _) | (_, _, Err(error)) => return error,+ };+ if !matches!(action.as_str(), "inspect" | "accept" | "dismiss") {+ return ToolResult::error("action must be inspect, accept, or dismiss");+ }+ let session = match require_explicit_session(&args) {+ Ok(session) => session,+ Err(error) => return error,+ };+ let _mutation = match self+ .engine+ .lock_mutation(&session, &target_id, &tab_id)+ .await+ {+ Ok(guard) => guard,+ Err(refusal) => return refusal.to_tool_result(),+ };+ let validated = match self+ .engine+ .revalidate_for_mutation(&session, &target_id, Some(&tab_id))+ .await+ {+ Ok(validated) => validated,+ Err(refusal) => return refusal.to_tool_result(),+ };+ let conn = &validated.conn;+ let cdp_session = validated.cdp_session.as_str();+ if let Err(error) = conn.call(Some(cdp_session), "Page.enable", json!({})).await {+ return BrowserRefusal::new(+ BrowserRefusalCode::BrowserActionUnavailable,+ format!("the exact tab does not expose JavaScript dialog events: {error}"),+ )+ .to_tool_result();+ }+ tokio::task::yield_now().await;++ let Some(dialog) = conn.dialog_state(cdp_session) else {+ return if action == "inspect" {+ ToolResult::text(format!(+ "no page-owned JavaScript dialog is open in {tab_id}"+ ))+ .with_structured(json!({+ "status": "ok", "target_id": target_id, "tab_id": tab_id,+ "present": false+ }))+ } else {+ BrowserRefusal::new(+ BrowserRefusalCode::BrowserActionUnavailable,+ "no current page-owned JavaScript dialog exists on the exact tab",+ )+ .to_tool_result()+ };+ };+ let dialog_id = format!("dialog-{}", dialog.generation);+ if action == "inspect" {+ return ToolResult::text(format!("{} dialog is open in {tab_id}", dialog.kind))+ .with_structured(json!({+ "status": "ok", "target_id": target_id, "tab_id": tab_id,+ "present": true, "dialog_id": dialog_id, "kind": dialog.kind+ }));+ }++ let supplied_id = match args.require_str("dialog_id") {+ Ok(id) => id,+ Err(error) => return error,+ };+ if supplied_id != dialog_id {+ return BrowserRefusal::new(+ BrowserRefusalCode::BrowserActionUnavailable,+ "the dialog capability is stale or belongs to another dialog",+ )+ .to_tool_result();+ }+ let prompt_text = args.opt_str("prompt_text");+ if prompt_text.is_some() && (action != "accept" || dialog.kind != "prompt") {+ return ToolResult::error("prompt_text is valid only when accepting a prompt dialog");+ }+ let mut params = json!({ "accept": action == "accept" });+ if let Some(text) = prompt_text {+ params["promptText"] = Value::String(text);+ }+ match conn+ .call(Some(cdp_session), "Page.handleJavaScriptDialog", params)+ .await+ {+ Ok(_) => {+ conn.clear_dialog_state(cdp_session, dialog.generation);+ ToolResult::text(format!("{action}ed {} dialog in {tab_id}", dialog.kind))+ .with_structured(json!({+ "status": "ok", "target_id": target_id, "tab_id": tab_id,+ "dialog_id": dialog_id, "kind": dialog.kind, "action": action+ }))+ }+ Err(error) => BrowserRefusal::new(+ BrowserRefusalCode::BrowserActionUnavailable,+ format!("the exact JavaScript dialog could not be resolved: {error}"),+ )+ .to_tool_result(),+ }+ }+}++// ── browser_set_input_files ─────────────────────────────────────────────────++pub struct BrowserSetInputFilesTool {+ def: ToolDef,+ engine: Arc<BrowserEngine>,+}++impl BrowserSetInputFilesTool {+ pub fn new(engine: Arc<BrowserEngine>) -> Self {+ Self {+ def: ToolDef {+ name: "browser_set_input_files".into(),+ description: "Assign one or more explicit absolute local files to an exact live <input type=file> ref through CDP. This bypasses native file pickers, rejects symlinks and non-regular files, and never returns local paths.".into(),+ input_schema: json!({+ "type": "object",+ "properties": {+ "target_id": schema_target_id(),+ "tab_id": schema_tab_id(),+ "session": schema_session(),+ "ref": schema_ref(),+ "files": {+ "type": "array", "minItems": 1, "maxItems": 32,+ "items": { "type": "string", "description": "Absolute path to one local regular file." }+ }+ },+ "required": ["target_id", "tab_id", "ref", "files"],+ "additionalProperties": true+ }),+ read_only: false,+ destructive: false,+ idempotent: false,+ open_world: true,+ },+ engine,+ }+ }+}++fn validated_upload_paths(args: &Value) -> Result<Vec<String>, ToolResult> {+ let Some(files) = args.get("files").and_then(Value::as_array) else {+ return Err(ToolResult::error(+ "files must be a non-empty array of absolute paths",+ ));+ };+ if files.is_empty() || files.len() > 32 {+ return Err(ToolResult::error(+ "files must contain between 1 and 32 paths",+ ));+ }+ files+ .iter()+ .map(|value| {+ let raw = value+ .as_str()+ .ok_or_else(|| ToolResult::error("every files entry must be a string"))?;+ let path = std::path::Path::new(raw);+ if !path.is_absolute() {+ return Err(ToolResult::error("every upload path must be absolute"));+ }+ let metadata = std::fs::symlink_metadata(path)+ .map_err(|_| ToolResult::error("an approved upload path does not exist"))?;+ if metadata.file_type().is_symlink() || !metadata.is_file() {+ return Err(ToolResult::error(+ "upload paths must name regular files directly, not links or directories",+ ));+ }+ std::fs::canonicalize(path)+ .map(|path| path.to_string_lossy().into_owned())+ .map_err(|_| ToolResult::error("an upload path could not be canonicalized"))+ })+ .collect()+}++#[async_trait]+impl Tool for BrowserSetInputFilesTool {+ fn def(&self) -> &ToolDef {+ &self.def+ }++ async fn invoke(&self, args: Value) -> ToolResult {+ let (target_id, tab_id, ext_ref) = match (+ args.require_str("target_id"),+ args.require_str("tab_id"),+ args.require_str("ref"),+ ) {+ (Ok(target), Ok(tab), Ok(ext_ref)) => (target, tab, ext_ref),+ (Err(error), _, _) | (_, Err(error), _) | (_, _, Err(error)) => return error,+ };+ let files = match validated_upload_paths(&args) {+ Ok(files) => files,+ Err(error) => return error,+ };+ let session = match require_explicit_session(&args) {+ Ok(session) => session,+ Err(error) => return error,+ };+ let _mutation = match self+ .engine+ .lock_mutation(&session, &target_id, &tab_id)+ .await+ {+ Ok(guard) => guard,+ Err(refusal) => return refusal.to_tool_result(),+ };+ let validated = match self+ .engine+ .revalidate_for_mutation(&session, &target_id, Some(&tab_id))+ .await+ {+ Ok(validated) => validated,+ Err(refusal) => return refusal.to_tool_result(),+ };+ let entry = match self+ .engine+ .store+ .resolve_ref(&session, &target_id, &tab_id, &ext_ref)+ {+ Ok(entry) => entry,+ Err(refusal) => return refusal.to_tool_result(),+ };+ if entry.semantic && !entry.actions.contains(&BrowserActionKind::Upload) {+ return BrowserRefusal::new(+ BrowserRefusalCode::BrowserActionUnavailable,+ "the semantic ref is not a file-upload control",+ )+ .to_tool_result();+ }+ let cdp_session = match self+ .engine+ .frame_session_for_mutation(&session, &target_id, &tab_id, &validated, &entry.frame)+ .await+ {+ Ok(session) => session,+ Err(refusal) => return refusal.to_tool_result(),+ };+ let described = match validated+ .conn+ .call(+ Some(&cdp_session),+ "DOM.describeNode",+ json!({ "backendNodeId": entry.backend_node_id }),+ )+ .await+ {+ Ok(node) => node,+ Err(_) => {+ return BrowserRefusal::new(+ BrowserRefusalCode::BrowserRefStale,+ "the upload ref no longer resolves in the live page",+ )+ .to_tool_result()+ }+ };+ let node = &described["node"];+ let attributes = node["attributes"].as_array().cloned().unwrap_or_default();+ let is_file_input = node["nodeName"]+ .as_str()+ .is_some_and(|name| name.eq_ignore_ascii_case("input"))+ && attributes.chunks_exact(2).any(|pair| {+ pair[0]+ .as_str()+ .is_some_and(|name| name.eq_ignore_ascii_case("type"))+ && pair[1]+ .as_str()+ .is_some_and(|value| value.eq_ignore_ascii_case("file"))+ });+ if !is_file_input {+ return BrowserRefusal::new(+ BrowserRefusalCode::BrowserActionUnavailable,+ "the live ref is not an input[type=file] element",+ )+ .to_tool_result();+ }+ match validated+ .conn+ .call(+ Some(&cdp_session),+ "DOM.setFileInputFiles",+ json!({ "backendNodeId": entry.backend_node_id, "files": files }),+ )+ .await+ {+ Ok(_) => ToolResult::text(format!("assigned {} file(s) in {tab_id}", files.len()))+ .with_structured(json!({+ "status": "ok", "target_id": target_id, "tab_id": tab_id,+ "ref": ext_ref, "frame": entry.frame.kind.as_str(), "file_count": files.len()+ })),+ Err(error) => BrowserRefusal::new(+ BrowserRefusalCode::BrowserActionUnavailable,+ format!("the browser refused the exact file input assignment: {error}"),+ )+ .to_tool_result(),+ }+ }+}+#[cfg(test)]mod tests {use super::*;@@ -1558,6 +1906,14 @@ mod tests {BrowserTypeTool::new(e.clone()).def().clone(),"browser_type",),+ (+ BrowserDialogTool::new(e.clone()).def().clone(),+ "browser_dialog",+ ),+ (+ BrowserSetInputFilesTool::new(e.clone()).def().clone(),+ "browser_set_input_files",+ ),] {assert_eq!(def.name, name);assert!(!def.read_only, "{name} is a mutation");@@ -1569,7 +1925,7 @@ mod tests {}#[test]- fn registration_registers_all_five() {+ fn registration_registers_all_browser_tools() {let e = engine();let mut registry = ToolRegistry::new();register_browser_tools(&e, &mut registry);@@ -1581,11 +1937,25 @@ mod tests {"browser_prepare","browser_navigate","browser_click",- "browser_type"+ "browser_type",+ "browser_dialog",+ "browser_set_input_files"]);}+ #[test]+ fn upload_paths_are_absolute_regular_files_and_outputs_need_no_path() {+ let path = std::env::temp_dir().join(format!("cua-upload-{}.txt", uuid::Uuid::new_v4()));+ std::fs::write(&path, b"fixture").unwrap();+ let args = json!({ "files": [path.to_string_lossy()] });+ let validated = validated_upload_paths(&args).unwrap();+ assert_eq!(validated.len(), 1);+ assert!(std::path::Path::new(&validated[0]).is_absolute());+ assert!(validated_upload_paths(&json!({ "files": ["relative.txt"] })).is_err());+ std::fs::remove_file(path).unwrap();+ }+#[tokio::test]async fn bind_requires_an_explicit_session() {let e = engine();diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/tool.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/tool.rsindex aa593ca324..581e2b8aaf 100644--- a/libs/cua-driver/rust/crates/cua-driver-core/src/tool.rs+++ b/libs/cua-driver/rust/crates/cua-driver-core/src/tool.rs@@ -265,6 +265,8 @@ pub fn default_capabilities_for(tool_name: &str) -> Vec<String> {"browser_navigate" => &["browser.navigate"],"browser_click" => &["browser.input.click"],"browser_type" => &["browser.input.type"],+ "browser_dialog" => &["browser.dialog"],+ "browser_set_input_files" => &["browser.input.files"],// ── driver self-service ──────────────────────────────────────"check_for_update" => &["driver.update_check"],@@ -667,6 +669,8 @@ mod capability_tests {"browser_navigate","browser_click","browser_type",+ "browser_dialog",+ "browser_set_input_files",];/// All capability tokens in the canonical vocabulary. Any token