Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
fix(cua-driver): support Windows ARM64 browser sessions
Francesco Bonacci9c5a78b65b09
diff --git a/libs/cua-driver/rust/.cargo/config.toml b/libs/cua-driver/rust/.cargo/config.tomlindex 1cd4eddc90..da405486c4 100644--- a/libs/cua-driver/rust/.cargo/config.toml+++ b/libs/cua-driver/rust/.cargo/config.toml@@ -1,5 +1,10 @@[env]MACOSX_DEPLOYMENT_TARGET = "13.0"+# Release archives must run on clean Windows hosts without the Visual C+++# Redistributable. Keep both MSVC targets statically linked to the CRT.[target.x86_64-pc-windows-msvc]rustflags = ["-C", "target-feature=+crt-static"]++[target.aarch64-pc-windows-msvc]+rustflags = ["-C", "target-feature=+crt-static"]diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/browser/prepare.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/browser/prepare.rsindex 0d8f36cb2e..02fe339b4f 100644--- a/libs/cua-driver/rust/crates/cua-driver-core/src/browser/prepare.rs+++ b/libs/cua-driver/rust/crates/cua-driver-core/src/browser/prepare.rs@@ -5,7 +5,7 @@ use std::future::Future;use std::io::Write;use std::path::{Path, PathBuf};use std::pin::Pin;-use std::process::{Child, Command, Stdio};+use std::process::{Child, Command, ExitStatus, Stdio};use std::sync::{Arc, Mutex};use std::time::{Duration, Instant};@@ -181,6 +181,8 @@ struct PreparedProfile {pub(crate) struct ManagedBrowser {child: Child,+ #[cfg(target_os = "windows")]+ owned_pid: i64,profile: PathBuf,delete_profile: bool,marker: ProfileMarker,@@ -196,6 +198,19 @@ impl Drop for ManagedBrowser {// root Child can leave profile writers alive after cleanup.libc::kill(-(self.child.id() as i32), libc::SIGKILL);}+ #[cfg(target_os = "windows")]+ if self.owned_pid != i64::from(self.child.id()) {+ // Edge on Windows ARM may use a short-lived launcher process and+ // transfer the browser role to a descendant. The listener owner+ // was attested inside that driver-spawned process tree, so reap+ // that exact process tree when its owning session ends.+ let _ = Command::new("taskkill.exe")+ .args(["/PID", &self.owned_pid.to_string(), "/T", "/F"])+ .stdin(Stdio::null())+ .stdout(Stdio::null())+ .stderr(Stdio::null())+ .status();+ }let _ = self.child.kill();let _ = self.child.wait();if self.delete_profile && profile_matches_marker(&self.profile, &self.marker) {@@ -324,6 +339,18 @@ fn isolated_browser_command(executable: &str, profile: &Path) -> Command {command}+fn clean_spawn_exit_can_be_launcher_handoff(status: &ExitStatus) -> bool {+ #[cfg(target_os = "windows")]+ {+ status.success()+ }+ #[cfg(not(target_os = "windows"))]+ {+ let _ = status;+ false+ }+}+fn write_profile_marker(path: &Path, marker: &ProfileMarker) -> Result<(), BrowserRefusal> {let marker_path = path.join(PROFILE_MARKER);let mut options = OpenOptions::new();@@ -454,6 +481,7 @@ async fn wait_for_spawned_endpoint() -> Result<OwnedEndpoint, BrowserRefusal> {let deadline = Instant::now() + Duration::from_secs(20);let port_file = profile.join("DevToolsActivePort");+ let mut observed_clean_launcher_exit = false;loop {if let Some(status) = child.try_wait().map_err(|error| {refusal(@@ -461,10 +489,18 @@ async fn wait_for_spawned_endpoint(format!("could not inspect the isolated browser process: {error}"),)})? {- return Err(refusal(- BrowserRefusalCode::BrowserRouteUnavailable,- format!("isolated browser exited before exposing DevTools ({status})"),- ));+ if clean_spawn_exit_can_be_launcher_handoff(&status) {+ // Edge on Windows ARM can transfer the browser role to a+ // descendant and let its launcher exit successfully. Keep+ // waiting for the driver-owned profile's port file; the live+ // listener and its descendant ownership are attested below.+ observed_clean_launcher_exit = true;+ } else {+ return Err(refusal(+ BrowserRefusalCode::BrowserRouteUnavailable,+ format!("isolated browser exited before exposing DevTools ({status})"),+ ));+ }}if let Ok(text) = fs::read_to_string(&port_file) {let mut lines = text.lines();@@ -494,7 +530,11 @@ async fn wait_for_spawned_endpoint(if Instant::now() >= deadline {return Err(refusal(BrowserRefusalCode::BrowserRouteUnavailable,- "isolated browser did not expose a loopback DevTools endpoint before timeout",+ if observed_clean_launcher_exit {+ "isolated browser launcher exited cleanly, but its process tree did not expose a loopback DevTools endpoint before timeout"+ } else {+ "isolated browser did not expose a loopback DevTools endpoint before timeout"+ },));}tokio::time::sleep(Duration::from_millis(100)).await;@@ -517,11 +557,14 @@ async fn attest_spawned_endpoint(http_port: live.http_port,ownership: EndpointOwnershipProof {method: EndpointOwnershipMethod::SpawnedByDriver,- owner_pid: child_pid,- detail: Some(+ owner_pid: live.ownership.owner_pid,+ detail: Some(if live.ownership.owner_pid == child_pid {"driver-owned profile port file plus live loopback socket owner"- .to_owned(),- ),+ .to_owned()+ } else {+ "driver-owned profile port file plus live loopback socket owner promoted from a short-lived launcher process"+ .to_owned()+ }),},});}@@ -651,6 +694,8 @@ impl BrowserEngine {}self.managed_browsers.lock().unwrap().push(ManagedBrowser {child,+ #[cfg(target_os = "windows")]+ owned_pid: prepared_pid,profile: prepared_profile.path,delete_profile: prepared_profile.delete_on_cleanup,marker: prepared_profile.marker,@@ -1209,6 +1254,24 @@ mod tests {assert!(args.iter().any(|arg| arg == "--password-store=basic"));}+ #[test]+ fn clean_launcher_exit_is_only_deferred_on_windows() {+ let mut command = if cfg!(target_os = "windows") {+ let mut command = Command::new("cmd.exe");+ command.args(["/C", "exit", "0"]);+ command+ } else {+ let mut command = Command::new("sh");+ command.args(["-c", "exit 0"]);+ command+ };+ let status = command.status().unwrap();+ assert_eq!(+ clean_spawn_exit_can_be_launcher_handoff(&status),+ cfg!(target_os = "windows")+ );+ }+#[cfg(target_os = "linux")]#[test]fn isolated_launch_can_select_native_wayland_and_test_vm_sandbox_mode() {diff --git a/libs/cua-driver/rust/crates/platform-windows/src/browser_platform.rs b/libs/cua-driver/rust/crates/platform-windows/src/browser_platform.rsindex 2b2a1062b3..baf10828f4 100644--- a/libs/cua-driver/rust/crates/platform-windows/src/browser_platform.rs+++ b/libs/cua-driver/rust/crates/platform-windows/src/browser_platform.rs@@ -147,32 +147,57 @@ fn cdp_comparable_window_bounds(window_id: u64) -> Result<Rect, BrowserRefusal>))}-fn parse_netstat_loopback_ports(text: &str, pid: u32) -> Vec<u16> {- let mut ports = text+fn parse_netstat_loopback_listeners(text: &str, allowed_pids: &[u32]) -> Vec<(u16, u32)> {+ let mut listeners = text.lines().filter_map(|line| {let fields = line.split_whitespace().collect::<Vec<_>>();if fields.len() < 5|| !fields[0].eq_ignore_ascii_case("TCP")|| !fields[3].eq_ignore_ascii_case("LISTENING")- || fields[4].parse::<u32>().ok() != Some(pid){return None;}+ let owner_pid = fields[4].parse::<u32>().ok()?;+ if !allowed_pids.contains(&owner_pid) {+ return None;+ }let local = fields[1];let (host, port) = local.rsplit_once(':')?;let host = host.trim_matches(['[', ']']);matches!(host, "127.0.0.1" | "::1" | "localhost")- .then(|| port.parse::<u16>().ok())+ .then(|| port.parse::<u16>().ok().map(|port| (port, owner_pid))).flatten()}).collect::<Vec<_>>();+ listeners.sort_unstable();+ listeners.dedup();+ listeners+}++#[cfg(test)]+fn parse_netstat_loopback_ports(text: &str, allowed_pids: &[u32]) -> Vec<u16> {+ let mut ports = parse_netstat_loopback_listeners(text, allowed_pids)+ .into_iter()+ .map(|(port, _owner_pid)| port)+ .collect::<Vec<_>>();ports.sort_unstable();ports.dedup();ports}-async fn loopback_ports_for_pid(pid: u32) -> Result<Vec<u16>, BrowserRefusal> {+async fn loopback_listeners_for_process_tree(+ root_pid: u32,+) -> Result<Vec<(u16, u32)>, BrowserRefusal> {+ let allowed_pids =+ tokio::task::spawn_blocking(move || crate::win32::list_descendants(root_pid))+ .await+ .map_err(|error| {+ refusal(+ BrowserRefusalCode::BrowserRouteUnavailable,+ format!("could not inspect browser process tree: {error}"),+ )+ })?;let output = tokio::process::Command::new("netstat.exe").args(["-ano", "-p", "tcp"]).stdin(Stdio::null())@@ -185,12 +210,23 @@ async fn loopback_ports_for_pid(pid: u32) -> Result<Vec<u16>, BrowserRefusal> {format!("could not inspect browser listeners: {error}"),)})?;- Ok(parse_netstat_loopback_ports(+ Ok(parse_netstat_loopback_listeners(&String::from_utf8_lossy(&output.stdout),- pid,+ &allowed_pids,))}+async fn loopback_ports_for_process_tree(root_pid: u32) -> Result<Vec<u16>, BrowserRefusal> {+ let mut ports = loopback_listeners_for_process_tree(root_pid)+ .await?+ .into_iter()+ .map(|(port, _owner_pid)| port)+ .collect::<Vec<_>>();+ ports.sort_unstable();+ ports.dedup();+ Ok(ports)+}+async fn browser_websocket_url(port: u16) -> Option<String> {tokio::time::timeout(Duration::from_secs(2), async move {let mut stream = tokio::net::TcpStream::connect(("127.0.0.1", port))@@ -238,24 +274,24 @@ async fn browser_websocket_url(port: u16) -> Option<String> {const ENDPOINT_DISCOVERY_ATTEMPTS: usize = 4;const ENDPOINT_DISCOVERY_RETRY_DELAY: Duration = Duration::from_millis(100);-async fn browser_endpoints_once(pid: u32) -> Result<Vec<(u16, String)>, BrowserRefusal> {+async fn browser_endpoints_once(pid: u32) -> Result<Vec<(u16, String, u32)>, BrowserRefusal> {let mut endpoints = Vec::new();- for port in loopback_ports_for_pid(pid).await? {+ for (port, owner_pid) in loopback_listeners_for_process_tree(pid).await? {if let Some(ws_url) = browser_websocket_url(port).await {- endpoints.push((port, ws_url));+ endpoints.push((port, ws_url, owner_pid));}}Ok(endpoints)}-async fn retry_empty_endpoint_discovery<F, Fut>(+async fn retry_empty_endpoint_discovery<T, F, Fut>(attempts: usize,delay: Duration,mut probe: F,-) -> Result<Vec<(u16, String)>, BrowserRefusal>+) -> Result<Vec<T>, BrowserRefusal>whereF: FnMut() -> Fut,- Fut: Future<Output = Result<Vec<(u16, String)>, BrowserRefusal>>,+ Fut: Future<Output = Result<Vec<T>, BrowserRefusal>>,{let attempts = attempts.max(1);for attempt in 0..attempts {@@ -268,7 +304,7 @@ whereunreachable!("the bounded endpoint-discovery loop always returns")}-async fn browser_endpoints_for_pid(pid: u32) -> Result<Vec<(u16, String)>, BrowserRefusal> {+async fn browser_endpoints_for_pid(pid: u32) -> Result<Vec<(u16, String, u32)>, BrowserRefusal> {retry_empty_endpoint_discovery(ENDPOINT_DISCOVERY_ATTEMPTS,ENDPOINT_DISCOVERY_RETRY_DELAY,@@ -285,7 +321,7 @@ async fn loopback_port_is_owned_with_retry(ENDPOINT_DISCOVERY_ATTEMPTS,ENDPOINT_DISCOVERY_RETRY_DELAY,expected_port,- || loopback_ports_for_pid(pid),+ || loopback_ports_for_process_tree(pid),).await}@@ -440,13 +476,15 @@ impl BrowserPlatform for WindowsBrowserPlatform {.await?.into_iter().next()- .map(|(port, ws_url)| OwnedEndpoint {+ .map(|(port, ws_url, owner_pid)| OwnedEndpoint {ws_url,http_port: Some(port),ownership: EndpointOwnershipProof {method: EndpointOwnershipMethod::ListeningSocketPid,- owner_pid: pid,- detail: Some("netstat listener owner pid".to_owned()),+ owner_pid: i64::from(owner_pid),+ detail: Some(+ "netstat listener owned by the approved browser process tree; the exact listener owner is the stable browser pid".to_owned(),+ ),},}))}@@ -464,18 +502,20 @@ impl BrowserPlatform for WindowsBrowserPlatform {let discovered = browser_endpoints_for_pid(pid_u32).await?;match discovered.as_slice() {[] => Ok(None),- [(port, ws_url)] => Ok(Some(OwnedEndpoint {+ [(port, ws_url, owner_pid)] => Ok(Some(OwnedEndpoint {ws_url: ws_url.clone(),http_port: Some(*port),ownership: EndpointOwnershipProof {method: EndpointOwnershipMethod::ListeningSocketPid,- owner_pid: pid,- detail: Some("Windows listener owner plus /json/version".to_owned()),+ owner_pid: i64::from(*owner_pid),+ detail: Some(+ "Windows browser process-tree listener plus /json/version".to_owned(),+ ),},})),_ => Err(refusal(BrowserRefusalCode::BrowserBindingAmbiguous,- "multiple browser-level DevTools endpoints are owned by the approved process",+ "multiple browser-level DevTools endpoints are owned by the approved browser process tree",)),}}@@ -509,7 +549,9 @@ impl BrowserPlatform for WindowsBrowserPlatform {ownership: EndpointOwnershipProof {method: EndpointOwnershipMethod::ListeningSocketPid,owner_pid: pid,- detail: Some("Windows owner of exact approved endpoint".to_owned()),+ detail: Some(+ "Windows browser process-tree owner of exact approved endpoint".to_owned(),+ ),},}))}@@ -534,7 +576,7 @@ impl BrowserPlatform for WindowsBrowserPlatform {)})?;let hwnd = request.window_id;- let listeners_before = loopback_ports_for_pid(pid_u32).await?;+ let listeners_before = loopback_ports_for_process_tree(pid_u32).await?;let handle =tokio::task::spawn_blocking(move || crate::browser_setup_ui::enable(hwnd, descriptor)).await@@ -555,14 +597,18 @@ impl BrowserPlatform for WindowsBrowserPlatform {let deadline = std::time::Instant::now() + Duration::from_secs(6);let endpoint_result = loop {- let ports = match loopback_ports_for_pid(pid_u32).await {+ let ports = match loopback_ports_for_process_tree(pid_u32).await {Ok(ports) => ports,Err(error) => break Err(error),};let mut endpoints = Vec::new();for port in &ports {if let Some(ws_url) = browser_websocket_url(*port).await {- endpoints.push((*port, ws_url, "Windows owner plus /json/version"));+ endpoints.push((+ *port,+ ws_url,+ "Windows browser process-tree owner plus /json/version",+ ));}}if endpoints.is_empty() {@@ -575,13 +621,13 @@ impl BrowserPlatform for WindowsBrowserPlatform {endpoints.push((*port,format!("ws://127.0.0.1:{port}/devtools/browser"),- "new PID-owned listener correlated with exact approved setup",+ "new browser process-tree listener correlated with exact approved setup",));} else if correlated.len() > 1 {break Err(refusal(BrowserRefusalCode::BrowserBindingAmbiguous,format!(- "{} exposed multiple newly correlated PID-owned listeners",+ "{} exposed multiple newly correlated browser process-tree listeners",descriptor.product_name),));@@ -606,7 +652,7 @@ impl BrowserPlatform for WindowsBrowserPlatform {break Err(refusal(BrowserRefusalCode::BrowserRequiresSetup,format!(- "{} did not expose a uniquely PID-owned loopback endpoint after the exact setup action",+ "{} did not expose a uniquely process-tree-owned loopback endpoint after the exact setup action",descriptor.product_name),))@@ -615,7 +661,7 @@ impl BrowserPlatform for WindowsBrowserPlatform {break Err(refusal(BrowserRefusalCode::BrowserBindingAmbiguous,format!(- "{} exposed multiple PID-owned endpoint candidates after the exact setup action",+ "{} exposed multiple process-tree-owned endpoint candidates after the exact setup action",descriptor.product_name),))@@ -740,13 +786,35 @@ mod tests {use std::sync::Arc;#[test]- fn netstat_parser_requires_loopback_listening_and_exact_pid() {+ fn netstat_parser_requires_loopback_listening_and_browser_process_tree() {let input = "\TCP 127.0.0.1:9222 0.0.0.0:0 LISTENING 42\n\- TCP 0.0.0.0:9333 0.0.0.0:0 LISTENING 42\n\- TCP [::1]:9444 [::]:0 LISTENING 42\n\+ TCP 0.0.0.0:9333 0.0.0.0:0 LISTENING 43\n\+ TCP [::1]:9444 [::]:0 LISTENING 43\n\TCP 127.0.0.1:9555 0.0.0.0:0 LISTENING 7\n";- assert_eq!(parse_netstat_loopback_ports(input, 42), vec![9222, 9444]);+ assert_eq!(+ parse_netstat_loopback_ports(input, &[42, 43]),+ vec![9222, 9444]+ );+ }++ #[test]+ fn netstat_parser_rejects_unrelated_process_trees() {+ let input = "\+ TCP 127.0.0.1:9222 0.0.0.0:0 LISTENING 42\n\+ TCP 127.0.0.1:9555 0.0.0.0:0 LISTENING 99\n";+ assert_eq!(parse_netstat_loopback_ports(input, &[42, 43]), vec![9222]);+ }++ #[test]+ fn netstat_parser_preserves_the_exact_listener_owner() {+ let input = "\+ TCP 127.0.0.1:9222 0.0.0.0:0 LISTENING 43\n\+ TCP 127.0.0.1:9555 0.0.0.0:0 LISTENING 99\n";+ assert_eq!(+ parse_netstat_loopback_listeners(input, &[42, 43]),+ vec![(9222, 43)]+ );}#[test]@@ -833,6 +901,7 @@ mod tests {Ok(vec![(9222,"ws://127.0.0.1:9222/devtools/browser/proven".to_owned(),+ 43,)])}}@@ -846,6 +915,7 @@ mod tests {vec![(9222,"ws://127.0.0.1:9222/devtools/browser/proven".to_owned(),+ 43,)]);}diff --git a/libs/cua-driver/rust/crates/platform-windows/src/win32/apps.rs b/libs/cua-driver/rust/crates/platform-windows/src/win32/apps.rsindex c49f66325e..3dc95dfc18 100644--- a/libs/cua-driver/rust/crates/platform-windows/src/win32/apps.rs+++ b/libs/cua-driver/rust/crates/platform-windows/src/win32/apps.rs@@ -64,10 +64,14 @@ fn decode_wstr(buf: &[u16]) -> String {/// no longer alive, callers handle the empty-windows case the same way).pub fn list_descendants(root_pid: u32) -> Vec<u32> {let all = list_processes();+ descendants_from_processes(root_pid, &all)+}++fn descendants_from_processes(root_pid: u32, all: &[ProcessInfo]) -> Vec<u32> {let mut result = vec![root_pid];let mut frontier = vec![root_pid];while let Some(parent) = frontier.pop() {- for p in &all {+ for p in all {if p.parent_pid == parent && !result.contains(&p.pid) {result.push(p.pid);frontier.push(p.pid);@@ -136,3 +140,34 @@ fn strip_version_suffix(basename: &str) -> String {}s[..cut].to_string()}++#[cfg(test)]+mod tests {+ use super::*;++ fn process(pid: u32, parent_pid: u32) -> ProcessInfo {+ ProcessInfo {+ pid,+ parent_pid,+ name: format!("process-{pid}.exe"),+ }+ }++ #[test]+ fn descendants_include_root_and_only_its_transitive_process_tree() {+ let processes = vec![+ process(42, 1),+ process(43, 42),+ process(44, 43),+ process(45, 42),+ process(99, 1),+ process(100, 99),+ ];++ let descendants = descendants_from_processes(42, &processes);++ assert_eq!(descendants, vec![42, 43, 45, 44]);+ assert!(!descendants.contains(&99));+ assert!(!descendants.contains(&100));+ }+}