Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
fix(cua-driver): attribute hosted Windows apps by window (#3227)
Francesco Bonacci30d9130f7c74
diff --git a/libs/cua-driver/rust/crates/cua-driver-core/examples/history_hook_bench.rs b/libs/cua-driver/rust/crates/cua-driver-core/examples/history_hook_bench.rsindex cd71ead74e..6efd25c17c 100644--- a/libs/cua-driver/rust/crates/cua-driver-core/examples/history_hook_bench.rs+++ b/libs/cua-driver/rust/crates/cua-driver-core/examples/history_hook_bench.rs@@ -70,7 +70,7 @@ impl BlockingApps {}}impl ApplicationIdentityProvider for BlockingApps {- fn resolve(&self, _: i64) -> Option<ApplicationIdentity> {+ fn resolve(&self, _: i64, _: Option<u64>) -> Option<ApplicationIdentity> {let mut s = self.state.lock().unwrap();s.0 = true;self.changed.notify_all();diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/history.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/history.rsindex 2b0483e794..a4e686afb8 100644--- a/libs/cua-driver/rust/crates/cua-driver-core/src/history.rs+++ b/libs/cua-driver/rust/crates/cua-driver-core/src/history.rs@@ -362,7 +362,7 @@ pub struct ApplicationIdentity {}pub trait ApplicationIdentityProvider: Send + Sync {- fn resolve(&self, pid: i64) -> Option<ApplicationIdentity>;+ fn resolve(&self, pid: i64, window_id: Option<u64>) -> Option<ApplicationIdentity>;}#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]@@ -516,12 +516,14 @@ pub struct PendingHistoryAction {session_id: Option<String>,capability: String,application_pid: Option<i64>,+ application_window_id: Option<u64>,}enum WriterMessage {Event {event: HistoryEvent,application_pid: Option<i64>,+ application_window_id: Option<u64>,},Flush(mpsc::Sender<Result<(), HistoryError>>),ReadSnapshot {@@ -889,11 +891,13 @@ impl HistoryManager {.get("pid").and_then(Value::as_i64).filter(|pid| *pid > 0);+ let application_window_id = args.get("window_id").and_then(Value::as_u64);let pending = PendingHistoryAction {action_id,session_id,capability,application_pid,+ application_window_id,};self.try_send_with_application_pid(self.event(@@ -905,6 +909,7 @@ impl HistoryManager {HistoryPayload::ActionStarted,),pending.application_pid,+ pending.application_window_id,);Some(pending)}@@ -961,6 +966,7 @@ impl HistoryManager {payload,),pending.application_pid,+ pending.application_window_id,);}@@ -1096,6 +1102,7 @@ impl HistoryManager {tx.send(WriterMessage::Event {event,application_pid: None,+ application_window_id: None,}).map_err(|_| HistoryError::new(HistoryHealthCategory::WriterStopped))}@@ -1117,10 +1124,15 @@ impl HistoryManager {}fn try_send(&self, event: HistoryEvent) {- self.try_send_with_application_pid(event, None);+ self.try_send_with_application_pid(event, None, None);}- fn try_send_with_application_pid(&self, mut event: HistoryEvent, application_pid: Option<i64>) {+ fn try_send_with_application_pid(+ &self,+ mut event: HistoryEvent,+ application_pid: Option<i64>,+ application_window_id: Option<u64>,+ ) {if validate_event(&event).is_err() {self.record_drop();return;@@ -1153,6 +1165,7 @@ impl HistoryManager {.try_send(WriterMessage::Event {event: health,application_pid: None,+ application_window_id: None,}).is_err(){@@ -1165,6 +1178,7 @@ impl HistoryManager {tx.try_send(WriterMessage::Event {event,application_pid,+ application_window_id,}),Err(TrySendError::Full(_)) | Err(TrySendError::Disconnected(_))) {@@ -1414,6 +1428,7 @@ fn writer_loop(WriterMessage::Event {mut event,application_pid,+ application_window_id,} => {if terminal_error.is_none() {let maintenance_due = retention_days == 0@@ -1434,7 +1449,7 @@ fn writer_loop(if let Some(pid) = application_pid {event.data.application = app_provider.as_ref()- .and_then(|provider| provider.resolve(pid))+ .and_then(|provider| provider.resolve(pid, application_window_id)).and_then(sanitize_application_identity);}if let Err(error) = store.append(&event) {@@ -2385,6 +2400,21 @@ mod tests {use super::*;use std::collections::HashMap;+ #[derive(Default)]+ struct RecordingApplicationProvider {+ targets: Mutex<Vec<(i64, Option<u64>)>>,+ }++ impl ApplicationIdentityProvider for RecordingApplicationProvider {+ fn resolve(&self, pid: i64, window_id: Option<u64>) -> Option<ApplicationIdentity> {+ self.targets.lock().unwrap().push((pid, window_id));+ Some(ApplicationIdentity {+ bundle_id: Some("test.app".to_owned()),+ display_name: Some("Test App".to_owned()),+ })+ }+ }+#[derive(Default)]struct MemoryKeyProvider {keys: Mutex<HashMap<String, Vec<u8>>>,@@ -2687,6 +2717,48 @@ mod tests {.all(|event| event.data.session_id.as_deref() == Some(opaque.as_str())));}+ #[test]+ fn action_attribution_receives_pid_and_window_identity() {+ let temp = tempfile::tempdir().unwrap();+ let provider = Arc::new(RecordingApplicationProvider::default());+ let manager = HistoryManager::new(+ config(temp.path()),+ Arc::new(MemoryKeyProvider::default()),+ Some(provider.clone()),+ );+ manager.enable().unwrap();+ let pending = manager+ .begin_action(+ "click",+ &serde_json::json!({"pid": 42, "window_id": 9001}),+ None,+ )+ .unwrap();+ manager.finish_action(pending, None, false);+ manager.flush().unwrap();++ assert_eq!(+ provider.targets.lock().unwrap().as_slice(),+ &[(42, Some(9001)), (42, Some(9001))]+ );+ let events = manager+ .query(+ HistoryQuery {+ limit: Some(MAX_QUERY_LIMIT),+ ..Default::default()+ },+ HistoryAccessOperation::LocalCli,+ )+ .unwrap();+ assert_eq!(+ events+ .iter()+ .filter_map(|event| event.data.application.as_ref())+ .count(),+ 2+ );+ }+#[test]fn malformed_multibyte_hex_is_rejected_without_panicking() {let malformed = format!("€{}", "a".repeat(29));diff --git a/libs/cua-driver/rust/crates/platform-linux/src/history.rs b/libs/cua-driver/rust/crates/platform-linux/src/history.rsindex 993ca73bfe..5f3768b7da 100644--- a/libs/cua-driver/rust/crates/platform-linux/src/history.rs+++ b/libs/cua-driver/rust/crates/platform-linux/src/history.rs@@ -165,7 +165,7 @@ fn application_identity_from_stable_sources(}impl ApplicationIdentityProvider for LinuxApplicationIdentityProvider {- fn resolve(&self, pid: i64) -> Option<ApplicationIdentity> {+ fn resolve(&self, pid: i64, _window_id: Option<u64>) -> Option<ApplicationIdentity> {let pid = u32::try_from(pid).ok()?;let process = crate::proc_fs::list_processes().into_iter()@@ -217,7 +217,7 @@ mod tests {#[test]fn current_process_identity_is_resolvable_without_titles_or_paths() {let identity = LinuxApplicationIdentityProvider- .resolve(i64::from(std::process::id()))+ .resolve(i64::from(std::process::id()), None).expect("current process identity");assert!(identity.bundle_id.is_some() || identity.display_name.is_some());}diff --git a/libs/cua-driver/rust/crates/platform-macos/src/history.rs b/libs/cua-driver/rust/crates/platform-macos/src/history.rsindex acf630b0c0..545904eb6f 100644--- a/libs/cua-driver/rust/crates/platform-macos/src/history.rs+++ b/libs/cua-driver/rust/crates/platform-macos/src/history.rs@@ -178,7 +178,7 @@ fn map_keychain_error(error: security_framework::base::Error) -> HistoryError {pub struct MacosApplicationIdentityProvider;impl ApplicationIdentityProvider for MacosApplicationIdentityProvider {- fn resolve(&self, pid: i64) -> Option<ApplicationIdentity> {+ fn resolve(&self, pid: i64, _window_id: Option<u64>) -> Option<ApplicationIdentity> {let pid = i32::try_from(pid).ok()?;crate::apps::list_running_apps().into_iter()diff --git a/libs/cua-driver/rust/crates/platform-windows/src/history.rs b/libs/cua-driver/rust/crates/platform-windows/src/history.rsindex 72916ae9fa..3dee295fd0 100644--- a/libs/cua-driver/rust/crates/platform-windows/src/history.rs+++ b/libs/cua-driver/rust/crates/platform-windows/src/history.rs@@ -137,19 +137,28 @@ fn map_keyring_error(error: KeyringError) -> HistoryError {pub struct WindowsApplicationIdentityProvider;impl ApplicationIdentityProvider for WindowsApplicationIdentityProvider {- fn resolve(&self, pid: i64) -> Option<ApplicationIdentity> {- let pid = u32::try_from(pid).ok()?;- let name = crate::win32::list_processes()- .into_iter()- .find(|process| process.pid == pid)?- .name;- (!name.trim().is_empty()).then(|| ApplicationIdentity {+ fn resolve(&self, pid: i64, window_id: Option<u64>) -> Option<ApplicationIdentity> {+ let mut pid = u32::try_from(pid).ok()?;+ let owner_name = process_name(pid)?;+ if owner_name.eq_ignore_ascii_case("ApplicationFrameHost.exe") {+ pid = crate::win32::resolve_uwp_app_pid(pid, window_id?)?;+ }+ let name = process_name(pid)?;+ Some(ApplicationIdentity {bundle_id: Some(name.clone()),display_name: Some(name),})}}+fn process_name(pid: u32) -> Option<String> {+ let name = crate::win32::list_processes()+ .into_iter()+ .find(|process| process.pid == pid)?+ .name;+ (!name.trim().is_empty()).then_some(name)+}+pub fn application_identity_provider() -> Arc<dyn ApplicationIdentityProvider> {Arc::new(WindowsApplicationIdentityProvider)}diff --git a/libs/cua-driver/rust/crates/platform-windows/src/win32/mod.rs b/libs/cua-driver/rust/crates/platform-windows/src/win32/mod.rsindex 38f98f89d7..bbb431ae9f 100644--- a/libs/cua-driver/rust/crates/platform-windows/src/win32/mod.rs+++ b/libs/cua-driver/rust/crates/platform-windows/src/win32/mod.rs@@ -8,6 +8,7 @@ pub use apps::{list_descendants, list_processes, related_processes, ProcessInfo}pub use installed_apps::{list_installed_apps, InstalledApp};pub(crate) use windows::{capture_foreground_target, find_window_by_pid_and_handle,- foreground_matches_target_or_owned_window, list_windows_via_win32, window_owner_pid,+ foreground_matches_target_or_owned_window, list_windows_via_win32, resolve_uwp_app_pid,+ window_owner_pid,};pub use windows::{list_windows, resolve_uwp_host_window, WindowInfo};diff --git a/libs/cua-driver/rust/crates/platform-windows/src/win32/windows.rs b/libs/cua-driver/rust/crates/platform-windows/src/win32/windows.rsindex 840b11ba5e..e68fe10ccc 100644--- a/libs/cua-driver/rust/crates/platform-windows/src/win32/windows.rs+++ b/libs/cua-driver/rust/crates/platform-windows/src/win32/windows.rs@@ -604,3 +604,55 @@ pub fn resolve_uwp_host_window(app_pid: u32) -> Option<WindowInfo> {minimized: false,})}++/// Resolve the real packaged-app process hosted by a specific+/// `ApplicationFrameWindow`.+///+/// An ApplicationFrameHost pid is shared by unrelated apps, so callers must+/// provide the frame HWND as well as the host pid. The hosted CoreWindow child+/// retains ownership by the packaged app process and supplies the identity we+/// need for attribution.+pub fn resolve_uwp_app_pid(host_pid: u32, frame_hwnd: u64) -> Option<u32> {+ let frame = HWND(frame_hwnd as *mut _);+ if unsafe { IsWindow(frame) }.0 == 0 || window_class_name(frame) != "ApplicationFrameWindow" {+ return None;+ }++ let mut owner_pid = 0;+ unsafe { GetWindowThreadProcessId(frame, Some(&mut owner_pid)) };+ if owner_pid != host_pid {+ return None;+ }++ struct ChildScan {+ host_pid: u32,+ app_pid: Option<u32>,+ }++ unsafe extern "system" fn child_cb(child: HWND, lparam: LPARAM) -> BOOL {+ let scan = &mut *(lparam.0 as *mut ChildScan);+ let mut child_pid = 0;+ GetWindowThreadProcessId(child, Some(&mut child_pid));+ if child_pid != 0+ && child_pid != scan.host_pid+ && window_class_name(child) == "Windows.UI.Core.CoreWindow"+ {+ scan.app_pid = Some(child_pid);+ return windows::Win32::Foundation::FALSE;+ }+ TRUE+ }++ let mut scan = ChildScan {+ host_pid,+ app_pid: None,+ };+ unsafe {+ let _ = EnumChildWindows(+ frame,+ Some(child_cb),+ LPARAM(&mut scan as *mut ChildScan as isize),+ );+ }+ scan.app_pid+}