Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

fix(cua-driver): attribute hosted Windows apps by window (#3227)

Francesco Bonacci30d9130f7c74
diff --git a/libs/cua-driver/rust/crates/cua-driver-core/examples/history_hook_bench.rs b/libs/cua-driver/rust/crates/cua-driver-core/examples/history_hook_bench.rs
index cd71ead74e..6efd25c17c 100644
--- a/libs/cua-driver/rust/crates/cua-driver-core/examples/history_hook_bench.rs
+++ b/libs/cua-driver/rust/crates/cua-driver-core/examples/history_hook_bench.rs
@@ -70,7 +70,7 @@ impl BlockingApps {
}
}
impl ApplicationIdentityProvider for BlockingApps {
- fn resolve(&self, _: i64) -> Option<ApplicationIdentity> {
+ fn resolve(&self, _: i64, _: Option<u64>) -> Option<ApplicationIdentity> {
let mut s = self.state.lock().unwrap();
s.0 = true;
self.changed.notify_all();
diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/history.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/history.rs
index 2b0483e794..a4e686afb8 100644
--- a/libs/cua-driver/rust/crates/cua-driver-core/src/history.rs
+++ b/libs/cua-driver/rust/crates/cua-driver-core/src/history.rs
@@ -362,7 +362,7 @@ pub struct ApplicationIdentity {
}
pub trait ApplicationIdentityProvider: Send + Sync {
- fn resolve(&self, pid: i64) -> Option<ApplicationIdentity>;
+ fn resolve(&self, pid: i64, window_id: Option<u64>) -> Option<ApplicationIdentity>;
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
@@ -516,12 +516,14 @@ pub struct PendingHistoryAction {
session_id: Option<String>,
capability: String,
application_pid: Option<i64>,
+ application_window_id: Option<u64>,
}
enum WriterMessage {
Event {
event: HistoryEvent,
application_pid: Option<i64>,
+ application_window_id: Option<u64>,
},
Flush(mpsc::Sender<Result<(), HistoryError>>),
ReadSnapshot {
@@ -889,11 +891,13 @@ impl HistoryManager {
.get("pid")
.and_then(Value::as_i64)
.filter(|pid| *pid > 0);
+ let application_window_id = args.get("window_id").and_then(Value::as_u64);
let pending = PendingHistoryAction {
action_id,
session_id,
capability,
application_pid,
+ application_window_id,
};
self.try_send_with_application_pid(
self.event(
@@ -905,6 +909,7 @@ impl HistoryManager {
HistoryPayload::ActionStarted,
),
pending.application_pid,
+ pending.application_window_id,
);
Some(pending)
}
@@ -961,6 +966,7 @@ impl HistoryManager {
payload,
),
pending.application_pid,
+ pending.application_window_id,
);
}
@@ -1096,6 +1102,7 @@ impl HistoryManager {
tx.send(WriterMessage::Event {
event,
application_pid: None,
+ application_window_id: None,
})
.map_err(|_| HistoryError::new(HistoryHealthCategory::WriterStopped))
}
@@ -1117,10 +1124,15 @@ impl HistoryManager {
}
fn try_send(&self, event: HistoryEvent) {
- self.try_send_with_application_pid(event, None);
+ self.try_send_with_application_pid(event, None, None);
}
- fn try_send_with_application_pid(&self, mut event: HistoryEvent, application_pid: Option<i64>) {
+ fn try_send_with_application_pid(
+ &self,
+ mut event: HistoryEvent,
+ application_pid: Option<i64>,
+ application_window_id: Option<u64>,
+ ) {
if validate_event(&event).is_err() {
self.record_drop();
return;
@@ -1153,6 +1165,7 @@ impl HistoryManager {
.try_send(WriterMessage::Event {
event: health,
application_pid: None,
+ application_window_id: None,
})
.is_err()
{
@@ -1165,6 +1178,7 @@ impl HistoryManager {
tx.try_send(WriterMessage::Event {
event,
application_pid,
+ application_window_id,
}),
Err(TrySendError::Full(_)) | Err(TrySendError::Disconnected(_))
) {
@@ -1414,6 +1428,7 @@ fn writer_loop(
WriterMessage::Event {
mut event,
application_pid,
+ application_window_id,
} => {
if terminal_error.is_none() {
let maintenance_due = retention_days == 0
@@ -1434,7 +1449,7 @@ fn writer_loop(
if let Some(pid) = application_pid {
event.data.application = app_provider
.as_ref()
- .and_then(|provider| provider.resolve(pid))
+ .and_then(|provider| provider.resolve(pid, application_window_id))
.and_then(sanitize_application_identity);
}
if let Err(error) = store.append(&event) {
@@ -2385,6 +2400,21 @@ mod tests {
use super::*;
use std::collections::HashMap;
+ #[derive(Default)]
+ struct RecordingApplicationProvider {
+ targets: Mutex<Vec<(i64, Option<u64>)>>,
+ }
+
+ impl ApplicationIdentityProvider for RecordingApplicationProvider {
+ fn resolve(&self, pid: i64, window_id: Option<u64>) -> Option<ApplicationIdentity> {
+ self.targets.lock().unwrap().push((pid, window_id));
+ Some(ApplicationIdentity {
+ bundle_id: Some("test.app".to_owned()),
+ display_name: Some("Test App".to_owned()),
+ })
+ }
+ }
+
#[derive(Default)]
struct MemoryKeyProvider {
keys: Mutex<HashMap<String, Vec<u8>>>,
@@ -2687,6 +2717,48 @@ mod tests {
.all(|event| event.data.session_id.as_deref() == Some(opaque.as_str())));
}
+ #[test]
+ fn action_attribution_receives_pid_and_window_identity() {
+ let temp = tempfile::tempdir().unwrap();
+ let provider = Arc::new(RecordingApplicationProvider::default());
+ let manager = HistoryManager::new(
+ config(temp.path()),
+ Arc::new(MemoryKeyProvider::default()),
+ Some(provider.clone()),
+ );
+ manager.enable().unwrap();
+ let pending = manager
+ .begin_action(
+ "click",
+ &serde_json::json!({"pid": 42, "window_id": 9001}),
+ None,
+ )
+ .unwrap();
+ manager.finish_action(pending, None, false);
+ manager.flush().unwrap();
+
+ assert_eq!(
+ provider.targets.lock().unwrap().as_slice(),
+ &[(42, Some(9001)), (42, Some(9001))]
+ );
+ let events = manager
+ .query(
+ HistoryQuery {
+ limit: Some(MAX_QUERY_LIMIT),
+ ..Default::default()
+ },
+ HistoryAccessOperation::LocalCli,
+ )
+ .unwrap();
+ assert_eq!(
+ events
+ .iter()
+ .filter_map(|event| event.data.application.as_ref())
+ .count(),
+ 2
+ );
+ }
+
#[test]
fn malformed_multibyte_hex_is_rejected_without_panicking() {
let malformed = format!("€{}", "a".repeat(29));
diff --git a/libs/cua-driver/rust/crates/platform-linux/src/history.rs b/libs/cua-driver/rust/crates/platform-linux/src/history.rs
index 993ca73bfe..5f3768b7da 100644
--- a/libs/cua-driver/rust/crates/platform-linux/src/history.rs
+++ b/libs/cua-driver/rust/crates/platform-linux/src/history.rs
@@ -165,7 +165,7 @@ fn application_identity_from_stable_sources(
}
impl ApplicationIdentityProvider for LinuxApplicationIdentityProvider {
- fn resolve(&self, pid: i64) -> Option<ApplicationIdentity> {
+ fn resolve(&self, pid: i64, _window_id: Option<u64>) -> Option<ApplicationIdentity> {
let pid = u32::try_from(pid).ok()?;
let process = crate::proc_fs::list_processes()
.into_iter()
@@ -217,7 +217,7 @@ mod tests {
#[test]
fn current_process_identity_is_resolvable_without_titles_or_paths() {
let identity = LinuxApplicationIdentityProvider
- .resolve(i64::from(std::process::id()))
+ .resolve(i64::from(std::process::id()), None)
.expect("current process identity");
assert!(identity.bundle_id.is_some() || identity.display_name.is_some());
}
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/history.rs b/libs/cua-driver/rust/crates/platform-macos/src/history.rs
index acf630b0c0..545904eb6f 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/history.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/history.rs
@@ -178,7 +178,7 @@ fn map_keychain_error(error: security_framework::base::Error) -> HistoryError {
pub struct MacosApplicationIdentityProvider;
impl ApplicationIdentityProvider for MacosApplicationIdentityProvider {
- fn resolve(&self, pid: i64) -> Option<ApplicationIdentity> {
+ fn resolve(&self, pid: i64, _window_id: Option<u64>) -> Option<ApplicationIdentity> {
let pid = i32::try_from(pid).ok()?;
crate::apps::list_running_apps()
.into_iter()
diff --git a/libs/cua-driver/rust/crates/platform-windows/src/history.rs b/libs/cua-driver/rust/crates/platform-windows/src/history.rs
index 72916ae9fa..3dee295fd0 100644
--- a/libs/cua-driver/rust/crates/platform-windows/src/history.rs
+++ b/libs/cua-driver/rust/crates/platform-windows/src/history.rs
@@ -137,19 +137,28 @@ fn map_keyring_error(error: KeyringError) -> HistoryError {
pub struct WindowsApplicationIdentityProvider;
impl ApplicationIdentityProvider for WindowsApplicationIdentityProvider {
- fn resolve(&self, pid: i64) -> Option<ApplicationIdentity> {
- let pid = u32::try_from(pid).ok()?;
- let name = crate::win32::list_processes()
- .into_iter()
- .find(|process| process.pid == pid)?
- .name;
- (!name.trim().is_empty()).then(|| ApplicationIdentity {
+ fn resolve(&self, pid: i64, window_id: Option<u64>) -> Option<ApplicationIdentity> {
+ let mut pid = u32::try_from(pid).ok()?;
+ let owner_name = process_name(pid)?;
+ if owner_name.eq_ignore_ascii_case("ApplicationFrameHost.exe") {
+ pid = crate::win32::resolve_uwp_app_pid(pid, window_id?)?;
+ }
+ let name = process_name(pid)?;
+ Some(ApplicationIdentity {
bundle_id: Some(name.clone()),
display_name: Some(name),
})
}
}
+fn process_name(pid: u32) -> Option<String> {
+ let name = crate::win32::list_processes()
+ .into_iter()
+ .find(|process| process.pid == pid)?
+ .name;
+ (!name.trim().is_empty()).then_some(name)
+}
+
pub fn application_identity_provider() -> Arc<dyn ApplicationIdentityProvider> {
Arc::new(WindowsApplicationIdentityProvider)
}
diff --git a/libs/cua-driver/rust/crates/platform-windows/src/win32/mod.rs b/libs/cua-driver/rust/crates/platform-windows/src/win32/mod.rs
index 38f98f89d7..bbb431ae9f 100644
--- a/libs/cua-driver/rust/crates/platform-windows/src/win32/mod.rs
+++ b/libs/cua-driver/rust/crates/platform-windows/src/win32/mod.rs
@@ -8,6 +8,7 @@ pub use apps::{list_descendants, list_processes, related_processes, ProcessInfo}
pub use installed_apps::{list_installed_apps, InstalledApp};
pub(crate) use windows::{
capture_foreground_target, find_window_by_pid_and_handle,
- foreground_matches_target_or_owned_window, list_windows_via_win32, window_owner_pid,
+ foreground_matches_target_or_owned_window, list_windows_via_win32, resolve_uwp_app_pid,
+ window_owner_pid,
};
pub use windows::{list_windows, resolve_uwp_host_window, WindowInfo};
diff --git a/libs/cua-driver/rust/crates/platform-windows/src/win32/windows.rs b/libs/cua-driver/rust/crates/platform-windows/src/win32/windows.rs
index 840b11ba5e..e68fe10ccc 100644
--- a/libs/cua-driver/rust/crates/platform-windows/src/win32/windows.rs
+++ b/libs/cua-driver/rust/crates/platform-windows/src/win32/windows.rs
@@ -604,3 +604,55 @@ pub fn resolve_uwp_host_window(app_pid: u32) -> Option<WindowInfo> {
minimized: false,
})
}
+
+/// Resolve the real packaged-app process hosted by a specific
+/// `ApplicationFrameWindow`.
+///
+/// An ApplicationFrameHost pid is shared by unrelated apps, so callers must
+/// provide the frame HWND as well as the host pid. The hosted CoreWindow child
+/// retains ownership by the packaged app process and supplies the identity we
+/// need for attribution.
+pub fn resolve_uwp_app_pid(host_pid: u32, frame_hwnd: u64) -> Option<u32> {
+ let frame = HWND(frame_hwnd as *mut _);
+ if unsafe { IsWindow(frame) }.0 == 0 || window_class_name(frame) != "ApplicationFrameWindow" {
+ return None;
+ }
+
+ let mut owner_pid = 0;
+ unsafe { GetWindowThreadProcessId(frame, Some(&mut owner_pid)) };
+ if owner_pid != host_pid {
+ return None;
+ }
+
+ struct ChildScan {
+ host_pid: u32,
+ app_pid: Option<u32>,
+ }
+
+ unsafe extern "system" fn child_cb(child: HWND, lparam: LPARAM) -> BOOL {
+ let scan = &mut *(lparam.0 as *mut ChildScan);
+ let mut child_pid = 0;
+ GetWindowThreadProcessId(child, Some(&mut child_pid));
+ if child_pid != 0
+ && child_pid != scan.host_pid
+ && window_class_name(child) == "Windows.UI.Core.CoreWindow"
+ {
+ scan.app_pid = Some(child_pid);
+ return windows::Win32::Foundation::FALSE;
+ }
+ TRUE
+ }
+
+ let mut scan = ChildScan {
+ host_pid,
+ app_pid: None,
+ };
+ unsafe {
+ let _ = EnumChildWindows(
+ frame,
+ Some(child_cb),
+ LPARAM(&mut scan as *mut ChildScan as isize),
+ );
+ }
+ scan.app_pid
+}