Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
fix(cua-driver): add DPI awareness manifest for Windows (#1821)
ddupontb286d9ccd8ea
diff --git a/libs/cua-driver/rust/Cargo.lock b/libs/cua-driver/rust/Cargo.lockindex 66afcb0b8d..04d91fb754 100644--- a/libs/cua-driver/rust/Cargo.lock+++ b/libs/cua-driver/rust/Cargo.lock@@ -282,6 +282,7 @@ dependencies = ["base64","cua-driver-core","cursor-overlay",+ "embed-resource","flate2","image","libc",@@ -414,6 +415,20 @@ version = "1.5.0"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "94cdc65b1cf9e871453ce2f86f5aaec24ff2eaa36a1fa3e02e441dddc3613b99"+[[package]]+name = "embed-resource"+version = "2.5.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "d506610004cfc74a6f5ee7e8c632b355de5eca1f03ee5e5e0ec11b77d4eb3d61"+dependencies = [+ "cc",+ "memchr",+ "rustc_version",+ "toml",+ "vswhom",+ "winreg",+]+[[package]]name = "equivalent"version = "1.0.2"@@ -1469,6 +1484,15 @@ version = "0.20.0"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "6c20b6793b5c2fa6553b250154b78d6d0db37e72700ae35fad9387a46f487c97"+[[package]]+name = "rustc_version"+version = "0.4.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92"+dependencies = [+ "semver",+]+[[package]]name = "rustix"version = "1.1.4"@@ -1638,6 +1662,15 @@ dependencies = ["zmij",]+[[package]]+name = "serde_spanned"+version = "0.6.9"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "bf41e0cfaf7226dca15e8197172c295a782857fcb97fad1808a166870dee75a3"+dependencies = [+ "serde",+]+[[package]]name = "sha1"version = "0.10.6"@@ -1965,6 +1998,47 @@ dependencies = ["tungstenite",]+[[package]]+name = "toml"+version = "0.8.23"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "dc1beb996b9d83529a9e75c17a1686767d148d70663143c7854d8b4a09ced362"+dependencies = [+ "serde",+ "serde_spanned",+ "toml_datetime",+ "toml_edit",+]++[[package]]+name = "toml_datetime"+version = "0.6.11"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "22cddaf88f4fbc13c51aebbf5f8eceb5c7c5a9da2ac40a13519eb5b0a0e8f11c"+dependencies = [+ "serde",+]++[[package]]+name = "toml_edit"+version = "0.22.27"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a"+dependencies = [+ "indexmap",+ "serde",+ "serde_spanned",+ "toml_datetime",+ "toml_write",+ "winnow",+]++[[package]]+name = "toml_write"+version = "0.1.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "5d99f8c9a7727884afe522e9bd5edbfc91a3312b36a77b5fb8926e4c31a41801"+[[package]]name = "tracing"version = "0.1.44"@@ -2232,6 +2306,26 @@ version = "0.9.5"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a"+[[package]]+name = "vswhom"+version = "0.1.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "be979b7f07507105799e854203b470ff7c78a1639e330a58f183b5fea574608b"+dependencies = [+ "libc",+ "vswhom-sys",+]++[[package]]+name = "vswhom-sys"+version = "0.1.3"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "fb067e4cbd1ff067d1df46c9194b5de0e98efd2810bbc95c5d5e5f25a3231150"+dependencies = [+ "cc",+ "libc",+]+[[package]]name = "wait-timeout"version = "0.2.1"@@ -2360,7 +2454,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "dd04d41d93c4992d421894c18c8b43496aa748dd4c081bac0dc93eb0489272b6"dependencies = ["windows-core 0.58.0",- "windows-targets",+ "windows-targets 0.52.6",][[package]]@@ -2395,7 +2489,7 @@ dependencies = ["windows-interface 0.58.0","windows-result 0.2.0","windows-strings 0.1.0",- "windows-targets",+ "windows-targets 0.52.6",][[package]]@@ -2494,7 +2588,7 @@ version = "0.2.0"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "1d1043d8214f791817bab27572aaa8af63732e11bf84aa21a45a78d6c317ae0e"dependencies = [- "windows-targets",+ "windows-targets 0.52.6",][[package]]@@ -2513,7 +2607,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "4cd9b125c486025df0eabcb585e62173c6c9eddcec5d117d3b6e8c30e2ee4d10"dependencies = ["windows-result 0.2.0",- "windows-targets",+ "windows-targets 0.52.6",][[package]]@@ -2525,13 +2619,22 @@ dependencies = ["windows-link 0.1.3",]+[[package]]+name = "windows-sys"+version = "0.48.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9"+dependencies = [+ "windows-targets 0.48.5",+]+[[package]]name = "windows-sys"version = "0.52.0"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d"dependencies = [- "windows-targets",+ "windows-targets 0.52.6",][[package]]@@ -2543,20 +2646,35 @@ dependencies = ["windows-link 0.2.1",]+[[package]]+name = "windows-targets"+version = "0.48.5"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c"+dependencies = [+ "windows_aarch64_gnullvm 0.48.5",+ "windows_aarch64_msvc 0.48.5",+ "windows_i686_gnu 0.48.5",+ "windows_i686_msvc 0.48.5",+ "windows_x86_64_gnu 0.48.5",+ "windows_x86_64_gnullvm 0.48.5",+ "windows_x86_64_msvc 0.48.5",+]+[[package]]name = "windows-targets"version = "0.52.6"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973"dependencies = [- "windows_aarch64_gnullvm",- "windows_aarch64_msvc",- "windows_i686_gnu",+ "windows_aarch64_gnullvm 0.52.6",+ "windows_aarch64_msvc 0.52.6",+ "windows_i686_gnu 0.52.6","windows_i686_gnullvm",- "windows_i686_msvc",- "windows_x86_64_gnu",- "windows_x86_64_gnullvm",- "windows_x86_64_msvc",+ "windows_i686_msvc 0.52.6",+ "windows_x86_64_gnu 0.52.6",+ "windows_x86_64_gnullvm 0.52.6",+ "windows_x86_64_msvc 0.52.6",][[package]]@@ -2568,18 +2686,36 @@ dependencies = ["windows-link 0.1.3",]+[[package]]+name = "windows_aarch64_gnullvm"+version = "0.48.5"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8"+[[package]]name = "windows_aarch64_gnullvm"version = "0.52.6"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3"+[[package]]+name = "windows_aarch64_msvc"+version = "0.48.5"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc"+[[package]]name = "windows_aarch64_msvc"version = "0.52.6"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469"+[[package]]+name = "windows_i686_gnu"+version = "0.48.5"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e"+[[package]]name = "windows_i686_gnu"version = "0.52.6"@@ -2592,30 +2728,73 @@ version = "0.52.6"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66"+[[package]]+name = "windows_i686_msvc"+version = "0.48.5"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406"+[[package]]name = "windows_i686_msvc"version = "0.52.6"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66"+[[package]]+name = "windows_x86_64_gnu"+version = "0.48.5"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e"+[[package]]name = "windows_x86_64_gnu"version = "0.52.6"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78"+[[package]]+name = "windows_x86_64_gnullvm"+version = "0.48.5"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc"+[[package]]name = "windows_x86_64_gnullvm"version = "0.52.6"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d"+[[package]]+name = "windows_x86_64_msvc"+version = "0.48.5"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538"+[[package]]name = "windows_x86_64_msvc"version = "0.52.6"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec"+[[package]]+name = "winnow"+version = "0.7.15"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "df79d97927682d2fd8adb29682d1140b343be4ac0f08fd68b7765d9c059d3945"+dependencies = [+ "memchr",+]++[[package]]+name = "winreg"+version = "0.52.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "a277a57398d4bfa075df44f501a17cfdf8542d224f0d36095a2adc7aee4ef0a5"+dependencies = [+ "cfg-if",+ "windows-sys 0.48.0",+]+[[package]]name = "wit-bindgen"version = "0.51.0"diff --git a/libs/cua-driver/rust/crates/cua-driver/Cargo.toml b/libs/cua-driver/rust/crates/cua-driver/Cargo.tomlindex 57156b1f29..fea806f76e 100644--- a/libs/cua-driver/rust/crates/cua-driver/Cargo.toml+++ b/libs/cua-driver/rust/crates/cua-driver/Cargo.toml@@ -59,6 +59,13 @@ platform-windows = { path = "../platform-windows" }[target.'cfg(target_os = "linux")'.dependencies]platform-linux = { path = "../platform-linux" }+[build-dependencies]++[target.'cfg(target_os = "windows")'.build-dependencies]+# Embed the Windows manifest for DPI awareness (fixes coordinate mismatches+# at 125%/150%/200% scaling).+embed-resource = "2"+[dev-dependencies]tokio = { workspace = true, features = ["full"] }image = { workspace = true }diff --git a/libs/cua-driver/rust/crates/cua-driver/build.rs b/libs/cua-driver/rust/crates/cua-driver/build.rsindex 387b7c3f56..e0de604b1a 100644--- a/libs/cua-driver/rust/crates/cua-driver/build.rs+++ b/libs/cua-driver/rust/crates/cua-driver/build.rs@@ -7,9 +7,15 @@// emitting crate is the final binary crate — for transitive deps Cargo// silently drops them. So we re-emit the same rpaths from here.//-// No-op on Windows / Linux — those builds don't pull in a Swift runtime.+// On Windows, embed the DPI-awareness manifest so the process receives+// logical (not physical) screen coordinates at 125%/150%/200% scaling.fn main() {+ #[cfg(target_os = "windows")]+ {+ embed_resource::compile("cua-driver.rc", embed_resource::NONE);+ }+if std::env::var("CARGO_CFG_TARGET_OS").as_deref() != Ok("macos") {return;}diff --git a/libs/cua-driver/rust/crates/cua-driver/cua-driver.manifest b/libs/cua-driver/rust/crates/cua-driver/cua-driver.manifestnew file mode 100644index 0000000000..a1e8062975--- /dev/null+++ b/libs/cua-driver/rust/crates/cua-driver/cua-driver.manifest@@ -0,0 +1,27 @@+<?xml version="1.0" encoding="UTF-8" standalone="yes"?>+<assembly xmlns="urn:schemas-microsoft-com:asm.v1" manifestVersion="1.0">+ <assemblyIdentity+ type="win32"+ name="com.trycua.driver"+ version="1.0.0.0"+ processorArchitecture="*"/>+ <trustInfo xmlns="urn:schemas-microsoft-com:asm.v3">+ <security>+ <requestedPrivileges>+ <requestedExecutionLevel level="asInvoker" uiAccess="false"/>+ </requestedPrivileges>+ </security>+ </trustInfo>+ <application xmlns="urn:schemas-microsoft-com:asm.v3">+ <windowsSettings>+ <dpiAware xmlns="http://schemas.microsoft.com/SMI/2005/WindowsSettings">true/pm</dpiAware>+ <dpiAwareness xmlns="http://schemas.microsoft.com/SMI/2016/WindowsSettings">permonitorv2</dpiAwareness>+ </windowsSettings>+ </application>+ <compatibility xmlns="urn:schemas-microsoft-com:compatibility.v1">+ <application>+ <!-- Windows 10 / Windows 11 -->+ <supportedOS Id="{8e0f7a12-bfb3-4fe8-b9a5-48fd50a15a9a}"/>+ </application>+ </compatibility>+</assembly>diff --git a/libs/cua-driver/rust/crates/cua-driver/cua-driver.rc b/libs/cua-driver/rust/crates/cua-driver/cua-driver.rcnew file mode 100644index 0000000000..d7c5fc9f80--- /dev/null+++ b/libs/cua-driver/rust/crates/cua-driver/cua-driver.rc@@ -0,0 +1 @@+1 RT_MANIFEST "cua-driver.manifest"diff --git a/libs/cua-driver/rust/crates/platform-windows/src/capture.rs b/libs/cua-driver/rust/crates/platform-windows/src/capture.rsindex 4b3b0c664d..2247e39c90 100644--- a/libs/cua-driver/rust/crates/platform-windows/src/capture.rs+++ b/libs/cua-driver/rust/crates/platform-windows/src/capture.rs@@ -131,11 +131,23 @@ unsafe fn target_is_obscured(target: HWND) -> bool {unsafe fn screenshot_via_screen_region(hwnd: HWND) -> Result<(Vec<u8>, i32, i32)> {use windows::Win32::Foundation::RECT;use windows::Win32::UI::WindowsAndMessaging::GetWindowRect;+ use windows::Win32::UI::HiDpi::GetDpiForWindow;let mut rect = RECT::default();GetWindowRect(hwnd, &mut rect)?;- let w = rect.right - rect.left;- let h = rect.bottom - rect.top;++ // With permonitorv2 DPI awareness, GetWindowRect returns logical coordinates.+ // BitBlt needs physical pixel coordinates, so scale by the window's DPI.+ let dpi = GetDpiForWindow(hwnd);+ let scale = if dpi == 0 { 1.0 } else { dpi as f64 / 96.0 };++ let physical_left = (rect.left as f64 * scale).round() as i32;+ let physical_top = (rect.top as f64 * scale).round() as i32;+ let physical_right = (rect.right as f64 * scale).round() as i32;+ let physical_bottom = (rect.bottom as f64 * scale).round() as i32;++ let w = physical_right - physical_left;+ let h = physical_bottom - physical_top;if w <= 0 || h <= 0 {bail!("screen-region fallback: window has zero/negative bounds: {w}x{h}");}@@ -145,8 +157,8 @@ unsafe fn screenshot_via_screen_region(hwnd: HWND) -> Result<(Vec<u8>, i32, i32)let bitmap = CreateCompatibleBitmap(screen_dc, w, h);let old_bitmap = SelectObject(mem_dc, bitmap);- // Copy from screen coords (rect.left, rect.top) into our memory DC at (0, 0).- let blt_ok = BitBlt(mem_dc, 0, 0, w, h, screen_dc, rect.left, rect.top, SRCCOPY);+ // Copy from physical screen coords into our memory DC at (0, 0).+ let blt_ok = BitBlt(mem_dc, 0, 0, w, h, screen_dc, physical_left, physical_top, SRCCOPY);let mut bmi = BITMAPINFO {bmiHeader: BITMAPINFOHEADER {@@ -272,14 +284,24 @@ unsafe fn screenshot_window_bytes_with_occlusion_unsafe(hwnd: u64) -> Result<(Ve// client-sized buffer loses the Save/Cancel/OK row at the bottom.// Window-sized buffer captures title bar + body + non-client trim// correctly.+ //+ // With permonitorv2 DPI awareness, GetWindowRect returns logical dimensions+ // but GetWindowDC and PrintWindow/BitBlt operate in physical pixels.+ // Scale to physical dimensions for the bitmap.let mut win_rect = RECT::default();GetWindowRect(hwnd, &mut win_rect)?;- let w = (win_rect.right - win_rect.left) as i32;- let h = (win_rect.bottom - win_rect.top) as i32;- if w <= 0 || h <= 0 {- bail!("Window has zero/negative size: {}x{}", w, h);+ let logical_w = (win_rect.right - win_rect.left) as i32;+ let logical_h = (win_rect.bottom - win_rect.top) as i32;+ if logical_w <= 0 || logical_h <= 0 {+ bail!("Window has zero/negative size: {}x{}", logical_w, logical_h);}+ use windows::Win32::UI::HiDpi::GetDpiForWindow;+ let dpi = GetDpiForWindow(hwnd);+ let scale = if dpi == 0 { 1.0 } else { dpi as f64 / 96.0 };+ let w = (logical_w as f64 * scale).round() as i32;+ let h = (logical_h as f64 * scale).round() as i32;+let screen_dc = GetWindowDC(hwnd);let mem_dc = CreateCompatibleDC(screen_dc);let bitmap = CreateCompatibleBitmap(screen_dc, w, h);@@ -454,9 +476,17 @@ unsafe fn screenshot_window_bytes_with_occlusion_unsafe(hwnd: u64) -> Result<(Vepub fn screenshot_display_bytes() -> Result<Vec<u8>> {unsafe {use windows::Win32::UI::WindowsAndMessaging::{GetSystemMetrics, SM_CXSCREEN, SM_CYSCREEN};- let w = GetSystemMetrics(SM_CXSCREEN);- let h = GetSystemMetrics(SM_CYSCREEN);- if w <= 0 || h <= 0 { bail!("Could not get screen metrics"); }+ use windows::Win32::UI::HiDpi::GetDpiForSystem;+ // With permonitorv2 DPI awareness, GetSystemMetrics returns logical pixels.+ // BitBlt captures physical pixels, so we must scale by DPI factor.+ let logical_w = GetSystemMetrics(SM_CXSCREEN);+ let logical_h = GetSystemMetrics(SM_CYSCREEN);+ if logical_w <= 0 || logical_h <= 0 { bail!("Could not get screen metrics"); }++ let dpi = GetDpiForSystem();+ let scale = if dpi == 0 { 1.0 } else { dpi as f64 / 96.0 };+ let w = (logical_w as f64 * scale).round() as i32;+ let h = (logical_h as f64 * scale).round() as i32;let screen_dc = GetDC(HWND::default());let mem_dc = CreateCompatibleDC(screen_dc);let bitmap = CreateCompatibleBitmap(screen_dc, w, h);diff --git a/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs b/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rsindex b1678d28b4..e0c60bbd7b 100644--- a/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs+++ b/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs@@ -3882,9 +3882,10 @@ impl Tool for GetScreenSizeTool {async fn invoke(&self, _args: Value) -> ToolResult {use windows::Win32::UI::WindowsAndMessaging::{GetSystemMetrics, SM_CXSCREEN, SM_CYSCREEN};use windows::Win32::UI::HiDpi::GetDpiForSystem;- // SM_CXSCREEN/SM_CYSCREEN return values in the DPI of the current- // process; for a DPI-unaware process they're in *logical* points,- // matching Swift's NSScreen.frame. Scale = system DPI / 96.+ // With permonitorv2 DPI awareness (set in cua-driver.manifest),+ // SM_CXSCREEN/SM_CYSCREEN already return logical points (DPI-scaled).+ // Report these as-is, along with the scale factor for reference.+ // Matches Swift's NSScreen.frame behavior on macOS.let (w, h) = unsafe { (GetSystemMetrics(SM_CXSCREEN), GetSystemMetrics(SM_CYSCREEN)) };let dpi = unsafe { GetDpiForSystem() };let scale = if dpi == 0 { 1.0 } else { dpi as f64 / 96.0 };diff --git a/nix/cua-driver/package.nix b/nix/cua-driver/package.nixindex e73d636a0d..52f07e10e5 100644--- a/nix/cua-driver/package.nix+++ b/nix/cua-driver/package.nix@@ -23,7 +23,7 @@ pkgs.rustPlatform.buildRustPackage {# the workspace Cargo.lock includes macOS-only crates (apple-metal, apple-cf)# that may be unreachable from crates.io. fetchCargoVendor handles this# gracefully via `cargo vendor`.- cargoHash = "sha256-nkVI+O2P/QSTx15dCraNNGDAsEw3m7c/u/V6BXXSTww=";+ cargoHash = "sha256-TezobhZKan2E087x8cECCqZS0lafEBAOd0Cx70BgP9w=";# Build only the main binary crate. The workspace also contains# platform-macos, platform-windows, cua-driver-uia, and focus-monitor-win