Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
test(cua-driver): scale Linux screenshot coordinates (#3724)
Francesco Bonacci55f4304fb8cf
diff --git a/libs/cua-driver/rust/crates/cua-driver/tests/cross_platform_behavior_test.rs b/libs/cua-driver/rust/crates/cua-driver/tests/cross_platform_behavior_test.rsindex 8ad195ba0f..2a69547341 100644--- a/libs/cua-driver/rust/crates/cua-driver/tests/cross_platform_behavior_test.rs+++ b/libs/cua-driver/rust/crates/cua-driver/tests/cross_platform_behavior_test.rs@@ -477,26 +477,92 @@ fn window_origin(fixture: &Fixture, _state: &ToolResponse) -> (f64, f64) {(fixture.window_x, fixture.window_y)}-fn screenshot_scale(state: &ToolResponse) -> f64 {- let Some(width) = state.structured()["screenshot_width"].as_f64() else {- return 1.0;- };- let window_width = state.structured()["elements"]+fn screenshot_scale(state: &ToolResponse) -> (f64, f64) {+ screenshot_scale_for_platform(state.structured(), cfg!(target_os = "linux"))+}++fn screenshot_scale_for_platform(state: &serde_json::Value, linux: bool) -> (f64, f64) {+ let window_role = if linux { "frame" } else { "AXWindow" };+ let capture = (+ state["screenshot_width"].as_f64().unwrap_or(0.0),+ state["screenshot_height"].as_f64().unwrap_or(0.0),+ );+ let window = state["elements"].as_array().and_then(|elements| {elements.iter()- .find(|element| element["role"].as_str() == Some("AXWindow"))+ .find(|element| element["role"].as_str() == Some(window_role))+ })+ .map(|window| {+ (+ window["frame"]["w"].as_f64().unwrap_or(0.0),+ window["frame"]["h"].as_f64().unwrap_or(0.0),+ )})- .and_then(|window| window["frame"]["w"].as_f64())- .unwrap_or(0.0);- if window_width > 0.0 && width > 0.0 {- width / window_width+ .unwrap_or((0.0, 0.0));+ if capture.0 > 0.0 && capture.1 > 0.0 && window.0 > 0.0 && window.1 > 0.0 {+ (capture.0 / window.0, capture.1 / window.1)} else {- 1.0+ (1.0, 1.0)}}+fn screenshot_local_point(+ screen_point: (f64, f64),+ window_origin: (f64, f64),+ scale: (f64, f64),+) -> (f64, f64) {+ (+ (screen_point.0 - window_origin.0) * scale.0,+ (screen_point.1 - window_origin.1) * scale.1,+ )+}++#[test]+fn linux_frame_geometry_drives_resized_screenshot_scale() {+ let state = serde_json::json!({+ "screenshot_width": 1568,+ "screenshot_height": 852,+ "elements": [{+ "element_index": 0,+ "role": "frame",+ "frame": {"x": 12, "y": 38, "w": 1896, "h": 1030}+ }, {+ "element_index": 1,+ "role": "AXWindow",+ "frame": {"x": 0, "y": 0, "w": 1000, "h": 500}+ }]+ });++ let linux_scale = screenshot_scale_for_platform(&state, true);+ assert!((linux_scale.0 - 1568.0 / 1896.0).abs() < f64::EPSILON);+ assert!((linux_scale.1 - 852.0 / 1030.0).abs() < f64::EPSILON);++ let ax_scale = screenshot_scale_for_platform(&state, false);+ assert!((ax_scale.0 - 1568.0 / 1000.0).abs() < f64::EPSILON);+ assert!((ax_scale.1 - 852.0 / 500.0).abs() < f64::EPSILON);+}++#[test]+fn resized_screenshot_coordinates_use_capture_scale() {+ let state = serde_json::json!({+ "screenshot_width": 1568,+ "screenshot_height": 852,+ "elements": [{+ "role": "frame",+ "frame": {"x": 12, "y": 38, "w": 1896, "h": 1030}+ }]+ });+ let scale = screenshot_scale_for_platform(&state, true);+ let point = screenshot_local_point((949.0, 398.0), (12.0, 38.0), scale);++ assert!((point.0 - 774.903).abs() < 0.001);+ assert!((point.1 - 297.786).abs() < 0.001);+ assert!(point.0 < 1568.0);+ assert!(point.1 < 852.0);+}+fn require_element(snapshot: &ToolResponse, id: &str) -> u64 {// Some WebKit/Chromium AX adapters preserve the DOM id as an annotation// instead of emitting the common `id=...` form. Searching the visible@@ -705,9 +771,8 @@ fn action_target_args(} else {let origin = window_origin(fixture, state);let scale = screenshot_scale(state);- let (x, y) = element_center(state, index);- let local_x = (x - origin.0) * scale;- let local_y = (y - origin.1) * scale;+ let screen_point = element_center(state, index);+ let (local_x, local_y) = screenshot_local_point(screen_point, origin, scale);let width = state.structured()["screenshot_width"].as_f64();let height = state.structured()["screenshot_height"].as_f64();if width.is_none() || height.is_none() {@@ -728,8 +793,8 @@ fn action_target_args(let width = width.unwrap();let height = height.unwrap();eprintln!(- "[shared-px] {} target={id} screen=({x:.1},{y:.1}) origin=({:.1},{:.1}) scale={scale:.3} local=({local_x:.1},{local_y:.1}) capture=({width:.1}x{height:.1})",- fixture.name, origin.0, origin.1+ "[shared-px] {} target={id} screen=({:.1},{:.1}) origin=({:.1},{:.1}) scale=({:.3},{:.3}) local=({local_x:.1},{local_y:.1}) capture=({width:.1}x{height:.1})",+ fixture.name, screen_point.0, screen_point.1, origin.0, origin.1, scale.0, scale.1);assert!(local_x >= 0.0 && local_x < width && local_y >= 0.0 && local_y < height,@@ -1338,10 +1403,7 @@ fn run_drag_action(fixture: &mut Fixture, delivery: &str) -> Observation {let target = require_element(&pre, "drop-target");let origin = window_origin(fixture, &pre);let scale = screenshot_scale(&pre);- let point = |index: u64| {- let (x, y) = element_center(&pre, index);- ((x - origin.0) * scale, (y - origin.1) * scale)- };+ let point = |index: u64| screenshot_local_point(element_center(&pre, index), origin, scale);let (from_x, from_y) = point(source);let (to_x, to_y) = point(target);let response = fixture.driver.call(