Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

test(cua-driver): scale Linux screenshot coordinates (#3724)

Francesco Bonacci55f4304fb8cf
diff --git a/libs/cua-driver/rust/crates/cua-driver/tests/cross_platform_behavior_test.rs b/libs/cua-driver/rust/crates/cua-driver/tests/cross_platform_behavior_test.rs
index 8ad195ba0f..2a69547341 100644
--- a/libs/cua-driver/rust/crates/cua-driver/tests/cross_platform_behavior_test.rs
+++ b/libs/cua-driver/rust/crates/cua-driver/tests/cross_platform_behavior_test.rs
@@ -477,26 +477,92 @@ fn window_origin(fixture: &Fixture, _state: &ToolResponse) -> (f64, f64) {
(fixture.window_x, fixture.window_y)
}
-fn screenshot_scale(state: &ToolResponse) -> f64 {
- let Some(width) = state.structured()["screenshot_width"].as_f64() else {
- return 1.0;
- };
- let window_width = state.structured()["elements"]
+fn screenshot_scale(state: &ToolResponse) -> (f64, f64) {
+ screenshot_scale_for_platform(state.structured(), cfg!(target_os = "linux"))
+}
+
+fn screenshot_scale_for_platform(state: &serde_json::Value, linux: bool) -> (f64, f64) {
+ let window_role = if linux { "frame" } else { "AXWindow" };
+ let capture = (
+ state["screenshot_width"].as_f64().unwrap_or(0.0),
+ state["screenshot_height"].as_f64().unwrap_or(0.0),
+ );
+ let window = state["elements"]
.as_array()
.and_then(|elements| {
elements
.iter()
- .find(|element| element["role"].as_str() == Some("AXWindow"))
+ .find(|element| element["role"].as_str() == Some(window_role))
+ })
+ .map(|window| {
+ (
+ window["frame"]["w"].as_f64().unwrap_or(0.0),
+ window["frame"]["h"].as_f64().unwrap_or(0.0),
+ )
})
- .and_then(|window| window["frame"]["w"].as_f64())
- .unwrap_or(0.0);
- if window_width > 0.0 && width > 0.0 {
- width / window_width
+ .unwrap_or((0.0, 0.0));
+ if capture.0 > 0.0 && capture.1 > 0.0 && window.0 > 0.0 && window.1 > 0.0 {
+ (capture.0 / window.0, capture.1 / window.1)
} else {
- 1.0
+ (1.0, 1.0)
}
}
+fn screenshot_local_point(
+ screen_point: (f64, f64),
+ window_origin: (f64, f64),
+ scale: (f64, f64),
+) -> (f64, f64) {
+ (
+ (screen_point.0 - window_origin.0) * scale.0,
+ (screen_point.1 - window_origin.1) * scale.1,
+ )
+}
+
+#[test]
+fn linux_frame_geometry_drives_resized_screenshot_scale() {
+ let state = serde_json::json!({
+ "screenshot_width": 1568,
+ "screenshot_height": 852,
+ "elements": [{
+ "element_index": 0,
+ "role": "frame",
+ "frame": {"x": 12, "y": 38, "w": 1896, "h": 1030}
+ }, {
+ "element_index": 1,
+ "role": "AXWindow",
+ "frame": {"x": 0, "y": 0, "w": 1000, "h": 500}
+ }]
+ });
+
+ let linux_scale = screenshot_scale_for_platform(&state, true);
+ assert!((linux_scale.0 - 1568.0 / 1896.0).abs() < f64::EPSILON);
+ assert!((linux_scale.1 - 852.0 / 1030.0).abs() < f64::EPSILON);
+
+ let ax_scale = screenshot_scale_for_platform(&state, false);
+ assert!((ax_scale.0 - 1568.0 / 1000.0).abs() < f64::EPSILON);
+ assert!((ax_scale.1 - 852.0 / 500.0).abs() < f64::EPSILON);
+}
+
+#[test]
+fn resized_screenshot_coordinates_use_capture_scale() {
+ let state = serde_json::json!({
+ "screenshot_width": 1568,
+ "screenshot_height": 852,
+ "elements": [{
+ "role": "frame",
+ "frame": {"x": 12, "y": 38, "w": 1896, "h": 1030}
+ }]
+ });
+ let scale = screenshot_scale_for_platform(&state, true);
+ let point = screenshot_local_point((949.0, 398.0), (12.0, 38.0), scale);
+
+ assert!((point.0 - 774.903).abs() < 0.001);
+ assert!((point.1 - 297.786).abs() < 0.001);
+ assert!(point.0 < 1568.0);
+ assert!(point.1 < 852.0);
+}
+
fn require_element(snapshot: &ToolResponse, id: &str) -> u64 {
// Some WebKit/Chromium AX adapters preserve the DOM id as an annotation
// instead of emitting the common `id=...` form. Searching the visible
@@ -705,9 +771,8 @@ fn action_target_args(
} else {
let origin = window_origin(fixture, state);
let scale = screenshot_scale(state);
- let (x, y) = element_center(state, index);
- let local_x = (x - origin.0) * scale;
- let local_y = (y - origin.1) * scale;
+ let screen_point = element_center(state, index);
+ let (local_x, local_y) = screenshot_local_point(screen_point, origin, scale);
let width = state.structured()["screenshot_width"].as_f64();
let height = state.structured()["screenshot_height"].as_f64();
if width.is_none() || height.is_none() {
@@ -728,8 +793,8 @@ fn action_target_args(
let width = width.unwrap();
let height = height.unwrap();
eprintln!(
- "[shared-px] {} target={id} screen=({x:.1},{y:.1}) origin=({:.1},{:.1}) scale={scale:.3} local=({local_x:.1},{local_y:.1}) capture=({width:.1}x{height:.1})",
- fixture.name, origin.0, origin.1
+ "[shared-px] {} target={id} screen=({:.1},{:.1}) origin=({:.1},{:.1}) scale=({:.3},{:.3}) local=({local_x:.1},{local_y:.1}) capture=({width:.1}x{height:.1})",
+ fixture.name, screen_point.0, screen_point.1, origin.0, origin.1, scale.0, scale.1
);
assert!(
local_x >= 0.0 && local_x < width && local_y >= 0.0 && local_y < height,
@@ -1338,10 +1403,7 @@ fn run_drag_action(fixture: &mut Fixture, delivery: &str) -> Observation {
let target = require_element(&pre, "drop-target");
let origin = window_origin(fixture, &pre);
let scale = screenshot_scale(&pre);
- let point = |index: u64| {
- let (x, y) = element_center(&pre, index);
- ((x - origin.0) * scale, (y - origin.1) * scale)
- };
+ let point = |index: u64| screenshot_local_point(element_center(&pre, index), origin, scale);
let (from_x, from_y) = point(source);
let (to_x, to_y) = point(target);
let response = fixture.driver.call(