Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

fix(cua-driver): scale Windows foreground scroll targets from the screenshot context

injaneitye6de7c1708af
diff --git a/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs b/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs
index 94d1c68610..07d22a84b4 100644
--- a/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs
+++ b/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs
@@ -6585,7 +6585,18 @@ impl Tool for ScrollTool {
return ToolResult::error("scroll requires x and y together.");
}
let center = if let (Some(x), Some(y)) = (px, py) {
- Some(bitmap_to_screen(hwnd, x as i32, y as i32))
+ let (x, y) = match super::scroll_coordinates::window_bitmap_point(
+ &self.state.snapshots,
+ &args,
+ pid,
+ hwnd,
+ x,
+ y,
+ ) {
+ Ok(point) => point,
+ Err(refusal) => return refusal,
+ };
+ Some(bitmap_to_screen(hwnd, x, y))
} else {
tokio::task::spawn_blocking({
let admitted = admitted.clone();
diff --git a/libs/cua-driver/rust/crates/platform-windows/src/tools/mod.rs b/libs/cua-driver/rust/crates/platform-windows/src/tools/mod.rs
index b7c4793603..575f2ed2e2 100644
--- a/libs/cua-driver/rust/crates/platform-windows/src/tools/mod.rs
+++ b/libs/cua-driver/rust/crates/platform-windows/src/tools/mod.rs
@@ -13,6 +13,7 @@ pub(crate) mod page;
pub(crate) mod page_bookmark;
// Pure title parsing for `page_bookmark`; ungated so its tests run on any host.
pub(crate) mod page_title_marker;
+mod scroll_coordinates;
#[cfg(not(target_os = "windows"))]
mod stubs;
diff --git a/libs/cua-driver/rust/crates/platform-windows/src/tools/scroll_coordinates.rs b/libs/cua-driver/rust/crates/platform-windows/src/tools/scroll_coordinates.rs
new file mode 100644
index 0000000000..21e3c3e8a8
--- /dev/null
+++ b/libs/cua-driver/rust/crates/platform-windows/src/tools/scroll_coordinates.rs
@@ -0,0 +1,98 @@
+//! Window scroll pixels use the same screenshot context as pixel clicks.
+
+use cua_driver_core::{
+ protocol::ToolResult,
+ snapshot_store::{SnapshotPayload, SnapshotStore},
+};
+use serde_json::Value;
+
+pub(super) fn window_bitmap_point<S: SnapshotPayload>(
+ snapshots: &SnapshotStore<S>,
+ args: &Value,
+ pid: u32,
+ window_id: u64,
+ x: f64,
+ y: f64,
+) -> Result<(i32, i32), ToolResult> {
+ let ratio = snapshots.screenshot_scale(pid as i32, Some(window_id), args)?;
+ Ok(((x * ratio) as i32, (y * ratio) as i32))
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use serde_json::json;
+
+ struct Empty;
+ impl SnapshotPayload for Empty {
+ type Element = ();
+ fn len(&self) -> usize {
+ 0
+ }
+ fn retain(&self, _: usize) -> Option<()> {
+ None
+ }
+ }
+
+ fn args(session: &str) -> Value {
+ json!({"_session_id": session})
+ }
+
+ #[test]
+ fn resized_scroll_targets_native_bitmap_pixels() {
+ let snapshots = SnapshotStore::new();
+ snapshots.publish_for_session(42, 7, Empty, Some("a"), Some(1920.0 / 1400.0));
+ assert_eq!(
+ window_bitmap_point(&snapshots, &args("a"), 42, 7, 1000.0, 650.0).unwrap(),
+ (1371, 891)
+ );
+ }
+
+ #[test]
+ fn native_size_scroll_preserves_fractional_pixel_cast() {
+ let snapshots = SnapshotStore::new();
+ snapshots.publish_for_session(42, 7, Empty, Some("a"), Some(1.0));
+ assert_eq!(
+ window_bitmap_point(&snapshots, &args("a"), 42, 7, 1000.75, 650.25).unwrap(),
+ (1000, 650)
+ );
+ }
+
+ #[test]
+ fn scroll_uses_the_selected_window_and_session() {
+ let snapshots = SnapshotStore::new();
+ snapshots.publish_for_session(42, 7, Empty, Some("a"), Some(2.0));
+ snapshots.publish_for_session(42, 8, Empty, Some("b"), Some(3.0));
+ assert_eq!(
+ window_bitmap_point(&snapshots, &args("a"), 42, 7, 10.0, 20.0).unwrap(),
+ (20, 40)
+ );
+ assert_eq!(
+ window_bitmap_point(&snapshots, &args("b"), 42, 8, 10.0, 20.0).unwrap(),
+ (30, 60)
+ );
+ }
+
+ #[test]
+ fn scroll_refuses_missing_or_foreign_screenshot_context() {
+ let snapshots = SnapshotStore::<Empty>::new();
+ let refusal = window_bitmap_point(&snapshots, &args("a"), 42, 7, 10.0, 20.0).unwrap_err();
+ assert_eq!(
+ refusal.structured_content.unwrap()["code"],
+ "screenshot_context_missing"
+ );
+ snapshots.publish_for_session(42, 7, Empty, Some("b"), Some(2.0));
+ assert!(window_bitmap_point(&snapshots, &args("a"), 42, 7, 10.0, 20.0).is_err());
+ }
+
+ #[test]
+ fn trusted_native_pixels_are_not_scaled_twice() {
+ let snapshots = SnapshotStore::new();
+ snapshots.publish_for_session(42, 7, Empty, Some("a"), Some(2.0));
+ let native = json!({"_session_id":"a", "_native_window_pixels":true});
+ assert_eq!(
+ window_bitmap_point(&snapshots, &native, 42, 7, 10.0, 20.0).unwrap(),
+ (10, 20)
+ );
+ }
+}