Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

fix(cua-auto): normalize screenshots to logical pixels for DPI scaling (#1118)

Sarina Li909e6bffd6df
diff --git a/libs/python/cua-auto/cua_auto/keyboard.py b/libs/python/cua-auto/cua_auto/keyboard.py
index 25c15a8a92..e154cf1239 100644
--- a/libs/python/cua-auto/cua_auto/keyboard.py
+++ b/libs/python/cua-auto/cua_auto/keyboard.py
@@ -47,11 +47,6 @@ _SPECIAL: dict = {
"option_r": _Key.alt_r,
"capslock": _Key.caps_lock,
"caps_lock": _Key.caps_lock,
- "insert": _Key.insert,
- "print_screen": _Key.print_screen,
- "pause": _Key.pause,
- "num_lock": _Key.num_lock,
- "scroll_lock": _Key.scroll_lock,
"f1": _Key.f1,
"f2": _Key.f2,
"f3": _Key.f3,
@@ -74,6 +69,18 @@ _SPECIAL: dict = {
"f20": _Key.f20,
}
+# Keys that may not exist on all platforms (e.g. macOS lacks insert, print_screen)
+for _name, _attr in [
+ ("insert", "insert"),
+ ("print_screen", "print_screen"),
+ ("pause", "pause"),
+ ("num_lock", "num_lock"),
+ ("scroll_lock", "scroll_lock"),
+]:
+ _val = getattr(_Key, _attr, None)
+ if _val is not None:
+ _SPECIAL[_name] = _val
+
def _resolve(key: str) -> Optional[Union[str, _Key]]:
"""Resolve a key string to a pynput Key or single character."""
diff --git a/libs/python/cua-auto/cua_auto/screen.py b/libs/python/cua-auto/cua_auto/screen.py
index e1c1988af5..d3f6739e47 100644
--- a/libs/python/cua-auto/cua_auto/screen.py
+++ b/libs/python/cua-auto/cua_auto/screen.py
@@ -2,14 +2,73 @@
import base64
import io
+import os
+import platform
from typing import Tuple
from PIL import Image
+def _get_display_scale() -> float:
+ """Detect the OS display scale factor (DPI scaling).
+
+ Returns the ratio of physical pixels to logical pixels:
+ - macOS Retina: typically 2.0
+ - Windows 150% DPI: 1.5
+ - Linux HiDPI (Wayland/X11): reads GDK_SCALE or QT_SCALE_FACTOR
+ - Falls back to 1.0 on failure or unknown platforms.
+ """
+ system = platform.system()
+
+ # macOS: use AppKit to query the main screen backing scale factor
+ if system == "Darwin":
+ try:
+ from AppKit import NSScreen # type: ignore[import-untyped]
+
+ return float(NSScreen.mainScreen().backingScaleFactor())
+ except Exception:
+ return 1.0
+
+ # Windows: use shcore to get the monitor scale factor
+ if system == "Windows":
+ try:
+ import ctypes
+
+ return ctypes.windll.shcore.GetScaleFactorForDevice(0) / 100.0 # type: ignore[attr-defined]
+ except Exception:
+ return 1.0
+
+ # Linux: check common env vars set by Wayland/X11 compositors and toolkits
+ if system == "Linux":
+ try:
+ gdk = os.environ.get("GDK_SCALE")
+ if gdk:
+ return float(gdk)
+ qt = os.environ.get("QT_SCALE_FACTOR")
+ if qt:
+ return float(qt)
+ except (ValueError, TypeError):
+ pass
+ return 1.0
+
+ return 1.0
+
+
+def get_display_scale() -> float:
+ """Return the OS display scale factor (public API).
+
+ This is a convenience wrapper around _get_display_scale() for use by
+ external code (e.g. MCP servers) that need to know the current DPI scale.
+ """
+ return _get_display_scale()
+
+
def screenshot() -> Image.Image:
"""Capture a screenshot of all monitors and return a PIL Image.
+ The returned image is normalized to logical pixels so that its coordinate
+ space matches pynput mouse input (which operates in logical coordinates).
+
Tries PIL.ImageGrab first (works on Windows and macOS).
Falls back to mss if ImageGrab fails (better Linux / multi-monitor support).
"""
@@ -19,6 +78,12 @@ def screenshot() -> Image.Image:
img = ImageGrab.grab(all_screens=True)
if isinstance(img, Image.Image):
+ scale = _get_display_scale()
+ if scale > 1.0:
+ img = img.resize(
+ (int(img.width / scale), int(img.height / scale)),
+ Image.LANCZOS,
+ )
return img
except Exception:
pass
@@ -31,7 +96,14 @@ def screenshot() -> Image.Image:
with mss.mss() as sct:
monitor = sct.monitors[0] # combined virtual desktop
sct_img = sct.grab(monitor)
- return Image.frombytes("RGB", sct_img.size, sct_img.bgra, "raw", "BGRX")
+ img = Image.frombytes("RGB", sct_img.size, sct_img.bgra, "raw", "BGRX")
+ scale = _get_display_scale()
+ if scale > 1.0:
+ img = img.resize(
+ (int(img.width / scale), int(img.height / scale)),
+ Image.LANCZOS,
+ )
+ return img
except Exception as e:
raise RuntimeError(f"screenshot failed: {e}") from e