Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
fix(cua-auto): normalize screenshots to logical pixels for DPI scaling (#1118)
Sarina Li909e6bffd6df
diff --git a/libs/python/cua-auto/cua_auto/keyboard.py b/libs/python/cua-auto/cua_auto/keyboard.pyindex 25c15a8a92..e154cf1239 100644--- a/libs/python/cua-auto/cua_auto/keyboard.py+++ b/libs/python/cua-auto/cua_auto/keyboard.py@@ -47,11 +47,6 @@ _SPECIAL: dict = {"option_r": _Key.alt_r,"capslock": _Key.caps_lock,"caps_lock": _Key.caps_lock,- "insert": _Key.insert,- "print_screen": _Key.print_screen,- "pause": _Key.pause,- "num_lock": _Key.num_lock,- "scroll_lock": _Key.scroll_lock,"f1": _Key.f1,"f2": _Key.f2,"f3": _Key.f3,@@ -74,6 +69,18 @@ _SPECIAL: dict = {"f20": _Key.f20,}+# Keys that may not exist on all platforms (e.g. macOS lacks insert, print_screen)+for _name, _attr in [+ ("insert", "insert"),+ ("print_screen", "print_screen"),+ ("pause", "pause"),+ ("num_lock", "num_lock"),+ ("scroll_lock", "scroll_lock"),+]:+ _val = getattr(_Key, _attr, None)+ if _val is not None:+ _SPECIAL[_name] = _val+def _resolve(key: str) -> Optional[Union[str, _Key]]:"""Resolve a key string to a pynput Key or single character."""diff --git a/libs/python/cua-auto/cua_auto/screen.py b/libs/python/cua-auto/cua_auto/screen.pyindex e1c1988af5..d3f6739e47 100644--- a/libs/python/cua-auto/cua_auto/screen.py+++ b/libs/python/cua-auto/cua_auto/screen.py@@ -2,14 +2,73 @@import base64import io+import os+import platformfrom typing import Tuplefrom PIL import Image+def _get_display_scale() -> float:+ """Detect the OS display scale factor (DPI scaling).++ Returns the ratio of physical pixels to logical pixels:+ - macOS Retina: typically 2.0+ - Windows 150% DPI: 1.5+ - Linux HiDPI (Wayland/X11): reads GDK_SCALE or QT_SCALE_FACTOR+ - Falls back to 1.0 on failure or unknown platforms.+ """+ system = platform.system()++ # macOS: use AppKit to query the main screen backing scale factor+ if system == "Darwin":+ try:+ from AppKit import NSScreen # type: ignore[import-untyped]++ return float(NSScreen.mainScreen().backingScaleFactor())+ except Exception:+ return 1.0++ # Windows: use shcore to get the monitor scale factor+ if system == "Windows":+ try:+ import ctypes++ return ctypes.windll.shcore.GetScaleFactorForDevice(0) / 100.0 # type: ignore[attr-defined]+ except Exception:+ return 1.0++ # Linux: check common env vars set by Wayland/X11 compositors and toolkits+ if system == "Linux":+ try:+ gdk = os.environ.get("GDK_SCALE")+ if gdk:+ return float(gdk)+ qt = os.environ.get("QT_SCALE_FACTOR")+ if qt:+ return float(qt)+ except (ValueError, TypeError):+ pass+ return 1.0++ return 1.0+++def get_display_scale() -> float:+ """Return the OS display scale factor (public API).++ This is a convenience wrapper around _get_display_scale() for use by+ external code (e.g. MCP servers) that need to know the current DPI scale.+ """+ return _get_display_scale()++def screenshot() -> Image.Image:"""Capture a screenshot of all monitors and return a PIL Image.+ The returned image is normalized to logical pixels so that its coordinate+ space matches pynput mouse input (which operates in logical coordinates).+Tries PIL.ImageGrab first (works on Windows and macOS).Falls back to mss if ImageGrab fails (better Linux / multi-monitor support)."""@@ -19,6 +78,12 @@ def screenshot() -> Image.Image:img = ImageGrab.grab(all_screens=True)if isinstance(img, Image.Image):+ scale = _get_display_scale()+ if scale > 1.0:+ img = img.resize(+ (int(img.width / scale), int(img.height / scale)),+ Image.LANCZOS,+ )return imgexcept Exception:pass@@ -31,7 +96,14 @@ def screenshot() -> Image.Image:with mss.mss() as sct:monitor = sct.monitors[0] # combined virtual desktopsct_img = sct.grab(monitor)- return Image.frombytes("RGB", sct_img.size, sct_img.bgra, "raw", "BGRX")+ img = Image.frombytes("RGB", sct_img.size, sct_img.bgra, "raw", "BGRX")+ scale = _get_display_scale()+ if scale > 1.0:+ img = img.resize(+ (int(img.width / scale), int(img.height / scale)),+ Image.LANCZOS,+ )+ return imgexcept Exception as e:raise RuntimeError(f"screenshot failed: {e}") from e