Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

feat(computer-server): add JPEG format option to screenshot() (#1201)

ddupontd0308d31b194
diff --git a/libs/python/computer-server/computer_server/handlers/android.py b/libs/python/computer-server/computer_server/handlers/android.py
index bc25f5b13d..b74d3ee1e2 100644
--- a/libs/python/computer-server/computer_server/handlers/android.py
+++ b/libs/python/computer-server/computer_server/handlers/android.py
@@ -10,6 +10,7 @@ from .base import (
BaseDesktopHandler,
BaseFileHandler,
BaseWindowHandler,
+ normalize_screenshot_format,
)
# Map common key names to Android keycodes
@@ -439,16 +440,35 @@ class AndroidAutomationHandler(BaseAutomationHandler):
return {}
# Screen Actions
- async def screenshot(self) -> Dict[str, Any]:
- """Take a screenshot and return base64 encoded image."""
+ async def screenshot(self, format: str = "png", quality: int = 95) -> Dict[str, Any]:
+ """Take a screenshot and return base64 encoded image.
+
+ Args:
+ format: "png" (lossless, default) or "jpeg" / "jpg" (lossy, ~5-10x smaller).
+ quality: JPEG quality 1-95 (clamped); ignored for PNG.
+ """
+ try:
+ fmt, quality = normalize_screenshot_format(format, quality)
+ except ValueError as e:
+ return {"success": False, "error": str(e)}
try:
success, output = await adb_exec.run("shell", "screencap", "-p")
- if success and output:
- image_b64 = base64.b64encode(output).decode("utf-8")
- return {"success": True, "image_data": image_b64}
- else:
+ if not (success and output):
error_msg = output.decode("utf-8") if isinstance(output, bytes) else str(output)
return {"success": False, "error": f"Screenshot failed: {error_msg}"}
+
+ if fmt == "jpeg":
+ from io import BytesIO
+
+ from PIL import Image as PILImage
+
+ img = PILImage.open(BytesIO(output)).convert("RGB")
+ buf = BytesIO()
+ img.save(buf, format="JPEG", quality=quality, optimize=True)
+ output = buf.getvalue()
+
+ image_b64 = base64.b64encode(output).decode("utf-8")
+ return {"success": True, "image_data": image_b64, "format": fmt}
except Exception as e:
return {"success": False, "error": f"Screenshot error: {str(e)}"}
diff --git a/libs/python/computer-server/computer_server/handlers/base.py b/libs/python/computer-server/computer_server/handlers/base.py
index 833b21b416..a555e0e528 100644
--- a/libs/python/computer-server/computer_server/handlers/base.py
+++ b/libs/python/computer-server/computer_server/handlers/base.py
@@ -186,6 +186,30 @@ class BaseWindowHandler(ABC):
pass
+_SUPPORTED_FORMATS = {"png", "jpeg"}
+_FORMAT_ALIASES = {"jpg": "jpeg"}
+
+
+def normalize_screenshot_format(format: str, quality: int) -> tuple[str, int]:
+ """Normalize and validate screenshot format/quality.
+
+ Returns ``(normalized_format, clamped_quality)`` or raises ``ValueError``.
+
+ - Lowercases the format string.
+ - Maps "jpg" → "jpeg".
+ - Rejects anything not in ``{"png", "jpeg"}``.
+ - Clamps JPEG quality to 1–95 (silently caps values above 95).
+ """
+ fmt = _FORMAT_ALIASES.get(format.lower(), format.lower())
+ if fmt not in _SUPPORTED_FORMATS:
+ raise ValueError(
+ f"Unsupported screenshot format {format!r}. " f"Supported: {sorted(_SUPPORTED_FORMATS)}"
+ )
+ if fmt == "jpeg":
+ quality = max(1, min(95, quality))
+ return fmt, quality
+
+
class BaseAutomationHandler(ABC):
"""Abstract base class for OS-specific automation handlers.
@@ -312,8 +336,13 @@ class BaseAutomationHandler(ABC):
# Screen Actions
@abstractmethod
- async def screenshot(self) -> Dict[str, Any]:
- """Take a screenshot and return base64 encoded image data."""
+ async def screenshot(self, format: str = "png", quality: int = 95) -> Dict[str, Any]:
+ """Take a screenshot and return base64 encoded image data.
+
+ Args:
+ format: Image format - "png" (lossless, default) or "jpeg" (lossy, smaller).
+ quality: JPEG quality 1-95, ignored for PNG.
+ """
pass
@abstractmethod
diff --git a/libs/python/computer-server/computer_server/handlers/linux.py b/libs/python/computer-server/computer_server/handlers/linux.py
index 261189ede3..948fc79db0 100644
--- a/libs/python/computer-server/computer_server/handlers/linux.py
+++ b/libs/python/computer-server/computer_server/handlers/linux.py
@@ -28,7 +28,11 @@ from pynput.keyboard import Key
from pynput.mouse import Button
from pynput.mouse import Controller as MouseController
-from .base import BaseAccessibilityHandler, BaseAutomationHandler
+from .base import (
+ BaseAccessibilityHandler,
+ BaseAutomationHandler,
+ normalize_screenshot_format,
+)
class LinuxAccessibilityHandler(BaseAccessibilityHandler):
@@ -489,22 +493,31 @@ class LinuxAutomationHandler(BaseAutomationHandler):
return {"success": False, "error": str(e)}
# Screen Actions
- async def screenshot(self) -> Dict[str, Any]:
+ async def screenshot(self, format: str = "png", quality: int = 95) -> Dict[str, Any]:
"""Take a screenshot of the current screen.
- Returns:
- Dict[str, Any]: A dictionary containing success status and base64-encoded image data,
- or error message if failed.
+ Args:
+ format: "png" (lossless, default), "jpeg" or "jpg" (lossy, smaller).
+ quality: JPEG quality 1-95 (clamped); ignored for PNG.
"""
+ try:
+ fmt, quality = normalize_screenshot_format(format, quality)
+ except ValueError as e:
+ return {"success": False, "error": str(e)}
try:
screenshot = ImageGrab.grab()
if not isinstance(screenshot, Image.Image):
return {"success": False, "error": "Failed to capture screenshot"}
buffered = BytesIO()
- screenshot.save(buffered, format="PNG", optimize=True)
+ if fmt == "jpeg":
+ screenshot.convert("RGB").save(
+ buffered, format="JPEG", quality=quality, optimize=True
+ )
+ else:
+ screenshot.save(buffered, format="PNG", optimize=True)
buffered.seek(0)
image_data = base64.b64encode(buffered.getvalue()).decode()
- return {"success": True, "image_data": image_data}
+ return {"success": True, "image_data": image_data, "format": fmt}
except Exception as e:
return {"success": False, "error": f"Screenshot error: {str(e)}"}
diff --git a/libs/python/computer-server/computer_server/handlers/macos.py b/libs/python/computer-server/computer_server/handlers/macos.py
index 3846508b33..146f282641 100644
--- a/libs/python/computer-server/computer_server/handlers/macos.py
+++ b/libs/python/computer-server/computer_server/handlers/macos.py
@@ -49,7 +49,11 @@ from pynput.mouse import Controller as MouseController
from Quartz.CoreGraphics import * # type: ignore
from Quartz.CoreGraphics import CGPoint, CGSize # type: ignore
-from .base import BaseAccessibilityHandler, BaseAutomationHandler
+from .base import (
+ BaseAccessibilityHandler,
+ BaseAutomationHandler,
+ normalize_screenshot_format,
+)
logger = logging.getLogger(__name__)
@@ -1357,12 +1361,17 @@ class MacOSAutomationHandler(BaseAutomationHandler):
return {"success": False, "error": str(e)}
# Screen Actions
- async def screenshot(self) -> Dict[str, Any]:
+ async def screenshot(self, format: str = "png", quality: int = 95) -> Dict[str, Any]:
"""Capture a screenshot of the current screen.
- Returns:
- Dictionary containing success status and base64-encoded image data or error message
+ Args:
+ format: "png" (lossless, default), "jpeg" or "jpg" (lossy, smaller).
+ quality: JPEG quality 1-95 (clamped); ignored for PNG.
"""
+ try:
+ fmt, quality = normalize_screenshot_format(format, quality)
+ except ValueError as e:
+ return {"success": False, "error": str(e)}
try:
screenshot = ImageGrab.grab()
if not isinstance(screenshot, Image.Image):
@@ -1376,11 +1385,15 @@ class MacOSAutomationHandler(BaseAutomationHandler):
screenshot = screenshot.resize((max_width, new_height), Image.Resampling.LANCZOS)
buffered = BytesIO()
- # Use PNG format with optimization to reduce file size
- screenshot.save(buffered, format="PNG", optimize=True)
+ if fmt == "jpeg":
+ screenshot.convert("RGB").save(
+ buffered, format="JPEG", quality=quality, optimize=True
+ )
+ else:
+ screenshot.save(buffered, format="PNG", optimize=True)
buffered.seek(0)
image_data = base64.b64encode(buffered.getvalue()).decode()
- return {"success": True, "image_data": image_data}
+ return {"success": True, "image_data": image_data, "format": fmt}
except Exception as e:
return {"success": False, "error": f"Screenshot error: {str(e)}"}
diff --git a/libs/python/computer-server/computer_server/handlers/windows.py b/libs/python/computer-server/computer_server/handlers/windows.py
index c4af537a84..f42749f1a9 100644
--- a/libs/python/computer-server/computer_server/handlers/windows.py
+++ b/libs/python/computer-server/computer_server/handlers/windows.py
@@ -63,7 +63,11 @@ except Exception as e:
)
WINDOWS_API_AVAILABLE = False
-from .base import BaseAccessibilityHandler, BaseAutomationHandler
+from .base import (
+ BaseAccessibilityHandler,
+ BaseAutomationHandler,
+ normalize_screenshot_format,
+)
class WindowsAccessibilityHandler(BaseAccessibilityHandler):
@@ -632,25 +636,32 @@ class WindowsAutomationHandler(BaseAutomationHandler):
# Screen Actions
@require_unlocked_desktop
- async def screenshot(self) -> Dict[str, Any]:
+ async def screenshot(self, format: str = "png", quality: int = 95) -> Dict[str, Any]:
"""Capture a screenshot of the entire screen.
- Returns:
- Dict[str, Any]: A dictionary containing the success status and either
- base64-encoded image data or an error message.
- Structure: {"success": bool, "image_data": str} or
- {"success": bool, "error": str}
+ Args:
+ format: "png" (lossless, default), "jpeg" or "jpg" (lossy, smaller).
+ quality: JPEG quality 1-95 (clamped); ignored for PNG.
"""
+ try:
+ fmt, quality = normalize_screenshot_format(format, quality)
+ except ValueError as e:
+ return {"success": False, "error": str(e)}
try:
screenshot = ImageGrab.grab()
if not isinstance(screenshot, Image.Image):
return {"success": False, "error": "Failed to capture screenshot"}
buffered = BytesIO()
- screenshot.save(buffered, format="PNG", optimize=True)
+ if fmt == "jpeg":
+ screenshot.convert("RGB").save(
+ buffered, format="JPEG", quality=quality, optimize=True
+ )
+ else:
+ screenshot.save(buffered, format="PNG", optimize=True)
buffered.seek(0)
image_data = base64.b64encode(buffered.getvalue()).decode()
- return {"success": True, "image_data": image_data}
+ return {"success": True, "image_data": image_data, "format": fmt}
except Exception as e:
return {"success": False, "error": f"Screenshot error: {str(e)}"}