Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
feat(computer-server): add JPEG format option to screenshot() (#1201)
ddupontd0308d31b194
diff --git a/libs/python/computer-server/computer_server/handlers/android.py b/libs/python/computer-server/computer_server/handlers/android.pyindex bc25f5b13d..b74d3ee1e2 100644--- a/libs/python/computer-server/computer_server/handlers/android.py+++ b/libs/python/computer-server/computer_server/handlers/android.py@@ -10,6 +10,7 @@ from .base import (BaseDesktopHandler,BaseFileHandler,BaseWindowHandler,+ normalize_screenshot_format,)# Map common key names to Android keycodes@@ -439,16 +440,35 @@ class AndroidAutomationHandler(BaseAutomationHandler):return {}# Screen Actions- async def screenshot(self) -> Dict[str, Any]:- """Take a screenshot and return base64 encoded image."""+ async def screenshot(self, format: str = "png", quality: int = 95) -> Dict[str, Any]:+ """Take a screenshot and return base64 encoded image.++ Args:+ format: "png" (lossless, default) or "jpeg" / "jpg" (lossy, ~5-10x smaller).+ quality: JPEG quality 1-95 (clamped); ignored for PNG.+ """+ try:+ fmt, quality = normalize_screenshot_format(format, quality)+ except ValueError as e:+ return {"success": False, "error": str(e)}try:success, output = await adb_exec.run("shell", "screencap", "-p")- if success and output:- image_b64 = base64.b64encode(output).decode("utf-8")- return {"success": True, "image_data": image_b64}- else:+ if not (success and output):error_msg = output.decode("utf-8") if isinstance(output, bytes) else str(output)return {"success": False, "error": f"Screenshot failed: {error_msg}"}++ if fmt == "jpeg":+ from io import BytesIO++ from PIL import Image as PILImage++ img = PILImage.open(BytesIO(output)).convert("RGB")+ buf = BytesIO()+ img.save(buf, format="JPEG", quality=quality, optimize=True)+ output = buf.getvalue()++ image_b64 = base64.b64encode(output).decode("utf-8")+ return {"success": True, "image_data": image_b64, "format": fmt}except Exception as e:return {"success": False, "error": f"Screenshot error: {str(e)}"}diff --git a/libs/python/computer-server/computer_server/handlers/base.py b/libs/python/computer-server/computer_server/handlers/base.pyindex 833b21b416..a555e0e528 100644--- a/libs/python/computer-server/computer_server/handlers/base.py+++ b/libs/python/computer-server/computer_server/handlers/base.py@@ -186,6 +186,30 @@ class BaseWindowHandler(ABC):pass+_SUPPORTED_FORMATS = {"png", "jpeg"}+_FORMAT_ALIASES = {"jpg": "jpeg"}+++def normalize_screenshot_format(format: str, quality: int) -> tuple[str, int]:+ """Normalize and validate screenshot format/quality.++ Returns ``(normalized_format, clamped_quality)`` or raises ``ValueError``.++ - Lowercases the format string.+ - Maps "jpg" → "jpeg".+ - Rejects anything not in ``{"png", "jpeg"}``.+ - Clamps JPEG quality to 1–95 (silently caps values above 95).+ """+ fmt = _FORMAT_ALIASES.get(format.lower(), format.lower())+ if fmt not in _SUPPORTED_FORMATS:+ raise ValueError(+ f"Unsupported screenshot format {format!r}. " f"Supported: {sorted(_SUPPORTED_FORMATS)}"+ )+ if fmt == "jpeg":+ quality = max(1, min(95, quality))+ return fmt, quality++class BaseAutomationHandler(ABC):"""Abstract base class for OS-specific automation handlers.@@ -312,8 +336,13 @@ class BaseAutomationHandler(ABC):# Screen Actions@abstractmethod- async def screenshot(self) -> Dict[str, Any]:- """Take a screenshot and return base64 encoded image data."""+ async def screenshot(self, format: str = "png", quality: int = 95) -> Dict[str, Any]:+ """Take a screenshot and return base64 encoded image data.++ Args:+ format: Image format - "png" (lossless, default) or "jpeg" (lossy, smaller).+ quality: JPEG quality 1-95, ignored for PNG.+ """pass@abstractmethoddiff --git a/libs/python/computer-server/computer_server/handlers/linux.py b/libs/python/computer-server/computer_server/handlers/linux.pyindex 261189ede3..948fc79db0 100644--- a/libs/python/computer-server/computer_server/handlers/linux.py+++ b/libs/python/computer-server/computer_server/handlers/linux.py@@ -28,7 +28,11 @@ from pynput.keyboard import Keyfrom pynput.mouse import Buttonfrom pynput.mouse import Controller as MouseController-from .base import BaseAccessibilityHandler, BaseAutomationHandler+from .base import (+ BaseAccessibilityHandler,+ BaseAutomationHandler,+ normalize_screenshot_format,+)class LinuxAccessibilityHandler(BaseAccessibilityHandler):@@ -489,22 +493,31 @@ class LinuxAutomationHandler(BaseAutomationHandler):return {"success": False, "error": str(e)}# Screen Actions- async def screenshot(self) -> Dict[str, Any]:+ async def screenshot(self, format: str = "png", quality: int = 95) -> Dict[str, Any]:"""Take a screenshot of the current screen.- Returns:- Dict[str, Any]: A dictionary containing success status and base64-encoded image data,- or error message if failed.+ Args:+ format: "png" (lossless, default), "jpeg" or "jpg" (lossy, smaller).+ quality: JPEG quality 1-95 (clamped); ignored for PNG."""+ try:+ fmt, quality = normalize_screenshot_format(format, quality)+ except ValueError as e:+ return {"success": False, "error": str(e)}try:screenshot = ImageGrab.grab()if not isinstance(screenshot, Image.Image):return {"success": False, "error": "Failed to capture screenshot"}buffered = BytesIO()- screenshot.save(buffered, format="PNG", optimize=True)+ if fmt == "jpeg":+ screenshot.convert("RGB").save(+ buffered, format="JPEG", quality=quality, optimize=True+ )+ else:+ screenshot.save(buffered, format="PNG", optimize=True)buffered.seek(0)image_data = base64.b64encode(buffered.getvalue()).decode()- return {"success": True, "image_data": image_data}+ return {"success": True, "image_data": image_data, "format": fmt}except Exception as e:return {"success": False, "error": f"Screenshot error: {str(e)}"}diff --git a/libs/python/computer-server/computer_server/handlers/macos.py b/libs/python/computer-server/computer_server/handlers/macos.pyindex 3846508b33..146f282641 100644--- a/libs/python/computer-server/computer_server/handlers/macos.py+++ b/libs/python/computer-server/computer_server/handlers/macos.py@@ -49,7 +49,11 @@ from pynput.mouse import Controller as MouseControllerfrom Quartz.CoreGraphics import * # type: ignorefrom Quartz.CoreGraphics import CGPoint, CGSize # type: ignore-from .base import BaseAccessibilityHandler, BaseAutomationHandler+from .base import (+ BaseAccessibilityHandler,+ BaseAutomationHandler,+ normalize_screenshot_format,+)logger = logging.getLogger(__name__)@@ -1357,12 +1361,17 @@ class MacOSAutomationHandler(BaseAutomationHandler):return {"success": False, "error": str(e)}# Screen Actions- async def screenshot(self) -> Dict[str, Any]:+ async def screenshot(self, format: str = "png", quality: int = 95) -> Dict[str, Any]:"""Capture a screenshot of the current screen.- Returns:- Dictionary containing success status and base64-encoded image data or error message+ Args:+ format: "png" (lossless, default), "jpeg" or "jpg" (lossy, smaller).+ quality: JPEG quality 1-95 (clamped); ignored for PNG."""+ try:+ fmt, quality = normalize_screenshot_format(format, quality)+ except ValueError as e:+ return {"success": False, "error": str(e)}try:screenshot = ImageGrab.grab()if not isinstance(screenshot, Image.Image):@@ -1376,11 +1385,15 @@ class MacOSAutomationHandler(BaseAutomationHandler):screenshot = screenshot.resize((max_width, new_height), Image.Resampling.LANCZOS)buffered = BytesIO()- # Use PNG format with optimization to reduce file size- screenshot.save(buffered, format="PNG", optimize=True)+ if fmt == "jpeg":+ screenshot.convert("RGB").save(+ buffered, format="JPEG", quality=quality, optimize=True+ )+ else:+ screenshot.save(buffered, format="PNG", optimize=True)buffered.seek(0)image_data = base64.b64encode(buffered.getvalue()).decode()- return {"success": True, "image_data": image_data}+ return {"success": True, "image_data": image_data, "format": fmt}except Exception as e:return {"success": False, "error": f"Screenshot error: {str(e)}"}diff --git a/libs/python/computer-server/computer_server/handlers/windows.py b/libs/python/computer-server/computer_server/handlers/windows.pyindex c4af537a84..f42749f1a9 100644--- a/libs/python/computer-server/computer_server/handlers/windows.py+++ b/libs/python/computer-server/computer_server/handlers/windows.py@@ -63,7 +63,11 @@ except Exception as e:)WINDOWS_API_AVAILABLE = False-from .base import BaseAccessibilityHandler, BaseAutomationHandler+from .base import (+ BaseAccessibilityHandler,+ BaseAutomationHandler,+ normalize_screenshot_format,+)class WindowsAccessibilityHandler(BaseAccessibilityHandler):@@ -632,25 +636,32 @@ class WindowsAutomationHandler(BaseAutomationHandler):# Screen Actions@require_unlocked_desktop- async def screenshot(self) -> Dict[str, Any]:+ async def screenshot(self, format: str = "png", quality: int = 95) -> Dict[str, Any]:"""Capture a screenshot of the entire screen.- Returns:- Dict[str, Any]: A dictionary containing the success status and either- base64-encoded image data or an error message.- Structure: {"success": bool, "image_data": str} or- {"success": bool, "error": str}+ Args:+ format: "png" (lossless, default), "jpeg" or "jpg" (lossy, smaller).+ quality: JPEG quality 1-95 (clamped); ignored for PNG."""+ try:+ fmt, quality = normalize_screenshot_format(format, quality)+ except ValueError as e:+ return {"success": False, "error": str(e)}try:screenshot = ImageGrab.grab()if not isinstance(screenshot, Image.Image):return {"success": False, "error": "Failed to capture screenshot"}buffered = BytesIO()- screenshot.save(buffered, format="PNG", optimize=True)+ if fmt == "jpeg":+ screenshot.convert("RGB").save(+ buffered, format="JPEG", quality=quality, optimize=True+ )+ else:+ screenshot.save(buffered, format="PNG", optimize=True)buffered.seek(0)image_data = base64.b64encode(buffered.getvalue()).decode()- return {"success": True, "image_data": image_data}+ return {"success": True, "image_data": image_data, "format": fmt}except Exception as e:return {"success": False, "error": f"Screenshot error: {str(e)}"}