Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
Added experiment flag for now
Dillon DuPont500465883ddd
diff --git a/libs/computer-server/computer_server/diorama/diorama.py b/libs/computer-server/computer_server/diorama/diorama.pyindex 7ee5ef8484..bf30a018b1 100644--- a/libs/computer-server/computer_server/diorama/diorama.py+++ b/libs/computer-server/computer_server/diorama/diorama.py@@ -97,6 +97,14 @@ class Diorama:await automation_handler.drag_to(x, y, duration=duration)if future:future.set_result(None)+ elif action in ["scroll_up", "scroll_down"]:+ clicks = args.get("clicks", 1)+ if action == "scroll_up":+ await automation_handler.scroll_up(clicks)+ else:+ await automation_handler.scroll_down(clicks)+ if future:+ future.set_result(None)# Keyboard actionselif action == "type_text":text = args.get("text")@@ -198,6 +206,12 @@ class Diorama:async def hotkey(self, keys):await self._send_cmd("hotkey", {"keys": list(keys)})+ async def scroll_up(self, clicks: int = 1):+ await self._send_cmd("scroll_up", {"clicks": clicks})++ async def scroll_down(self, clicks: int = 1):+ await self._send_cmd("scroll_down", {"clicks": clicks})+async def get_screen_size(self) -> dict[str, int]:if not self._scene_size:await self.screenshot()diff --git a/libs/computer-server/computer_server/diorama/macos.py b/libs/computer-server/computer_server/diorama/macos.pyindex 6ced04d722..be266caef6 100644--- a/libs/computer-server/computer_server/diorama/macos.py+++ b/libs/computer-server/computer_server/diorama/macos.py@@ -4,10 +4,11 @@ import platformimport inspectfrom computer_server.diorama.diorama import Dioramafrom computer_server.diorama.base import BaseDioramaHandler+from typing import Optionalclass MacOSDioramaHandler(BaseDioramaHandler):"""Handler for Diorama commands on macOS, using local diorama module."""- async def diorama_cmd(self, action: str, arguments: dict = None) -> dict:+ async def diorama_cmd(self, action: str, arguments: Optional[dict] = None) -> dict:if platform.system().lower() != "darwin":return {"success": False, "error": "Diorama is only supported on macOS."}try:diff --git a/libs/computer-server/computer_server/handlers/factory.py b/libs/computer-server/computer_server/handlers/factory.pyindex fecd9bbcca..4c076b8011 100644--- a/libs/computer-server/computer_server/handlers/factory.py+++ b/libs/computer-server/computer_server/handlers/factory.py@@ -55,7 +55,7 @@ class HandlerFactory:if os_type == 'darwin':return MacOSAccessibilityHandler(), MacOSAutomationHandler(), MacOSDioramaHandler()- else:+ elif os_type == 'linux':return LinuxAccessibilityHandler(), LinuxAutomationHandler(), BaseDioramaHandler()-+ else:raise NotImplementedError(f"OS '{os_type}' is not supported")\ No newline at end of filediff --git a/libs/computer/computer/computer.py b/libs/computer/computer/computer.pyindex b20922ce76..a1868cc40b 100644--- a/libs/computer/computer/computer.py+++ b/libs/computer/computer/computer.py@@ -31,6 +31,7 @@ class Computer:Returns:DioramaComputer: A proxy object with the Diorama interface, but using diorama_cmds."""+ assert "app-use" in self.experiments, "App Usage is an experimental feature. Enable it by passing experiments=['app-use'] to Computer()"from .diorama_computer import DioramaComputerreturn DioramaComputer(self, apps)@@ -52,7 +53,8 @@ class Computer:host: str = os.environ.get("PYLUME_HOST", "localhost"),storage: Optional[str] = None,ephemeral: bool = False,- api_key: Optional[str] = None+ api_key: Optional[str] = None,+ experiments: Optional[List[str]] = None):"""Initialize a new Computer instance.@@ -78,6 +80,8 @@ class Computer:host: Host to use for VM provider connections (e.g. "localhost", "host.docker.internal")storage: Optional path for persistent VM storage (Lumier provider)ephemeral: Whether to use ephemeral storage+ api_key: Optional API key for cloud providers+ experiments: Optional list of experimental features to enable (e.g. ["app-use"])"""self.logger = Logger("cua.computer", verbosity)@@ -93,6 +97,7 @@ class Computer:self.ephemeral = ephemeralself.api_key = api_key+ self.experiments = experiments or []# The default is currently to use non-ephemeral storageif storage and ephemeral and storage != "ephemeral":diff --git a/libs/computer/computer/diorama_computer.py b/libs/computer/computer/diorama_computer.pyindex 5e37af45da..608e6721b7 100644--- a/libs/computer/computer/diorama_computer.py+++ b/libs/computer/computer/diorama_computer.py@@ -26,7 +26,6 @@ class DioramaComputerInterface:def __init__(self, computer, apps):self.computer = computerself.apps = apps- self._scene_hitboxes = []self._scene_size = Noneasync def _send_cmd(self, action, arguments=None):@@ -43,11 +42,10 @@ class DioramaComputerInterface:async def screenshot(self, as_bytes=True):from PIL import Image+ import base64result = await self._send_cmd("screenshot")- img_bytes = result.get("image_bytes")- hitboxes = result.get("hitboxes", [])- self._scene_hitboxes = hitboxes- # Assume server returns PNG bytes+ # assume result is a b64 string of an image+ img_bytes = base64.b64decode(result)import ioimg = Image.open(io.BytesIO(img_bytes))self._scene_size = img.size@@ -70,6 +68,12 @@ class DioramaComputerInterface:async def double_click(self, x=None, y=None):await self._send_cmd("double_click", {"x": x, "y": y})+ async def scroll_up(self, clicks=1):+ await self._send_cmd("scroll_up", {"clicks": clicks})++ async def scroll_down(self, clicks=1):+ await self._send_cmd("scroll_down", {"clicks": clicks})+async def drag_to(self, x, y, duration=0.5):await self._send_cmd("drag_to", {"x": x, "y": y, "duration": duration})diff --git a/libs/computer/computer/interface/macos.py b/libs/computer/computer/interface/macos.pyindex cb0deed159..a96c44d123 100644--- a/libs/computer/computer/interface/macos.py+++ b/libs/computer/computer/interface/macos.py@@ -346,7 +346,7 @@ class MacOSComputerInterface(BaseComputerInterface):asyncio.create_task(self._ws.close())self._ws = None- async def diorama_cmd(self, action: str, arguments: dict = None) -> dict:+ async def diorama_cmd(self, action: str, arguments: Optional[dict] = None) -> dict:"""Send a diorama command to the server (macOS only)."""return await self._send_command("diorama_cmd", {"action": action, "arguments": arguments or {}})