Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
Merge upstream/main to resolve conflicts with trycua/cua
Sarina Li5376afba7967
diff --cc libs/python/agent/agent/computers/base.pyindex 358fbbf426,51da978b41..673c6693e2--- a/libs/python/agent/agent/computers/base.py+++ b/libs/python/agent/agent/computers/base.py@@@ -18,15 -27,11 +27,15 @@@ class AsyncComputerHandler(Protocol)async def get_dimensions(self) -> tuple[int, int]:"""Get screen dimensions as (width, height)."""...-+- async def screenshot(self) -> str:- """Take a screenshot and return as base64 string."""+ async def screenshot(self, text: Optional[str] = None) -> str:+ """Take a screenshot and return as base64 string.++ Args:+ text: Optional descriptive text (for compatibility with GPT-4o models, ignored)+ """...-+async def click(self, x: int, y: int, button: str = "left") -> None:"""Click at coordinates with specified button."""...diff --cc libs/python/agent/agent/computers/cua.pyindex 4033795019,99723bda1e..00b0720f42--- a/libs/python/agent/agent/computers/cua.py+++ b/libs/python/agent/agent/computers/cua.py@@@ -32,17 -35,13 +35,17 @@@ class cuaComputerHandler(AsyncComputerHassert self.interface is not Nonescreen_size = await self.interface.get_screen_size()return screen_size["width"], screen_size["height"]-+- async def screenshot(self) -> str:- """Take a screenshot and return as base64 string."""+ async def screenshot(self, text: Optional[str] = None) -> str:+ """Take a screenshot and return as base64 string.++ Args:+ text: Optional descriptive text (for compatibility with GPT-4o models, ignored)+ """assert self.interface is not Nonescreenshot_bytes = await self.interface.screenshot()- return base64.b64encode(screenshot_bytes).decode('utf-8')-+ return base64.b64encode(screenshot_bytes).decode("utf-8")+async def click(self, x: int, y: int, button: str = "left") -> None:"""Click at coordinates with specified button."""assert self.interface is not Nonediff --cc libs/python/agent/agent/computers/custom.pyindex 5ab7d535f2,e87b151999..5165c497eb--- a/libs/python/agent/agent/computers/custom.py+++ b/libs/python/agent/agent/computers/custom.py@@@ -117,18 -119,14 +119,18 @@@ class CustomComputerHandler(AsyncComputif not self._last_screenshot_size:await self.screenshot()assert self._last_screenshot_size is not None, "Failed to get screenshot size"-+return self._last_screenshot_size-+- async def screenshot(self) -> str:- """Take a screenshot and return as base64 string."""+ async def screenshot(self, text: Optional[str] = None) -> str:+ """Take a screenshot and return as base64 string.++ Args:+ text: Optional descriptive text (for compatibility with GPT-4o models, ignored)+ """- result = await self._call_function(self.functions['screenshot'])- b64_str = self._to_b64_str(result) # type: ignore-+ result = await self._call_function(self.functions["screenshot"])+ b64_str = self._to_b64_str(result) # type: ignore+# Try to extract dimensions for fallback usetry:if isinstance(result, Image.Image):