Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
fix(computer-server): use typed desktop driver targets
Francesco Bonacci6e20fcc7e489
diff --git a/libs/python/computer-server/.bumpversion.cfg b/libs/python/computer-server/.bumpversion.cfgindex 9f8ca438db..d803b45448 100644--- a/libs/python/computer-server/.bumpversion.cfg+++ b/libs/python/computer-server/.bumpversion.cfg@@ -1,5 +1,5 @@[bumpversion]-current_version = 0.3.43+current_version = 0.3.44commit = Truetag = Truetag_name = computer-server-v{new_version}diff --git a/libs/python/computer-server/computer_server/handlers/cua_driver.py b/libs/python/computer-server/computer_server/handlers/cua_driver.pyindex 8db0221223..25ce755f12 100644--- a/libs/python/computer-server/computer_server/handlers/cua_driver.py+++ b/libs/python/computer-server/computer_server/handlers/cua_driver.py@@ -168,6 +168,20 @@ class CuaDriverAutomationHandler(BaseAutomationHandler):def capture_scope(self) -> str:return self._capture_scope+ def _desktop_target(self) -> tuple[Any, Optional[Any]]:+ """Build the typed desktop target used by the current SDK contract."""++ target_type = getattr(self._sdk, "ActionTarget", None)+ if target_type is None:+ # cua-driver 0.22.x keeps the generated type in its private module+ # while older package roots do not re-export it yet.+ contract = getattr(self._sdk, "_native_contract", None)+ target_type = getattr(contract, "ActionTarget", None)+ if target_type is None:+ # Preserve compatibility with lightweight injected SDK test doubles.+ return None, self._sdk.DesktopScope.DESKTOP+ return target_type.DESKTOP(display_id="primary"), None+async def _ensure_session(self) -> None:if self._closed:raise RuntimeError("Cua Driver automation handler is closed")@@ -349,12 +363,13 @@ class CuaDriverAutomationHandler(BaseAutomationHandler):try:await self._ensure_session()px, py = await self._point(x, y)+ target, scope = self._desktop_target()result = await self._driver.click(self._sdk.ClickInput(x=float(px),y=float(py),- target=None,- scope=self._sdk.DesktopScope.DESKTOP,+ target=target,+ scope=scope,session=self._session_id,button={"left": self._sdk.ClickButton.LEFT,@@ -387,12 +402,13 @@ class CuaDriverAutomationHandler(BaseAutomationHandler):async def move_cursor(self, x: int, y: int) -> Dict[str, Any]:try:await self._ensure_session()+ target, scope = self._desktop_target()result = await self._driver.move_cursor(self._sdk.MoveCursorInput(x=float(x),y=float(y),- target=None,- scope=self._sdk.DesktopScope.DESKTOP,+ target=target,+ scope=scope,session=self._session_id,))@@ -423,14 +439,15 @@ class CuaDriverAutomationHandler(BaseAutomationHandler):steps: Optional[int],) -> Dict[str, Any]:await self._ensure_session()+ target, scope = self._desktop_target()result = await self._driver.drag(self._sdk.DragInput(from_x=float(start_x),from_y=float(start_y),to_x=float(end_x),to_y=float(end_y),- target=None,- scope=self._sdk.DesktopScope.DESKTOP,+ target=target,+ scope=scope,session=self._session_id,duration_ms=min(10000, max(0, int(duration * 1000))),steps=steps,@@ -477,11 +494,12 @@ class CuaDriverAutomationHandler(BaseAutomationHandler):async def type_text(self, text: str) -> Dict[str, Any]:try:await self._ensure_session()+ target, scope = self._desktop_target()result = await self._driver.type_text(self._sdk.TypeTextInput(text=text,- target=None,- scope=self._sdk.DesktopScope.DESKTOP,+ target=target,+ scope=scope,session=self._session_id,))@@ -492,11 +510,12 @@ class CuaDriverAutomationHandler(BaseAutomationHandler):async def press_key(self, key: str) -> Dict[str, Any]:try:await self._ensure_session()+ target, scope = self._desktop_target()result = await self._driver.press_key(self._sdk.PressKeyInput(key=key,- target=None,- scope=self._sdk.DesktopScope.DESKTOP,+ target=target,+ scope=scope,session=self._session_id,modifiers=None,)@@ -512,11 +531,12 @@ class CuaDriverAutomationHandler(BaseAutomationHandler):return await self.press_key(keys[0])try:await self._ensure_session()+ target, scope = self._desktop_target()result = await self._driver.hotkey(self._sdk.HotkeyInput(keys=keys,- target=None,- scope=self._sdk.DesktopScope.DESKTOP,+ target=target,+ scope=scope,session=self._session_id,))@@ -528,6 +548,7 @@ class CuaDriverAutomationHandler(BaseAutomationHandler):try:await self._ensure_session()x, y = await self._point(None, None)+ target, scope = self._desktop_target()result = await self._driver.scroll(self._sdk.ScrollInput(x=float(x),@@ -538,8 +559,8 @@ class CuaDriverAutomationHandler(BaseAutomationHandler):"left": self._sdk.ScrollDirection.LEFT,"right": self._sdk.ScrollDirection.RIGHT,}[direction],- target=None,- scope=self._sdk.DesktopScope.DESKTOP,+ target=target,+ scope=scope,session=self._session_id,by=self._sdk.ScrollBy.LINE,amount=min(50, max(1, amount)),diff --git a/libs/python/computer-server/pyproject.toml b/libs/python/computer-server/pyproject.tomlindex 9865689802..e621e64961 100644--- a/libs/python/computer-server/pyproject.toml+++ b/libs/python/computer-server/pyproject.toml@@ -4,7 +4,7 @@ build-backend = "pdm.backend"[project]name = "cua-computer-server"-version = "0.3.43"+version = "0.3.44"description = "Server component for the Computer-Use Interface (CUI) framework powering Cua"authors = [diff --git a/libs/python/computer-server/tests/test_cua_driver_handler.py b/libs/python/computer-server/tests/test_cua_driver_handler.pyindex 79ed11318b..c3b59dcb90 100644--- a/libs/python/computer-server/tests/test_cua_driver_handler.py+++ b/libs/python/computer-server/tests/test_cua_driver_handler.py@@ -26,6 +26,12 @@ class _DesktopScope(Enum):DESKTOP = "desktop"+class _ActionTarget:+ class DESKTOP:+ def __init__(self, *, display_id):+ self.display_id = display_id++class _EffectiveScope(Enum):WINDOW = "window"DESKTOP = "desktop"@@ -194,6 +200,7 @@ def sdk():return SimpleNamespace(CaptureScope=_CaptureScope,DesktopScope=_DesktopScope,+ ActionTarget=_ActionTarget,EffectiveScope=_EffectiveScope,EscalationReason=_EscalationReason,ClickButton=_ClickButton,@@ -258,8 +265,8 @@ async def test_desktop_actions_share_one_typed_session(sdk, fallback):assert start.cursor_theme is Noneclicked = next(value for name, value in driver.calls if name == "click")assert (clicked.x, clicked.y) == (12.0, 34.0)- assert clicked.target is None- assert clicked.scope is _DesktopScope.DESKTOP+ assert clicked.target.display_id == "primary"+ assert clicked.scope is None@pytest.mark.asyncio@@ -326,7 +333,8 @@ async def test_adapter_uses_the_generated_python_contract(fallback):request = next(value for call, value in driver.calls if call == name)assert isinstance(request, expected_type)if hasattr(request, "target"):- assert request.target is None+ assert request.target.display_id == "primary"+ assert request.scope is None@pytest.mark.asyncio