Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

fix(computer-server): use typed desktop driver targets

Francesco Bonacci6e20fcc7e489
diff --git a/libs/python/computer-server/.bumpversion.cfg b/libs/python/computer-server/.bumpversion.cfg
index 9f8ca438db..d803b45448 100644
--- a/libs/python/computer-server/.bumpversion.cfg
+++ b/libs/python/computer-server/.bumpversion.cfg
@@ -1,5 +1,5 @@
[bumpversion]
-current_version = 0.3.43
+current_version = 0.3.44
commit = True
tag = True
tag_name = computer-server-v{new_version}
diff --git a/libs/python/computer-server/computer_server/handlers/cua_driver.py b/libs/python/computer-server/computer_server/handlers/cua_driver.py
index 8db0221223..25ce755f12 100644
--- a/libs/python/computer-server/computer_server/handlers/cua_driver.py
+++ b/libs/python/computer-server/computer_server/handlers/cua_driver.py
@@ -168,6 +168,20 @@ class CuaDriverAutomationHandler(BaseAutomationHandler):
def capture_scope(self) -> str:
return self._capture_scope
+ def _desktop_target(self) -> tuple[Any, Optional[Any]]:
+ """Build the typed desktop target used by the current SDK contract."""
+
+ target_type = getattr(self._sdk, "ActionTarget", None)
+ if target_type is None:
+ # cua-driver 0.22.x keeps the generated type in its private module
+ # while older package roots do not re-export it yet.
+ contract = getattr(self._sdk, "_native_contract", None)
+ target_type = getattr(contract, "ActionTarget", None)
+ if target_type is None:
+ # Preserve compatibility with lightweight injected SDK test doubles.
+ return None, self._sdk.DesktopScope.DESKTOP
+ return target_type.DESKTOP(display_id="primary"), None
+
async def _ensure_session(self) -> None:
if self._closed:
raise RuntimeError("Cua Driver automation handler is closed")
@@ -349,12 +363,13 @@ class CuaDriverAutomationHandler(BaseAutomationHandler):
try:
await self._ensure_session()
px, py = await self._point(x, y)
+ target, scope = self._desktop_target()
result = await self._driver.click(
self._sdk.ClickInput(
x=float(px),
y=float(py),
- target=None,
- scope=self._sdk.DesktopScope.DESKTOP,
+ target=target,
+ scope=scope,
session=self._session_id,
button={
"left": self._sdk.ClickButton.LEFT,
@@ -387,12 +402,13 @@ class CuaDriverAutomationHandler(BaseAutomationHandler):
async def move_cursor(self, x: int, y: int) -> Dict[str, Any]:
try:
await self._ensure_session()
+ target, scope = self._desktop_target()
result = await self._driver.move_cursor(
self._sdk.MoveCursorInput(
x=float(x),
y=float(y),
- target=None,
- scope=self._sdk.DesktopScope.DESKTOP,
+ target=target,
+ scope=scope,
session=self._session_id,
)
)
@@ -423,14 +439,15 @@ class CuaDriverAutomationHandler(BaseAutomationHandler):
steps: Optional[int],
) -> Dict[str, Any]:
await self._ensure_session()
+ target, scope = self._desktop_target()
result = await self._driver.drag(
self._sdk.DragInput(
from_x=float(start_x),
from_y=float(start_y),
to_x=float(end_x),
to_y=float(end_y),
- target=None,
- scope=self._sdk.DesktopScope.DESKTOP,
+ target=target,
+ scope=scope,
session=self._session_id,
duration_ms=min(10000, max(0, int(duration * 1000))),
steps=steps,
@@ -477,11 +494,12 @@ class CuaDriverAutomationHandler(BaseAutomationHandler):
async def type_text(self, text: str) -> Dict[str, Any]:
try:
await self._ensure_session()
+ target, scope = self._desktop_target()
result = await self._driver.type_text(
self._sdk.TypeTextInput(
text=text,
- target=None,
- scope=self._sdk.DesktopScope.DESKTOP,
+ target=target,
+ scope=scope,
session=self._session_id,
)
)
@@ -492,11 +510,12 @@ class CuaDriverAutomationHandler(BaseAutomationHandler):
async def press_key(self, key: str) -> Dict[str, Any]:
try:
await self._ensure_session()
+ target, scope = self._desktop_target()
result = await self._driver.press_key(
self._sdk.PressKeyInput(
key=key,
- target=None,
- scope=self._sdk.DesktopScope.DESKTOP,
+ target=target,
+ scope=scope,
session=self._session_id,
modifiers=None,
)
@@ -512,11 +531,12 @@ class CuaDriverAutomationHandler(BaseAutomationHandler):
return await self.press_key(keys[0])
try:
await self._ensure_session()
+ target, scope = self._desktop_target()
result = await self._driver.hotkey(
self._sdk.HotkeyInput(
keys=keys,
- target=None,
- scope=self._sdk.DesktopScope.DESKTOP,
+ target=target,
+ scope=scope,
session=self._session_id,
)
)
@@ -528,6 +548,7 @@ class CuaDriverAutomationHandler(BaseAutomationHandler):
try:
await self._ensure_session()
x, y = await self._point(None, None)
+ target, scope = self._desktop_target()
result = await self._driver.scroll(
self._sdk.ScrollInput(
x=float(x),
@@ -538,8 +559,8 @@ class CuaDriverAutomationHandler(BaseAutomationHandler):
"left": self._sdk.ScrollDirection.LEFT,
"right": self._sdk.ScrollDirection.RIGHT,
}[direction],
- target=None,
- scope=self._sdk.DesktopScope.DESKTOP,
+ target=target,
+ scope=scope,
session=self._session_id,
by=self._sdk.ScrollBy.LINE,
amount=min(50, max(1, amount)),
diff --git a/libs/python/computer-server/pyproject.toml b/libs/python/computer-server/pyproject.toml
index 9865689802..e621e64961 100644
--- a/libs/python/computer-server/pyproject.toml
+++ b/libs/python/computer-server/pyproject.toml
@@ -4,7 +4,7 @@ build-backend = "pdm.backend"
[project]
name = "cua-computer-server"
-version = "0.3.43"
+version = "0.3.44"
description = "Server component for the Computer-Use Interface (CUI) framework powering Cua"
authors = [
diff --git a/libs/python/computer-server/tests/test_cua_driver_handler.py b/libs/python/computer-server/tests/test_cua_driver_handler.py
index 79ed11318b..c3b59dcb90 100644
--- a/libs/python/computer-server/tests/test_cua_driver_handler.py
+++ b/libs/python/computer-server/tests/test_cua_driver_handler.py
@@ -26,6 +26,12 @@ class _DesktopScope(Enum):
DESKTOP = "desktop"
+class _ActionTarget:
+ class DESKTOP:
+ def __init__(self, *, display_id):
+ self.display_id = display_id
+
+
class _EffectiveScope(Enum):
WINDOW = "window"
DESKTOP = "desktop"
@@ -194,6 +200,7 @@ def sdk():
return SimpleNamespace(
CaptureScope=_CaptureScope,
DesktopScope=_DesktopScope,
+ ActionTarget=_ActionTarget,
EffectiveScope=_EffectiveScope,
EscalationReason=_EscalationReason,
ClickButton=_ClickButton,
@@ -258,8 +265,8 @@ async def test_desktop_actions_share_one_typed_session(sdk, fallback):
assert start.cursor_theme is None
clicked = next(value for name, value in driver.calls if name == "click")
assert (clicked.x, clicked.y) == (12.0, 34.0)
- assert clicked.target is None
- assert clicked.scope is _DesktopScope.DESKTOP
+ assert clicked.target.display_id == "primary"
+ assert clicked.scope is None
@pytest.mark.asyncio
@@ -326,7 +333,8 @@ async def test_adapter_uses_the_generated_python_contract(fallback):
request = next(value for call, value in driver.calls if call == name)
assert isinstance(request, expected_type)
if hasattr(request, "target"):
- assert request.target is None
+ assert request.target.display_id == "primary"
+ assert request.scope is None
@pytest.mark.asyncio