Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

17.2 KB
Raw Blame
from __future__ import annotations

import asyncio
import importlib.util
import json
import os
import platform
import socket
import tempfile
import threading
import unittest
from pathlib import Path


def _library_name() -> str:
    if os.name == "nt":
        return "cua_driver_sdk.dll"
    if platform.system() == "Darwin":
        return "libcua_driver_sdk.dylib"
    return "libcua_driver_sdk.so"


LIBRARY = Path(__file__).parents[1] / "src" / "cua_driver" / _library_name()


@unittest.skipUnless(LIBRARY.exists(), "host-native UniFFI library is not staged")
class GeneratedOptionsTests(unittest.TestCase):
    def test_public_constructors_keep_released_signatures(self) -> None:
        import inspect

        from cua_driver import CuaDriver

        # Released callers pass these by keyword; embedded is an additive keyword-only option.
        self.assertEqual(str(inspect.signature(CuaDriver.connect)), "(socket_path)")
        self.assertEqual(str(inspect.signature(CuaDriver.create)), "(options=None, *, embedded=False)")

    def test_action_target_is_public_and_uses_the_generated_contract(self) -> None:
        import cua_driver
        from cua_driver import (
            ActionTarget, ClickButton, ClickInput, ClickPosition, InputDeliveryMode,
        )
        from cua_driver._native_contract import ActionTarget as GeneratedActionTarget

        self.assertIn("ActionTarget", cua_driver.__all__)
        self.assertIs(ActionTarget, GeneratedActionTarget)
        target = ActionTarget.DESKTOP(display_id="primary")
        click = ClickInput(
            position=ClickPosition.COORDINATES(x=10.0, y=20.0), target=target,
            delivery_mode=InputDeliveryMode.FOREGROUND, session=None,
            button=ClickButton.LEFT, count=1,
        )
        self.assertIs(click.target, target)
        captured = ClickInput(
            position=ClickPosition.CAPTURED_COORDINATES(
                x=10.0, y=20.0, capture_id="capture-1"
            ),
            target=target, delivery_mode=InputDeliveryMode.FOREGROUND,
            session=None, button=ClickButton.LEFT, count=1,
        )
        self.assertEqual(captured.position.capture_id, "capture-1")
        self.assertTrue(ActionTarget.WINDOW(pid=1, window_id=2).is_WINDOW())
        with self.assertRaises(TypeError):
            ClickInput(x=10.0, y=20.0)
        for name in (
            "AppInfo", "ClickPosition", "ElementFrame", "GetWindowStateInput",
            "InputDeliveryMode", "ListAppsInput", "ListAppsOutput", "ListWindowsInput",
            "ListWindowsOutput", "SnapshotImage", "WindowBounds", "WindowElement",
            "WindowInfo", "WindowStateOutput",
            "ParseVisualRegionsInput", "ParseVisualRegionsOptions",
            "ParseVisualRegionsOutput", "VisualCaptureProvenance",
            "VisualParseError", "VisualParseErrorCode", "VisualRegion",
        ):
            self.assertIn(name, cua_driver.__all__)
            self.assertIsNotNone(getattr(cua_driver, name))

    def test_embedded_overlay_option_defaults_false_and_accepts_true(self) -> None:
        from cua_driver import EmbeddedDriverHostOptions

        required = {
            "binary_path": "/example/cua-driver",
            "host_bundle_id": "com.example.host",
            "socket_path": None,
            "startup_timeout_ms": None,
            "shutdown_timeout_ms": None,
            "permission_mode": None,
            "session_policy_path": None,
            "approve_session_policy": False,
            "dangerously_bypass_approvals": False,
            "environment": [],
            "inherit_stderr": False,
        }

        self.assertFalse(EmbeddedDriverHostOptions(**required).no_overlay)
        self.assertTrue(
            EmbeddedDriverHostOptions(**required, no_overlay=True).no_overlay
        )

    def test_embedded_diagnostics_record_is_public(self) -> None:
        from cua_driver import EmbeddedDriverDiagnostics

        diagnostics = EmbeddedDriverDiagnostics(
            lifecycle_phase="startup",
            exit_code=7,
            stderr_tail="failed",
            stderr_truncated=False,
        )
        self.assertEqual(diagnostics.exit_code, 7)


@unittest.skipUnless(LIBRARY.exists(), "host-native UniFFI library is not staged")
@unittest.skipIf(os.name == "nt", "Unix socket fixture")
class SdkLoaderTests(unittest.TestCase):
    def test_generated_python_embedded_host_owns_the_rust_lifecycle(self) -> None:
        from cua_driver import CuaDriver, EmbeddedCuaDriverHost

        with tempfile.TemporaryDirectory() as directory:
            binary_path = Path(directory) / "fake cua-driver"
            binary_path.write_text(
                """#!/usr/bin/env python3
import json
import os
import select
import socket
import sys

args = sys.argv[1:]
socket_path = args[args.index("--socket") + 1]
host_bundle_id = args[args.index("--host-bundle-id") + 1]
server = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
server.bind(socket_path)
os.chmod(socket_path, 0o600)
server.listen(8)
while True:
    readable, _, _ = select.select([server, sys.stdin.buffer], [], [], 1)
    if sys.stdin.buffer in readable:
        if not sys.stdin.buffer.read(1):
            break
    if server in readable:
        connection, _ = server.accept()
        with connection:
            request = json.loads(connection.makefile("r", encoding="utf-8").readline())
            if request["method"] == "metadata":
                result = {
                    "driver_version": "0.10.0",
                    "contract_version": "0.8.0",
                    "tools_list_schema_version": "1",
                    "capability_version": "1",
                    "mcp_protocol_version": "2025-06-18",
                    "pid": os.getpid(),
                    "embedded": True,
                    "host_bundle_id": host_bundle_id,
                }
            else:
                result = {"tools": [{"name": "embedded_fixture"}]}
            connection.sendall((json.dumps({"ok": True, "result": result}) + "\\n").encode())
server.close()
try:
    os.unlink(socket_path)
except FileNotFoundError:
    pass
""",
                encoding="utf-8",
            )
            binary_path.chmod(0o755)

            async def scenario() -> str:
                host = EmbeddedCuaDriverHost(
                    str(binary_path), "com.example.python-embedded"
                )
                connection = await host.start()
                driver = CuaDriver.connect(connection.socket_path)
                metadata = await driver.metadata()
                self.assertTrue(metadata.embedded)
                self.assertEqual(metadata.pid, connection.pid)
                self.assertEqual(
                    metadata.host_bundle_id, "com.example.python-embedded"
                )
                self.assertEqual(
                    json.loads(await driver.list_tools_json()),
                    {"tools": [{"name": "embedded_fixture"}]},
                )
                await host.stop()
                return connection.socket_path

            socket_path = asyncio.run(scenario())
            self.assertFalse(Path(socket_path).exists())

    def test_generated_python_sdk_calls_the_rust_daemon_interface(self) -> None:
        import cua_driver
        from cua_driver import (
            ActionEffect,
            ActionRoute,
            ActionTarget,
            ClickPosition,
            InputDeliveryMode,
            ClickButton,
            ClickInput,
            CuaDriver,
            EffectiveScope,
            StatePredicate,
            StartSessionOutput,
            ParseVisualRegionsInput,
            ParseVisualRegionsOptions,
            VerificationStatus,
            VerifyStateInput,
            WindowPredicate,
        )

        self.assertIsNotNone(EffectiveScope)
        self.assertIsNotNone(StartSessionOutput)
        self.assertIs(cua_driver.CuaDriver, CuaDriver)
        self.assertFalse(hasattr(cua_driver, "AsyncCuaDriver"))
        self.assertFalse(hasattr(cua_driver, "StdioMcpTransport"))
        self.assertIsNone(importlib.util.find_spec("cua_driver.sdk"))
        self.assertIsNone(importlib.util.find_spec("cua_driver.native"))

        with tempfile.TemporaryDirectory() as directory:
            socket_path = str(Path(directory) / "driver.sock")
            listener = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
            listener.bind(socket_path)
            listener.listen(4)
            captured: list[dict[str, object]] = []

            def serve() -> None:
                while len(captured) < 3:
                    connection, _ = listener.accept()
                    with connection:
                        line = connection.makefile("r", encoding="utf-8").readline()
                        request = json.loads(line)
                        if request["method"] == "metadata":
                            result = {
                                "driver_version": "0.12.6",
                                "contract_version": "0.8.0",
                                "tools_list_schema_version": "1",
                                "capability_version": "1",
                                "mcp_protocol_version": "2025-06-18",
                                "pid": os.getpid(),
                                "embedded": False,
                            }
                        else:
                            captured.append(request)
                            if request["name"] == "verify_state":
                                structured = {
                                    "status": "satisfied",
                                    "stable": True,
                                    "elapsed_ms": 12,
                                    "samples": 2,
                                    "predicates": [],
                                }
                            elif request["name"] == "parse_visual_regions":
                                structured = {
                                    "schema": "cua.visual_regions_v1",
                                    "capture": {
                                        "capture_id": "capture-123",
                                        "source": {"kind": "primary_desktop", "display_id": "primary"},
                                        "screenshot": {
                                            "reference": "sha256:abc",
                                            "width": 2,
                                            "height": 2,
                                            "mime_type": "image/png",
                                            "sha256": "abc",
                                        },
                                        "action_coordinate_space": {"kind": "identity"},
                                    },
                                    "parser": {
                                        "extension_id": "cua-perception",
                                        "extension_version": "1.0.0",
                                        "model_id": "fixture",
                                        "model_version": "1",
                                        "runtime": "fixture",
                                    },
                                    "regions": [],
                                    "timing": {"duration_ms": 1},
                                }
                            else:
                                structured = {
                                    "effect": "unverifiable",
                                    "route": "global_input",
                                    "delivery": {"mode": "not_applicable"},
                                }
                            result = {
                                "content": [
                                    {"type": "text", "text": "python ffi"},
                                    {
                                        "type": "image",
                                        "mimeType": "image/png",
                                        "data": "cG5n",
                                    },
                                ],
                                "structuredContent": structured,
                                "isError": False,
                            }
                        response = {"ok": True, "result": result}
                        connection.sendall((json.dumps(response) + "\n").encode())

            server = threading.Thread(target=serve)
            server.start()
            driver = CuaDriver.connect(socket_path)
            expected_methods = {
                "start_session",
                "escalate_session",
                "get_session",
                "list_sessions",
                "get_session_state",
                "end_session",
                "get_desktop_state",
                "get_screen_size",
                "get_cursor_position",
                "move_cursor",
                "click",
                "drag",
                "scroll",
                "type_text",
                "press_key",
                "hotkey",
                "verify_state",
                "parse_visual_regions",
            }
            self.assertTrue(all(hasattr(driver, name) for name in expected_methods))
            verification_result = asyncio.run(
                driver.verify_state(
                    VerifyStateInput(
                        pid=123,
                        window_id=456,
                        expect=[
                            StatePredicate(
                                window=WindowPredicate(exists=True, bounds=None),
                                element=None,
                            )
                        ],
                        session="python-run",
                        timeout_ms=0,
                        stable_samples=1,
                        include_screenshot=True,
                    )
                )
            )
            action_result = asyncio.run(
                driver.click(
                    ClickInput(
                        position=ClickPosition.COORDINATES(x=12.0, y=34.0),
                        target=ActionTarget.DESKTOP(display_id="primary"),
                        delivery_mode=InputDeliveryMode.FOREGROUND,
                        session="python-run",
                        button=ClickButton.LEFT,
                        count=1,
                    )
                )
            )
            visual_result = asyncio.run(
                driver.parse_visual_regions(
                    ParseVisualRegionsInput(
                        capture_id="capture-123",
                        options=ParseVisualRegionsOptions(
                            kinds=None, min_confidence=None, max_regions=None
                        ),
                    )
                )
            )
            server.join(timeout=5)
            listener.close()

        self.assertEqual(verification_result.text, "python ffi")
        self.assertEqual(verification_result.images[0].mime_type, "image/png")
        self.assertIsNone(verification_result.action)
        self.assertEqual(
            verification_result.verification.status, VerificationStatus.SATISFIED
        )
        self.assertEqual(action_result.effect, ActionEffect.UNVERIFIABLE)
        self.assertEqual(action_result.route, ActionRoute.GLOBAL_INPUT)
        self.assertFalse(hasattr(action_result, "verified"))
        self.assertEqual(
            json.loads(visual_result.structured_json)["schema"],
            "cua.visual_regions_v1",
        )
        self.assertEqual(captured[0]["name"], "verify_state")
        self.assertEqual(
            captured[0]["args"],
            {
                "pid": 123,
                "window_id": 456,
                "expect": [{"window": {"exists": True}}],
                "session": "python-run",
                "timeout_ms": 0,
                "stable_samples": 1,
                "include_screenshot": True,
            },
        )
        self.assertEqual(captured[0]["client_kind"], "python_sdk")
        self.assertEqual(captured[1]["name"], "click")
        self.assertEqual(
            captured[1]["args"],
            {
                "x": 12.0,
                "y": 34.0,
                "target": {"kind": "desktop", "display_id": "primary"},
                "delivery_mode": "foreground",
                "session": "python-run",
                "button": "left",
                "count": 1,
            },
        )
        self.assertEqual(captured[2]["name"], "parse_visual_regions")
        self.assertEqual(
            captured[2]["args"],
            {"capture_id": "capture-123", "options": {}},
        )

    def test_generated_python_sdk_can_own_the_runtime_in_process(self) -> None:
        from cua_driver import CuaDriver, DriverExecutionMode

        async def scenario() -> None:
            driver = CuaDriver.create(embedded=True)
            self.assertEqual(driver.execution_mode(), DriverExecutionMode.EMBEDDED)
            self.assertEqual(driver.socket_path(), "")
            self.assertTrue(driver.is_available())
            metadata = await driver.metadata()
            self.assertTrue(metadata.embedded)
            self.assertEqual(metadata.pid, os.getpid())
            await driver.shutdown()
            self.assertFalse(driver.is_available())

        asyncio.run(scenario())


if os.environ.get("CUA_DRIVER_REQUIRE_UNIFFI") == "1" and not LIBRARY.exists():
    raise RuntimeError(f"required staged UniFFI library is missing: {LIBRARY}")


if __name__ == "__main__":
    unittest.main()