Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

16.7 KB
Raw Blame
"""Exercise foreign callbacks through the generated native library."""

import asyncio
import json
import sys
import unittest
from pathlib import Path

from cua_driver import (
    CuaDriver,
    DriverError,
    DriverExecutionMode,
    ForeignDriverChannelCapabilities,
    ForeignDriverBoundChannel,
    ForeignDriverChannelError,
    ForeignDriverChannelIdentity,
    ForeignDriverEnvelopeChannel,
    ForeignDriverResponseEnvelope,
    GetScreenSizeInput,
    SessionPermissionMode,
    TrustedSessionOptions,
    connect_remote_channel,
    create_remote_trusted_session,
)


class Carrier(ForeignDriverEnvelopeChannel):
    def __init__(self):
        self.principal = "test-carrier-context"
        self.generation = "test-generation"
        self.requests = []
        self.closed = asyncio.Event()
        self.started = asyncio.Event()
        self.cancelled = asyncio.Event()
        self.cancelled_id = None
        self.bound = None
        self.response_mode = "ok"
        self.result = None
        self.supports_cancellation = True
        self.identity_error = False
        self.minimum_version = 1
        self.negotiate_wait = False

    def identity(self):
        if self.identity_error:
            raise ForeignDriverChannelError.Failed("fixture identity unavailable")
        return ForeignDriverChannelIdentity(
            authenticated_principal=self.principal, connection_generation=self.generation
        )

    async def negotiate(self):
        if self.negotiate_wait:
            self.started.set()
            await asyncio.Event().wait()
        return ForeignDriverChannelCapabilities(
            minimum_envelope_version=self.minimum_version,
            maximum_envelope_version=self.minimum_version,
            supports_cancellation=self.supports_cancellation,
        )

    async def exchange(self, request):
        self.requests.append(request)
        self.started.set()
        if self.response_mode == "wait":
            await asyncio.Event().wait()
        if self.response_mode == "error":
            raise ForeignDriverChannelError.Failed("fixture disconnected")
        result = self.result if self.result is not None else {
            "content": [{"type": "text", "text": "carrier response"}],
            "isError": False,
        }
        return ForeignDriverResponseEnvelope(
            envelope_version=request.envelope_version,
            request_id="wrong-id" if self.response_mode == "mismatch" else request.request_id,
            ok=self.response_mode != "denied",
            result_json="invalid-json" if self.response_mode == "invalid" else json.dumps(result),
            error="fixture denied" if self.response_mode == "denied" else None,
            error_code="policy_denied" if self.response_mode == "denied" else None,
            completion_known=self.response_mode != "unknown",
        )

    async def bind_session(self, options):
        self.options = options
        if self.bound is None:
            self.bound = Carrier()
        return ForeignDriverBoundChannel(channel=self.bound)

    async def cancel(self, request_id):
        self.cancelled_id = request_id
        self.cancelled.set()

    async def close(self):
        self.closed.set()


def session_options():
    return TrustedSessionOptions(
        public_session="test-session",
        mode=SessionPermissionMode.STANDARD,
        ttl_seconds=60,
        idle_ttl_seconds=30,
        capability_manifest_path=None,
        bounded_manifest_path=None,
    )


class RemoteChannelTests(unittest.IsolatedAsyncioTestCase):
    async def asyncSetUp(self):
        self.unraisable_errors = []
        self.original_unraisablehook = sys.unraisablehook
        sys.unraisablehook = self.unraisable_errors.append
        self.assertTrue(asyncio.get_running_loop().get_debug())
        self.carrier = Carrier()
        self.driver = connect_remote_channel(self.carrier)

    async def asyncTearDown(self):
        await self.driver.shutdown()
        await asyncio.sleep(0)
        sys.unraisablehook = self.original_unraisablehook
        self.assertEqual(self.unraisable_errors, [], "native callback raised an unhandled exception")

    async def test_generated_foreign_future_drop_uses_threadsafe_scheduler(self):
        from cua_driver import _native

        source = Path(_native.__file__).read_text()
        self.assertIn("eventloop.call_soon_threadsafe(_uniffi_cancel_task, task)", source)
        self.assertNotIn("eventloop.call_soon(_uniffi_cancel_task, task)", source)

    async def test_canonical_driver_projection_and_bound_lifecycle(self):
        self.assertIsInstance(self.driver, CuaDriver)
        self.assertEqual(self.driver.execution_mode(), DriverExecutionMode.REMOTE)
        result = await self.driver.call_tool("health_report", '{"sample":null}')
        self.assertEqual(result.text, "carrier response")
        request = self.carrier.requests[0]
        self.assertEqual(request.operation, "call")
        self.assertEqual(request.name, "health_report")
        self.assertEqual(json.loads(request.arguments_json), {"sample": None})
        self.assertGreater(request.deadline_unix_ms, 0)
        self.assertEqual(
            (await self.driver.get_screen_size(GetScreenSizeInput(session=None))).text,
            "carrier response",
        )
        self.assertEqual(self.carrier.requests[-1].name, "get_screen_size")
        session = await create_remote_trusted_session(self.driver, session_options())
        self.assertEqual((await session.call_tool("health_report", "{}")).text, "carrier response")
        session.close()
        await asyncio.wait_for(self.carrier.bound.closed.wait(), 2)
        with self.assertRaises(DriverError.Shutdown):
            await session.call_tool("health_report", "{}")
        await self.driver.shutdown()
        self.assertTrue(self.carrier.closed.is_set())

    async def test_identity_is_snapshotted_and_bound_identity_must_match(self):
        self.carrier.principal = "changed-after-connect"
        session = await create_remote_trusted_session(self.driver, session_options())
        session.close()
        await asyncio.wait_for(self.carrier.bound.closed.wait(), 2)
        self.carrier.bound = Carrier()
        self.carrier.bound.generation = "different-generation"
        with self.assertRaises(DriverError.Remote):
            await create_remote_trusted_session(self.driver, session_options())
        self.assertTrue(self.carrier.bound.closed.is_set())
        self.carrier.bound = Carrier()
        self.carrier.bound.identity_error = True
        with self.assertRaises(DriverError.Remote):
            await create_remote_trusted_session(self.driver, session_options())
        self.assertTrue(self.carrier.bound.closed.is_set())

    async def test_negotiation_fails_before_dispatch(self):
        self.carrier.supports_cancellation = False
        with self.assertRaises(DriverError.Protocol):
            await self.driver.call_tool("health_report", "{}")
        self.assertEqual(self.carrier.requests, [])

    async def test_response_validation_and_unknown_completion(self):
        for mode, error in [
            ("mismatch", DriverError.Protocol),
            ("denied", DriverError.Tool),
            ("unknown", DriverError.ActionInterrupted),
            ("invalid", DriverError.ActionInterrupted),
            ("error", DriverError.ActionInterrupted),
        ]:
            with self.subTest(mode=mode):
                self.carrier.response_mode = mode
                with self.assertRaises(error):
                    await self.driver.call_tool("health_report", "{}")

    async def test_cancel_forwards_exact_request_identity(self):
        self.carrier.response_mode = "wait"
        task = asyncio.create_task(self.driver.call_tool("health_report", "{}"))
        await asyncio.wait_for(self.carrier.started.wait(), 2)
        task.cancel()
        with self.assertRaises(asyncio.CancelledError):
            await task
        await asyncio.wait_for(self.carrier.cancelled.wait(), 2)
        self.assertEqual(self.carrier.cancelled_id, self.carrier.requests[0].request_id)

    async def test_rejected_bound_negotiation_closes_channel(self):
        for minimum_version, cancellation in [(1, False), (2, True)]:
            self.carrier.bound = Carrier()
            self.carrier.bound.minimum_version = minimum_version
            self.carrier.bound.supports_cancellation = cancellation
            with self.assertRaises(DriverError.Protocol):
                await create_remote_trusted_session(self.driver, session_options())
            self.assertTrue(self.carrier.bound.closed.is_set())
            self.assertEqual(self.carrier.bound.requests, [])

    async def test_cancelled_bound_negotiation_closes_channel(self):
        self.carrier.bound = Carrier()
        self.carrier.bound.negotiate_wait = True
        task = asyncio.create_task(create_remote_trusted_session(self.driver, session_options()))
        await asyncio.wait_for(self.carrier.bound.started.wait(), 2)
        task.cancel()
        with self.assertRaises(asyncio.CancelledError):
            await task
        await asyncio.wait_for(self.carrier.bound.closed.wait(), 2)

    async def test_channels_on_independent_loops_and_wrong_loop_rejected(self):
        async def other_loop():
            with self.assertRaisesRegex(RuntimeError, "different asyncio event loop"):
                connect_remote_channel(self.carrier)
            carrier = Carrier()
            driver = connect_remote_channel(carrier)
            try:
                self.assertEqual((await driver.call_tool("health_report", "{}")).text, "carrier response")
                session = await create_remote_trusted_session(driver, session_options())
                session.close()
                await asyncio.wait_for(carrier.bound.closed.wait(), 2)
            finally:
                await driver.shutdown()

        await asyncio.gather(
            asyncio.to_thread(lambda: asyncio.run(other_loop(), debug=True)),
            self.driver.call_tool("health_report", "{}"),
        )

    async def test_closed_owner_loop_returns_error_without_unraisable_callback(self):
        async def create_on_other_loop():
            return connect_remote_channel(Carrier())

        driver = await asyncio.to_thread(lambda: asyncio.run(create_on_other_loop(), debug=True))
        with self.assertRaisesRegex(DriverError.Remote, "event loop is closed"):
            await driver.call_tool("health_report", "{}")
        with self.assertRaisesRegex(DriverError.Remote, "event loop is closed"):
            await driver.shutdown()


class NativeWindowTests(unittest.IsolatedAsyncioTestCase):
    async def test_typed_discovery_snapshot_and_token_click_cross_ffi(self):
        import cua_driver as sdk

        carrier = Carrier()
        driver = connect_remote_channel(carrier)

        def respond(structured, *, images=False, error=False):
            carrier.result = {
                "content": [{"type": "text", "text": "native window fixture"}],
                "structuredContent": structured,
                "isError": error,
            }
            if images:
                carrier.result["content"].append(
                    {"type": "image", "mimeType": "image/png", "data": "cG5n"}
                )

        try:
            respond({"apps": [{"pid": 42, "name": "Editor", "running": True, "active": False}]})
            apps = await driver.list_apps(sdk.ListAppsInput())
            self.assertIsInstance(apps, sdk.ListAppsOutput)
            self.assertEqual(apps.apps[0].name, "Editor")
            self.assertEqual(json.loads(carrier.requests[-1].arguments_json), {})

            respond(
                {
                    "windows": [
                        {
                            "pid": 42,
                            "window_id": 123,
                            "app_name": "Editor",
                            "title": "Document",
                            "bounds": {"x": 0, "y": 0, "width": 800, "height": 600},
                            "is_on_screen": True,
                            "z_index": None,
                        }
                    ]
                }
            )
            windows = await driver.list_windows(sdk.ListWindowsInput(pid=42, on_screen_only=True))
            self.assertIsInstance(windows.windows[0], sdk.WindowInfo)
            self.assertEqual(windows.windows[0].bounds.width, 800)
            self.assertIsNone(windows.windows[0].z_index)
            self.assertEqual(
                json.loads(carrier.requests[-1].arguments_json), {"pid": 42, "on_screen_only": True}
            )

            respond(
                {
                    "pid": 42,
                    "window_id": 123,
                    "snapshot_id": "snapshot-1",
                    "screenshot_width": 800,
                    "screenshot_height": 600,
                    "elements": [
                        {
                            "element_index": 0,
                            "role": "button",
                            "depth": 0,
                            "element_token": "fresh-token",
                            "label": "Save",
                        }
                    ],
                },
                images=True,
            )
            state = await driver.get_window_state(
                sdk.GetWindowStateInput(
                    pid=42,
                    window_id=123,
                    session="native-window",
                    query="Save",
                    include_screenshot=True,
                    include_accessibility_tree=True,
                    screenshot_out_file=None,
                    max_elements=10,
                    max_depth=3,
                    max_dimension=800,
                    max_image_dimension=1600,
                    timeout_ms=None,
                )
            )
            self.assertIsInstance(state, sdk.WindowStateOutput)
            self.assertIsInstance(state.elements[0], sdk.WindowElement)
            self.assertEqual(
                state.images[0], sdk.SnapshotImage(mime_type="image/png", data_base64="cG5n")
            )
            self.assertEqual(
                json.loads(carrier.requests[-1].arguments_json),
                {
                    "pid": 42,
                    "window_id": 123,
                    "session": "native-window",
                    "query": "Save",
                    "include_screenshot": True,
                    "include_accessibility_tree": True,
                    "max_elements": 10,
                    "max_depth": 3,
                    "max_dimension": 800,
                    "max_image_dimension": 1600,
                    # The typed SDK keeps the full response unless told otherwise.
                    "full_output": True,
                },
            )

            respond(
                {
                    "effect": "unverifiable",
                    "route": "global_input",
                    "delivery": {"mode": "not_applicable"},
                }
            )
            click = sdk.ClickInput(
                target=sdk.ActionTarget.WINDOW(pid=42, window_id=123),
                position=sdk.ClickPosition.ELEMENT(element_token=state.elements[0].element_token),
                delivery_mode=sdk.InputDeliveryMode.BACKGROUND,
                session="native-window",
                button=None,
                count=None,
            )
            action = await driver.click(click)
            self.assertIsInstance(action, sdk.ActionResult)
            self.assertEqual(action.effect, sdk.ActionEffect.UNVERIFIABLE)
            self.assertEqual(
                json.loads(carrier.requests[-1].arguments_json),
                {
                    "target": {"kind": "window", "pid": 42, "window_id": 123},
                    "element_token": "fresh-token",
                    "delivery_mode": "background",
                    "session": "native-window",
                },
            )

            # These service refusals test SDK error propagation, not native token validation.
            for token, window_id, code in [
                ("stale-token", 123, "stale_element_token"),
                ("fresh-token", 124, "element_target_mismatch"),
            ]:
                respond({"code": code}, error=True)
                click.position = sdk.ClickPosition.ELEMENT(element_token=token)
                click.target = sdk.ActionTarget.WINDOW(pid=42, window_id=window_id)
                with self.assertRaises(sdk.DriverError.Tool) as error:
                    await driver.click(click)
                self.assertEqual(error.exception.tool, "click")
                self.assertEqual(error.exception.error_code, code)
                request = json.loads(carrier.requests[-1].arguments_json)
                self.assertEqual(request["element_token"], token)
                self.assertEqual(request["target"]["window_id"], window_id)
        finally:
            await driver.shutdown()


if __name__ == "__main__":
    unittest.main()