Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

16.6 KB
Raw Blame
"""Tests for the worker server HTTP endpoints.

This module tests:
- Action serialization/deserialization (unit tests)
- FastAPI server endpoints using TestClient with real simulated environments (e2e)
- The background timeout checker that reaps inactive environments
"""

import threading
import time

import pytest
from cua_bench.types import (
    ClickAction,
    DoneAction,
    DoubleClickAction,
    DragAction,
    HotkeyAction,
    KeyAction,
    MiddleClickAction,
    MoveToAction,
    RightClickAction,
    ScrollAction,
    TypeAction,
    WaitAction,
)
from cua_bench_rl.workers import worker_server
from cua_bench_rl.workers.worker_server import (
    ResetRequest,
    ShutdownRequest,
    StepRequest,
    app,
    deserialize_action,
    serialize_action,
)
from fastapi.testclient import TestClient

# Simple HTML for test tasks
SIMPLE_BUTTON_HTML = """
<div class="flex flex-col items-center justify-center h-full w-full bg-gray-100 p-4">
    <h1 class="text-xl mb-4">Test Task</h1>
    <button
        id="test-btn"
        class="btn bg-blue-500 text-white px-4 py-2 rounded"
        onclick="window.__clicked = true; window.__score = 1.0;"
    >
        Click Me
    </button>
</div>
<script>
    window.__clicked = false;
    window.__score = 0.0;
</script>
"""


class TestDeserializeAction:
    """Tests for deserialize_action function."""

    def test_click_action_type_format(self):
        """Test deserializing ClickAction with type format."""
        action_dict = {"type": "ClickAction", "x": 100, "y": 200}
        action = deserialize_action(action_dict)
        assert isinstance(action, ClickAction)
        assert action.x == 100
        assert action.y == 200

    def test_click_action_lowercase_format(self):
        """Test deserializing ClickAction with lowercase format."""
        action_dict = {"action_type": "click", "x": 100, "y": 200}
        action = deserialize_action(action_dict)
        assert isinstance(action, ClickAction)
        assert action.x == 100
        assert action.y == 200

    def test_right_click_action(self):
        """Test deserializing RightClickAction."""
        action_dict = {"type": "RightClickAction", "x": 50, "y": 75}
        action = deserialize_action(action_dict)
        assert isinstance(action, RightClickAction)
        assert action.x == 50
        assert action.y == 75

    def test_double_click_action(self):
        """Test deserializing DoubleClickAction."""
        action_dict = {"type": "DoubleClickAction", "x": 300, "y": 400}
        action = deserialize_action(action_dict)
        assert isinstance(action, DoubleClickAction)
        assert action.x == 300
        assert action.y == 400

    def test_middle_click_action(self):
        """Test deserializing MiddleClickAction."""
        action_dict = {"type": "MiddleClickAction", "x": 150, "y": 250}
        action = deserialize_action(action_dict)
        assert isinstance(action, MiddleClickAction)
        assert action.x == 150
        assert action.y == 250

    def test_drag_action(self):
        """Test deserializing DragAction."""
        action_dict = {
            "type": "DragAction",
            "from_x": 100,
            "from_y": 100,
            "to_x": 200,
            "to_y": 200,
            "duration": 0.5,
        }
        action = deserialize_action(action_dict)
        assert isinstance(action, DragAction)
        assert action.from_x == 100
        assert action.from_y == 100
        assert action.to_x == 200
        assert action.to_y == 200
        assert action.duration == 0.5

    def test_drag_action_default_duration(self):
        """Test deserializing DragAction with default duration."""
        action_dict = {
            "type": "DragAction",
            "from_x": 100,
            "from_y": 100,
            "to_x": 200,
            "to_y": 200,
        }
        action = deserialize_action(action_dict)
        assert action.duration == 1.0  # Default

    def test_move_to_action(self):
        """Test deserializing MoveToAction."""
        action_dict = {"type": "MoveToAction", "x": 500, "y": 600, "duration": 0.2}
        action = deserialize_action(action_dict)
        assert isinstance(action, MoveToAction)
        assert action.x == 500
        assert action.y == 600
        assert action.duration == 0.2

    def test_scroll_action(self):
        """Test deserializing ScrollAction."""
        action_dict = {"type": "ScrollAction", "direction": "down", "amount": 50}
        action = deserialize_action(action_dict)
        assert isinstance(action, ScrollAction)
        assert action.direction == "down"
        assert action.amount == 50

    def test_scroll_action_defaults(self):
        """Test deserializing ScrollAction with defaults."""
        action_dict = {"type": "ScrollAction"}
        action = deserialize_action(action_dict)
        assert action.direction == "up"  # Default
        assert action.amount == 100  # Default

    def test_type_action(self):
        """Test deserializing TypeAction."""
        action_dict = {"type": "TypeAction", "text": "Hello World"}
        action = deserialize_action(action_dict)
        assert isinstance(action, TypeAction)
        assert action.text == "Hello World"

    def test_key_action(self):
        """Test deserializing KeyAction."""
        action_dict = {"type": "KeyAction", "key": "Enter"}
        action = deserialize_action(action_dict)
        assert isinstance(action, KeyAction)
        assert action.key == "Enter"

    def test_hotkey_action(self):
        """Test deserializing HotkeyAction."""
        action_dict = {"type": "HotkeyAction", "keys": ["ctrl", "c"]}
        action = deserialize_action(action_dict)
        assert isinstance(action, HotkeyAction)
        assert action.keys == ["ctrl", "c"]

    def test_wait_action(self):
        """Test deserializing WaitAction."""
        action_dict = {"type": "WaitAction", "seconds": 2.5}
        action = deserialize_action(action_dict)
        assert isinstance(action, WaitAction)
        assert action.seconds == 2.5

    def test_wait_action_default(self):
        """Test deserializing WaitAction with default seconds."""
        action_dict = {"type": "WaitAction"}
        action = deserialize_action(action_dict)
        assert action.seconds == 1.0

    def test_done_action(self):
        """Test deserializing DoneAction."""
        action_dict = {"type": "DoneAction"}
        action = deserialize_action(action_dict)
        assert isinstance(action, DoneAction)

    def test_unknown_action_raises(self):
        """Test that unknown action type raises ValueError."""
        action_dict = {"type": "UnknownAction"}
        with pytest.raises(ValueError, match="Unknown action type"):
            deserialize_action(action_dict)


class TestSerializeAction:
    """Tests for serialize_action function."""

    def test_serialize_click_action(self):
        """Test serializing ClickAction."""
        action = ClickAction(x=100, y=200)
        result = serialize_action(action)
        assert result["type"] == "ClickAction"
        assert result["x"] == 100
        assert result["y"] == 200

    def test_serialize_right_click_action(self):
        """Test serializing RightClickAction."""
        action = RightClickAction(x=50, y=75)
        result = serialize_action(action)
        assert result["type"] == "RightClickAction"
        assert result["x"] == 50
        assert result["y"] == 75

    def test_serialize_drag_action(self):
        """Test serializing DragAction."""
        action = DragAction(from_x=100, from_y=100, to_x=200, to_y=200, duration=0.5)
        result = serialize_action(action)
        assert result["type"] == "DragAction"
        assert result["from_x"] == 100
        assert result["from_y"] == 100
        assert result["to_x"] == 200
        assert result["to_y"] == 200
        assert result["duration"] == 0.5

    def test_serialize_type_action(self):
        """Test serializing TypeAction."""
        action = TypeAction(text="Hello")
        result = serialize_action(action)
        assert result["type"] == "TypeAction"
        assert result["text"] == "Hello"

    def test_serialize_done_action(self):
        """Test serializing DoneAction."""
        action = DoneAction()
        result = serialize_action(action)
        assert result["type"] == "DoneAction"


class TestRequestModels:
    """Tests for Pydantic request models."""

    def test_reset_request_defaults(self):
        """Test ResetRequest with default values."""
        request = ResetRequest(env_path="./task")
        assert request.env_path == "./task"
        assert request.task_index == 0
        assert request.split == "train"
        assert request.timeout == 300

    def test_reset_request_custom(self):
        """Test ResetRequest with custom values."""
        request = ResetRequest(
            env_path="./custom-task",
            task_index=5,
            split="test",
            timeout=600,
        )
        assert request.env_path == "./custom-task"
        assert request.task_index == 5
        assert request.split == "test"
        assert request.timeout == 600

    def test_step_request(self):
        """Test StepRequest."""
        request = StepRequest(
            action={"type": "ClickAction", "x": 100, "y": 200},
            env_id=0,
        )
        assert request.action["type"] == "ClickAction"
        assert request.env_id == 0

    def test_shutdown_request_all(self):
        """Test ShutdownRequest for all envs."""
        request = ShutdownRequest()
        assert request.env_id is None

    def test_shutdown_request_specific(self):
        """Test ShutdownRequest for specific env."""
        request = ShutdownRequest(env_id=1)
        assert request.env_id == 1


class TestActionRoundTrip:
    """Tests for serialize/deserialize round-trip."""

    def test_click_roundtrip(self):
        """Test ClickAction serialize/deserialize round-trip."""
        original = ClickAction(x=123, y=456)
        serialized = serialize_action(original)
        restored = deserialize_action(serialized)
        assert restored.x == original.x
        assert restored.y == original.y

    def test_drag_roundtrip(self):
        """Test DragAction serialize/deserialize round-trip."""
        original = DragAction(from_x=10, from_y=20, to_x=30, to_y=40, duration=1.5)
        serialized = serialize_action(original)
        restored = deserialize_action(serialized)
        assert restored.from_x == original.from_x
        assert restored.from_y == original.from_y
        assert restored.to_x == original.to_x
        assert restored.to_y == original.to_y
        assert restored.duration == original.duration

    def test_type_roundtrip(self):
        """Test TypeAction serialize/deserialize round-trip."""
        original = TypeAction(text="Hello World!")
        serialized = serialize_action(original)
        restored = deserialize_action(serialized)
        assert restored.text == original.text

    def test_hotkey_roundtrip(self):
        """Test HotkeyAction serialize/deserialize round-trip."""
        original = HotkeyAction(keys=["ctrl", "shift", "n"])
        serialized = serialize_action(original)
        restored = deserialize_action(serialized)
        assert restored.keys == original.keys


class TestServerEndpoints:
    """Tests for FastAPI server endpoints using TestClient.

    Note: These tests only cover endpoints that don't require Playwright.
    Full e2e tests with real environments are in test_worker_manager.py.
    """

    def test_health_endpoint(self):
        """Test GET /health returns ok status."""
        client = TestClient(app)
        response = client.get("/health")
        assert response.status_code == 200
        data = response.json()
        assert data["status"] == "ok"
        assert "available_envs" in data
        assert "active_envs" in data
        assert "max_envs" in data

    def test_shutdown_endpoint(self):
        """Test POST /shutdown releases environments."""
        client = TestClient(app)
        response = client.post("/shutdown", json={})
        assert response.status_code == 200
        data = response.json()
        assert data["status"] == "ok"

    def test_step_without_env_returns_error(self):
        """Test POST /step without valid env_id returns 404."""
        client = TestClient(app)
        response = client.post(
            "/step",
            json={
                "action": {"type": "ClickAction", "x": 100, "y": 200},
                "env_id": 999,
            },
        )
        assert response.status_code == 404

    def test_screenshot_without_env_returns_error(self):
        """Test GET /screenshot without valid env_id returns 404."""
        client = TestClient(app)
        response = client.get("/screenshot", params={"env_id": 999})
        assert response.status_code == 404


class _SweepDone(Exception):
    """Raised by the fake clock to end the timeout checker's loop after one sweep."""


class _OneSweepClock:
    """Fake ``time`` module for a single pass of ``_timeout_checker``.

    Only the checker thread under test sees it: its first ``sleep`` returns at once,
    ``time()`` reports ``now``, and its second ``sleep`` ends the loop. Every other
    thread (request handlers, the module's own checker thread) gets the real clock.
    """

    def __init__(self, now: float):
        self.now = now
        self.thread: threading.Thread | None = None
        self.sleeps = 0

    def _is_checker(self) -> bool:
        return threading.current_thread() is self.thread

    def time(self) -> float:
        return self.now if self._is_checker() else time.time()

    def sleep(self, seconds: float) -> None:
        if not self._is_checker():
            time.sleep(seconds)
            return
        self.sleeps += 1
        if self.sleeps > 1:
            raise _SweepDone


def _run_one_sweep(monkeypatch, now: float) -> tuple[bool, list[BaseException]]:
    """Run one pass of the timeout checker at ``now``; return (finished, errors)."""
    clock = _OneSweepClock(now)
    errors: list[BaseException] = []

    def checker():
        try:
            worker_server._timeout_checker()
        except _SweepDone:
            pass
        except BaseException as e:
            errors.append(e)

    clock.thread = threading.Thread(target=checker, daemon=True)
    monkeypatch.setattr(worker_server, "time", clock)
    clock.thread.start()
    clock.thread.join(timeout=2.0)
    finished = not clock.thread.is_alive()
    if not finished:
        # Stuck on env_lock: release it from here so the checker and the rest of the
        # suite can carry on (a plain Lock can be released by any thread).
        worker_server.env_lock.release()
        clock.thread.join(timeout=2.0)
    return finished, errors


class _FakeEnv:
    """Stands in for a cua_bench Environment and records when close() is awaited."""

    def __init__(self):
        self.closed = threading.Event()

    async def reset(self, task_id: int = 0):
        return b"\x89PNG", {"description": "fake task"}

    async def close(self):
        self.closed.set()


class TestTimeoutChecker:
    """The timeout checker reaps environments idle for longer than their timeout."""

    @pytest.fixture
    def idle_env(self, monkeypatch):
        """Reset one fake env through the API, with the server's event loop running."""
        saved_available = list(worker_server.available_envs)
        saved_active = list(worker_server.active_envs)
        saved_map = dict(worker_server.env_map)
        env = _FakeEnv()
        monkeypatch.setattr(worker_server, "make", lambda env_path, split: env)
        with TestClient(app) as client:
            response = client.post("/reset", json={"env_path": "fake", "timeout": 60})
            assert response.status_code == 200
            yield client, env
        worker_server.available_envs[:] = saved_available
        worker_server.active_envs[:] = saved_active
        worker_server.env_map.clear()
        worker_server.env_map.update(saved_map)

    def test_sweep_releases_idle_env(self, idle_env, monkeypatch):
        """A sweep that finds an expired env finishes and returns its slot to the pool."""
        client, _ = idle_env
        finished, errors = _run_one_sweep(monkeypatch, now=time.time() + 3600)
        assert finished, "timeout checker deadlocked on env_lock"
        assert errors == []
        health = client.get("/health").json()
        assert health["active_envs"] == 0
        assert health["available_envs"] == health["max_envs"]

    def test_sweep_closes_released_env(self, idle_env, monkeypatch):
        """The checker thread has no event loop; the env it releases is still closed."""
        _, env = idle_env
        _run_one_sweep(monkeypatch, now=time.time() + 3600)
        assert env.closed.wait(timeout=2.0), "released env was never closed"