Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

16.2 KB
Raw Blame
"""Playground server implementation for Cua agents."""

import asyncio
import logging
import os
import platform
import re
import socket
import traceback
import webbrowser
from typing import Any, Dict, Iterable, List, Optional, Tuple, Union
from urllib.parse import quote

import uvicorn
from fastapi import FastAPI, HTTPException, Request
from fastapi.middleware.cors import CORSMiddleware
from fastapi.responses import JSONResponse

logger = logging.getLogger(__name__)

DEFAULT_HOST = "127.0.0.1"

# Browser origins served from this machine (any port) are always allowed.
_LOOPBACK_ORIGIN_RE = re.compile(r"https?://(localhost|127\.0\.0\.1|\[::1\])(:\d+)?")

# agent_kwargs a request may set by default: harmless tuning knobs only.
_SAFE_AGENT_KWARGS = frozenset(
    {
        "only_n_most_recent_images",
        "use_prompt_caching",
        "max_retries",
        "screenshot_delay",
        "instructions",
        "verbosity",
        "temperature",
        "top_p",
        "max_tokens",
    }
)

# agent_kwargs a request can never set, even with allow_request_overrides:
# they load code, write to disk or swap the agent's tools and loop.
_BLOCKED_AGENT_KWARGS = frozenset(
    {
        "trust_remote_code",
        "trajectory_dir",
        "tools",
        "callbacks",
        "custom_loop",
    }
)


def _is_loopback_host(host: str) -> bool:
    return host in ("127.0.0.1", "localhost", "::1")


class PlaygroundServer:
    """Playground server for running Cua agents via HTTP API.

    Security model: the server has no authentication, so it is only meant to be
    reachable from the machine it runs on. It binds to loopback by default, only
    answers browser origins served from loopback (plus any ``allowed_origins``
    you list), never writes request data into ``os.environ``, and ignores any
    request field that could redirect the agent's model traffic (``api_base``,
    ``api_key``, ...) unless ``allow_request_overrides=True`` is set.
    """

    def __init__(
        self,
        agent_instance=None,
        *,
        allowed_origins: Optional[Iterable[str]] = None,
        allow_request_overrides: bool = False,
    ):
        """
        Initialize the playground server.

        Args:
            agent_instance: Optional pre-configured agent instance to use
            allowed_origins: Extra browser origins (for example
                ``"https://dashboard.example.com"``) allowed to call the API, on
                top of loopback origins. ``"*"`` is not accepted.
            allow_request_overrides: Let a request set ``agent_kwargs`` that
                point the agent at another model endpoint or credentials
                (``api_base``, ``base_url``, ``api_key``, ...). Off by default.
                Only turn this on if you trust every client that can reach
                the server.
        """
        self.agent_instance = agent_instance
        self.allow_request_overrides = allow_request_overrides
        self.allowed_origins = self._normalize_origins(allowed_origins)
        self.app = FastAPI(
            title="Cua Playground Server",
            description="Playground server for Cua agents",
            version="0.1.0",
        )
        self._setup_middleware()
        self._setup_routes()
        self.server = None
        self.port = None

    @staticmethod
    def _normalize_origins(origins: Optional[Iterable[str]]) -> List[str]:
        normalized: List[str] = []
        for origin in origins or ():
            origin = origin.strip().rstrip("/").lower()
            if not origin:
                continue
            if "*" in origin:
                raise ValueError(
                    "Wildcard origins are not allowed for the playground server; "
                    "list each origin explicitly."
                )
            normalized.append(origin)
        return normalized

    def _is_origin_allowed(self, origin: str) -> bool:
        return (
            _LOOPBACK_ORIGIN_RE.fullmatch(origin) is not None
            or origin.rstrip("/").lower() in self.allowed_origins
        )

    def _setup_middleware(self):
        """Setup origin checks and CORS (added last, so it runs first)."""

        @self.app.middleware("http")
        async def reject_foreign_origins(request: Request, call_next):
            # CORS only stops a page from reading a response; a cross-origin
            # "simple" POST is still executed. Browsers always send Origin on
            # cross-origin requests, so refuse them before any route runs.
            origin = request.headers.get("origin")
            if origin is not None and not self._is_origin_allowed(origin):
                return JSONResponse(status_code=403, content={"detail": "Origin not allowed"})
            return await call_next(request)

        self.app.add_middleware(
            CORSMiddleware,
            allow_origins=self.allowed_origins,
            allow_origin_regex=_LOOPBACK_ORIGIN_RE.pattern,
            allow_credentials=False,
            allow_methods=["GET", "POST", "OPTIONS"],
            allow_headers=["*"],
        )

    def _setup_routes(self):
        """Setup API routes."""

        @self.app.get("/status")
        async def status():
            """Health check endpoint."""
            sys = platform.system().lower()
            if "darwin" in sys or sys in ("macos", "mac"):
                os_type = "macos"
            elif "windows" in sys:
                os_type = "windows"
            else:
                os_type = "linux"

            return {
                "status": "ok",
                "os_type": os_type,
                "features": ["agent", "playground"],
            }

        @self.app.post("/responses")
        async def responses_endpoint(request: Request):
            """
            Run ComputerAgent for up to 2 turns.

            Body JSON:
            {
              "model": "...",                 # required
              "input": "... or messages[]",   # required
              "agent_kwargs": { ... },         # optional tuning options for ComputerAgent
              "env": { ... }                   # ignored; kept so older clients still work
            }

            ``env`` is never applied. ``agent_kwargs`` is limited to harmless tuning
            options; anything else (``api_base``, ``api_key``, ...) is ignored unless
            the server was created with ``allow_request_overrides=True``.
            """
            # Import here to avoid circular imports
            try:
                from cua_agent import ComputerAgent
            except ImportError:
                raise HTTPException(status_code=501, detail="ComputerAgent not available")

            # Parse request body
            try:
                body = await request.json()
            except Exception as e:
                raise HTTPException(status_code=400, detail=f"Invalid JSON body: {str(e)}")
            if not isinstance(body, dict):
                raise HTTPException(status_code=400, detail="Body must be a JSON object")

            model = body.get("model")
            input_data = body.get("input")
            if not model or input_data is None:
                raise HTTPException(status_code=400, detail="'model' and 'input' are required")

            raw_agent_kwargs = body.get("agent_kwargs") or {}
            if not isinstance(raw_agent_kwargs, dict):
                raise HTTPException(status_code=400, detail="'agent_kwargs' must be an object")
            agent_kwargs, ignored = self._filter_agent_kwargs(raw_agent_kwargs)

            # Never write request data into the process environment.
            if body.get("env"):
                ignored.append("env")
            if ignored:
                logger.warning(
                    "Ignored request fields that the playground server does not accept: %s",
                    ", ".join(sorted(set(ignored))),
                )

            # Convert input to messages
            def _to_messages(data: Union[str, List[Dict[str, Any]]]) -> List[Dict[str, Any]]:
                if isinstance(data, str):
                    return [{"role": "user", "content": data}]
                if isinstance(data, list):
                    return data
                return []

            messages = _to_messages(input_data)

            error = None

            # Use pre-configured agent only if model matches, otherwise create new one
            # This ensures the model picker in the UI is respected
            if self.agent_instance and self.agent_instance.model == model:
                agent = self.agent_instance
            else:
                # Model changed or no pre-configured agent - create new agent with requested model
                agent = ComputerAgent(model=model, **agent_kwargs)  # type: ignore[arg-type]

            total_output: List[Any] = []
            total_usage: Dict[str, Any] = {}

            pending_computer_call_ids = set()
            try:
                async for result in agent.run(messages):
                    total_output += result["output"]
                    # Try to collect usage if present
                    if (
                        isinstance(result, dict)
                        and "usage" in result
                        and isinstance(result["usage"], dict)
                    ):
                        # Merge usage counters
                        for k, v in result["usage"].items():
                            if isinstance(v, (int, float)):
                                total_usage[k] = total_usage.get(k, 0) + v
                            else:
                                total_usage[k] = v
                    for msg in result.get("output", []):
                        if msg.get("type") == "computer_call":
                            pending_computer_call_ids.add(msg["call_id"])
                        elif msg.get("type") == "computer_call_output":
                            pending_computer_call_ids.discard(msg["call_id"])
                        elif msg.get("type") == "function_call":
                            pending_computer_call_ids.add(msg["call_id"])
                        elif msg.get("type") == "function_call_output":
                            pending_computer_call_ids.discard(msg["call_id"])
                    # exit if no pending computer calls
                    if not pending_computer_call_ids:
                        break
            except Exception as e:
                logger.error(f"Error running agent: {str(e)}")
                logger.error(traceback.format_exc())
                error = str(e)

            # Build response payload
            payload = {
                "model": model,
                "error": error,
                "output": total_output,
                "usage": total_usage,
                "status": "completed" if not error else "failed",
            }
            if ignored:
                payload["ignored_request_fields"] = sorted(set(ignored))

            headers = {
                "Cache-Control": "no-cache",
                "Connection": "keep-alive",
            }

            return JSONResponse(content=payload, headers=headers)

    def _filter_agent_kwargs(self, requested: Dict[str, Any]) -> Tuple[Dict[str, Any], List[str]]:
        """Split request agent_kwargs into (accepted, ignored key names)."""
        accepted: Dict[str, Any] = {}
        ignored: List[str] = []
        for key, value in requested.items():
            if key in _BLOCKED_AGENT_KWARGS:
                ignored.append(f"agent_kwargs.{key}")
            elif key in _SAFE_AGENT_KWARGS or self.allow_request_overrides:
                accepted[key] = value
            else:
                ignored.append(f"agent_kwargs.{key}")
        return accepted, ignored

    def _find_available_port(self, start_port: int = 8000, max_attempts: int = 100) -> int:
        """Find an available port starting from start_port."""
        for port in range(start_port, start_port + max_attempts):
            try:
                with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
                    s.bind(("127.0.0.1", port))
                    return port
            except OSError:
                continue
        raise RuntimeError(
            f"Could not find an available port in range {start_port}-{start_port + max_attempts}"
        )

    async def start_async(
        self,
        port: Optional[int] = None,
        open_browser: bool = False,
        host: str = DEFAULT_HOST,
    ):
        """
        Start the playground server asynchronously.

        Args:
            port: Port to run the server on. If None, finds an available port.
            open_browser: Whether to open the browser automatically.
            host: Interface to bind. Defaults to loopback. Pass "0.0.0.0" only if
                you really want other machines to reach this unauthenticated server.
        """
        if port is None:
            port = self._find_available_port()

        self.port = port
        base_url = f"http://localhost:{port}"

        if not _is_loopback_host(host):
            logger.warning(
                "Playground server is binding to %s and has no authentication; "
                "anyone who can reach this address can run the agent.",
                host,
            )
        logger.info(f"Starting playground server on {base_url} (bound to {host})")

        if open_browser:
            # Construct the playground URL
            encoded_host = quote(base_url, safe="")
            encoded_model = quote(self.agent_instance.model, safe="")
            encoded_vnc_url = quote("http://localhost:8006/?autoconnect=true", safe="")

            # Build URL with custom_model if agent instance is configured
            playground_url = (
                # f"http://cua.ai/dashboard/playground"
                f"http://localhost:3000/dashboard/playground"
                f"?host={encoded_host}"
                f"&port={port}"
                f"&id=localhost"
                f"&name=localhost"
                f"&custom_model={encoded_model}"
                f"&custom_vnc_url={encoded_vnc_url}"
                f"&vnc_password=null"
                f"&resize=scale"
                f"&fullscreen=true"
            )

            logger.info(f"Opening browser at: {playground_url}")
            webbrowser.open(playground_url)

        config = uvicorn.Config(
            self.app,
            host=host,
            port=port,
            log_level="info",
        )
        self.server = uvicorn.Server(config)
        await self.server.serve()

    def start(
        self,
        port: Optional[int] = None,
        open_browser: bool = False,
        host: str = DEFAULT_HOST,
    ):
        """
        Start the playground server (blocking).

        Args:
            port: Port to run the server on. If None, finds an available port.
            open_browser: Whether to open the browser automatically.
            host: Interface to bind. Defaults to loopback (see ``start_async``).
        """
        # Check if there's already a running event loop
        try:
            loop = asyncio.get_running_loop()
            # If we're in an async context, schedule as a task
            import threading

            # Run the server in a separate thread to avoid blocking
            server_thread = threading.Thread(
                target=self._run_in_new_loop,
                args=(port, open_browser, host),
                daemon=True,
            )
            server_thread.start()

            # Give the server a moment to start and open browser
            import time

            time.sleep(1)

        except RuntimeError:
            # No running loop, can use asyncio.run() safely
            asyncio.run(self.start_async(port=port, open_browser=open_browser, host=host))

    def _run_in_new_loop(
        self,
        port: Optional[int] = None,
        open_browser: bool = False,
        host: str = DEFAULT_HOST,
    ):
        """Helper to run server in a new event loop (for threading)."""
        new_loop = asyncio.new_event_loop()
        asyncio.set_event_loop(new_loop)
        try:
            new_loop.run_until_complete(
                self.start_async(port=port, open_browser=open_browser, host=host)
            )
        finally:
            new_loop.close()

    async def stop(self):
        """Stop the playground server."""
        if self.server:
            logger.info("Stopping playground server")
            await self.server.shutdown()