Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

7.3 KB
Raw Blame
#!/usr/bin/env -S uv run --script
# /// script
# requires-python = ">=3.10"
# dependencies = ["fastmcp", "httpx"]
# ///
"""
Computer-Use MCP Server (self-contained with uv)
Runs inside the container and connects to host's cuabotd HTTP server
Provides screenshot and input tools for Claude coding agents
"""

import os
import time

import httpx
from fastmcp import FastMCP
from fastmcp.utilities.types import Image

# Connect to host's cuabotd server
HOST_URL = os.environ.get("CUABOT_HOST", "http://host.docker.internal:7842")
TELEMETRY_ENABLED = os.environ.get("CUABOT_TELEMETRY", "false").lower() == "true"
STARTING_ERROR = "cuabotd is still starting"

# Overlay cursor socket
OVERLAY_SOCKET = "/tmp/cuabot-overlay-cursor.sock"

server = FastMCP("computer-use")

# HTTP client for cuabotd
client = httpx.Client(base_url=HOST_URL, timeout=30.0)

# Separate client for telemetry with short timeout
telemetry_client = httpx.Client(base_url=HOST_URL, timeout=1.0)


# Only these argument keys are ever sent, and only as numbers (or a mouse
# button name). Typed text, key names, save paths and anything else a tool
# receives never leave the container.
_TELEMETRY_NUMERIC_ARGS = frozenset(
    {"x", "y", "delta_x", "delta_y", "from_x", "from_y", "to_x", "to_y", "delay"}
)
_TELEMETRY_BUTTONS = frozenset({"left", "right", "middle"})


def _telemetry_safe_args(tool_args: dict) -> dict:
    safe = {}
    for key, value in tool_args.items():
        if (
            key in _TELEMETRY_NUMERIC_ARGS
            and isinstance(value, (int, float))
            and not isinstance(value, bool)
        ):
            safe[key] = value
        elif key == "button" and value in _TELEMETRY_BUTTONS:
            safe[key] = value
    return safe


def log_mcp_tool_call(tool_name: str, tool_args: dict) -> None:
    """Send MCP tool call telemetry to cuabotd (tool name + safe numeric args)."""
    if not TELEMETRY_ENABLED:
        return
    try:
        event = {
            "type": "mcp_tool_call",
            "timestamp": int(time.time() * 1000),
            "tool_name": tool_name,
            "tool_args": _telemetry_safe_args(tool_args),
        }
        telemetry_client.post("/telemetry", json=event)
    except Exception:
        pass  # Silently ignore telemetry errors


def request(endpoint: str, body: dict | None = None) -> dict:
    """Make HTTP request to cuabotd server with retry on starting error."""
    max_retries = 120  # 2 minutes max wait
    retry_delay = 1.0  # 1 second between retries

    for attempt in range(max_retries):
        res = client.post(f"/{endpoint}", json=body)
        data = res.json()
        if "error" in data:
            # If server is still starting, wait and retry
            if data["error"] == STARTING_ERROR:
                if attempt == 0:
                    print("Waiting for cuabotd to finish starting...", flush=True)
                time.sleep(retry_delay)
                continue
            raise Exception(data["error"])
        return data

    raise Exception("Timed out waiting for cuabotd to start")


@server.tool()
def screenshot(save_path: str | None = None) -> Image:
    """Take a screenshot of the display. Returns JPEG image.

    Args:
        save_path: Optional path to save the screenshot (e.g., "/tmp/screenshot.jpg")
    """
    import base64

    log_mcp_tool_call("screenshot", {})
    result = request("screenshot")
    image_bytes = base64.b64decode(result["image"])

    if save_path:
        with open(save_path, "wb") as f:
            f.write(image_bytes)

    return Image(data=image_bytes, format="jpeg")


@server.tool()
def click(x: int, y: int, button: str = "left") -> str:
    """Click at x,y coordinates.

    Args:
        x: X coordinate
        y: Y coordinate
        button: Mouse button - left, right, or middle (default: left)
    """
    log_mcp_tool_call("click", {"x": x, "y": y, "button": button})
    request("click", {"x": x, "y": y, "button": button})
    return "Clicked"


@server.tool()
def double_click(x: int, y: int) -> str:
    """Double-click at x,y coordinates.

    Args:
        x: X coordinate
        y: Y coordinate
    """
    log_mcp_tool_call("double_click", {"x": x, "y": y})
    request("doubleClick", {"x": x, "y": y})
    return "Double-clicked"


@server.tool()
def type_text(text: str, delay: int = 50) -> str:
    """Type text using the keyboard.

    Args:
        text: Text to type
        delay: Delay between keystrokes in ms (default: 50)
    """
    log_mcp_tool_call("type_text", {"delay": delay})
    request("type", {"text": text, "delay": delay})
    return "Typed"


@server.tool()
def mouse_move(x: int, y: int) -> str:
    """Move mouse to x,y coordinates.

    Args:
        x: X coordinate
        y: Y coordinate
    """
    log_mcp_tool_call("mouse_move", {"x": x, "y": y})
    request("mouseMove", {"x": x, "y": y})
    return "Mouse moved"


@server.tool()
def mouse_down(x: int, y: int, button: str = "left") -> str:
    """Press mouse button down at x,y.

    Args:
        x: X coordinate
        y: Y coordinate
        button: Mouse button - left, right, or middle (default: left)
    """
    log_mcp_tool_call("mouse_down", {"x": x, "y": y, "button": button})
    request("mouseDown", {"x": x, "y": y, "button": button})
    return "Mouse down"


@server.tool()
def mouse_up(x: int, y: int, button: str = "left") -> str:
    """Release mouse button at x,y.

    Args:
        x: X coordinate
        y: Y coordinate
        button: Mouse button - left, right, or middle (default: left)
    """
    log_mcp_tool_call("mouse_up", {"x": x, "y": y, "button": button})
    request("mouseUp", {"x": x, "y": y, "button": button})
    return "Mouse up"


@server.tool()
def scroll(x: int, y: int, delta_x: int, delta_y: int) -> str:
    """Scroll at x,y coordinates.

    Args:
        x: X coordinate
        y: Y coordinate
        delta_x: Horizontal scroll amount
        delta_y: Vertical scroll amount (negative = up, positive = down)
    """
    log_mcp_tool_call("scroll", {"x": x, "y": y, "delta_x": delta_x, "delta_y": delta_y})
    request("scroll", {"x": x, "y": y, "deltaX": delta_x, "deltaY": delta_y})
    return "Scrolled"


@server.tool()
def key_down(key: str) -> str:
    """Press a key down (without releasing).

    Args:
        key: Key to press (e.g., 'Shift', 'Control', 'a', 'Enter')
    """
    log_mcp_tool_call("key_down", {})
    request("keyDown", {"key": key})
    return "Key down"


@server.tool()
def key_up(key: str) -> str:
    """Release a key.

    Args:
        key: Key to release
    """
    log_mcp_tool_call("key_up", {})
    request("keyUp", {"key": key})
    return "Key up"


@server.tool()
def key_press(key: str) -> str:
    """Press and release a key.

    Args:
        key: Key to press (e.g., 'Enter', 'Tab', 'Escape', 'F1')
    """
    log_mcp_tool_call("key_press", {})
    request("keyPress", {"key": key})
    return "Key pressed"


@server.tool()
def drag(from_x: int, from_y: int, to_x: int, to_y: int) -> str:
    """Drag from one point to another.

    Args:
        from_x: Start X coordinate
        from_y: Start Y coordinate
        to_x: End X coordinate
        to_y: End Y coordinate
    """
    log_mcp_tool_call("drag", {"from_x": from_x, "from_y": from_y, "to_x": to_x, "to_y": to_y})
    request("drag", {"fromX": from_x, "fromY": from_y, "toX": to_x, "toY": to_y})
    return "Dragged"


if __name__ == "__main__":
    server.run()