Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
added agent proxy to computer-server
Dillon DuPont5e41e7577993
diff --git a/libs/python/computer-server/computer_server/main.py b/libs/python/computer-server/computer_server/main.pyindex ab1a9f309c..401b7b9a26 100644--- a/libs/python/computer-server/computer_server/main.py+++ b/libs/python/computer-server/computer_server/main.py@@ -1,6 +1,6 @@from fastapi import FastAPI, WebSocket, WebSocketDisconnect, Request, HTTPException, Headerfrom fastapi.responses import StreamingResponse-from typing import List, Dict, Any, Optional+from typing import List, Dict, Any, Optional, Union, Literal, castimport uvicornimport loggingimport asyncio@@ -14,6 +14,7 @@ import osimport aiohttpimport hashlibimport time+import platform# Set up logging with more detaillogger = logging.getLogger(__name__)@@ -427,5 +428,145 @@ async def cmd_endpoint()[email protected]("/responses")+async def agent_response_endpoint(+ request: Request,+ api_key: Optional[str] = Header(None, alias="X-API-Key"),+):+ """+ Run a ComputerAgent step using server-side handlers as a tool.++ Security:+ - If CONTAINER_NAME is set on the server, require X-API-Key+ and validate using AuthenticationManager.+ - If CUA_ENABLE_PUBLIC_PROXY is set, allow public access.++ Body JSON:+ {+ "model": "...", # required+ "input": "... or messages[]", # required+ "agent_kwargs": { ... }, # optional, will be merged; tools will be overridden+ "computer_kwargs": { ... }, # optional, ignored for this endpoint+ "env": { ... } # optional env overrides for agent+ }+ """+ from agent.proxy.handlers import ResponsesHandler+ from agent.computers import AsyncComputerHandler++ # Authenticate via AuthenticationManager if running in cloud (CONTAINER_NAME set)+ container_name = os.environ.get("CONTAINER_NAME")+ if container_name:+ is_public = os.environ.get("CUA_ENABLE_PUBLIC_PROXY", "").lower().strip() in ["1", "true", "yes", "y", "on"]+ if not is_public:+ if not api_key:+ raise HTTPException(status_code=401, detail="Missing AGENT PROXY auth headers")+ ok = await auth_manager.auth(container_name, api_key)+ if not ok:+ raise HTTPException(status_code=401, detail="Unauthorized")++ # Parse request body+ try:+ body = await request.json()+ except Exception as e:+ raise HTTPException(status_code=400, detail=f"Invalid JSON body: {str(e)}")++ model = body.get("model")+ input_data = body.get("input")+ if not model or input_data is None:+ raise HTTPException(status_code=400, detail="'model' and 'input' are required")++ agent_kwargs = body.get("agent_kwargs") or {}+ env_overrides = body.get("env") or {}++ # Local AsyncComputerHandler implementation backed by automation_handler+ class DirectComputerHandler(AsyncComputerHandler):+ async def get_environment(self) -> Literal["windows", "mac", "linux", "browser"]:+ sys = platform.system().lower()+ if "darwin" in sys or sys == "macos" or sys == "mac":+ return "mac"+ if "windows" in sys:+ return "windows"+ return "linux"++ async def get_dimensions(self) -> tuple[int, int]:+ try:+ res = await automation_handler.get_screen_size()+ size = res.get("size") or {}+ return int(size.get("width", 0)), int(size.get("height", 0))+ except Exception:+ return (0, 0)++ async def screenshot(self) -> str:+ res = await automation_handler.screenshot()+ if not res.get("success"):+ raise RuntimeError(res.get("error", "screenshot failed"))+ return res.get("image_data", "")++ async def click(self, x: int, y: int, button: str = "left") -> None:+ if button == "right":+ await automation_handler.right_click(x, y)+ else:+ await automation_handler.left_click(x, y)++ async def double_click(self, x: int, y: int) -> None:+ await automation_handler.double_click(x, y)++ async def scroll(self, x: int, y: int, scroll_x: int, scroll_y: int) -> None:+ try:+ if scroll_y:+ await automation_handler.scroll(scroll_y, 0)+ if scroll_x:+ await automation_handler.scroll(0, scroll_x)+ except Exception:+ await automation_handler.scroll(scroll_y or 0, scroll_x or 0)++ async def type(self, text: str) -> None:+ await automation_handler.type_text(text)++ async def wait(self, ms: int = 1000) -> None:+ await asyncio.sleep(ms / 1000.0)++ async def move(self, x: int, y: int) -> None:+ await automation_handler.move_cursor(x, y)++ async def keypress(self, keys: Union[List[str], str]) -> None:+ if isinstance(keys, list):+ if len(keys) <= 1:+ key = keys[0] if keys else ""+ if key:+ await automation_handler.press_key(key)+ else:+ await cast(Any, automation_handler).hotkey([str(k) for k in keys])+ elif isinstance(keys, str):+ await automation_handler.press_key(keys)++ async def drag(self, path: List[Dict[str, int]]) -> None:+ await automation_handler.drag([(p["x"], p["y"]) for p in path])++ async def get_current_url(self) -> str:+ return ""++ async def left_mouse_down(self, x: Optional[int] = None, y: Optional[int] = None) -> None:+ await automation_handler.mouse_down(x, y, button="left")++ async def left_mouse_up(self, x: Optional[int] = None, y: Optional[int] = None) -> None:+ await automation_handler.mouse_up(x, y, button="left")++ custom_handler = DirectComputerHandler()++ # Prepare request for ResponsesHandler and force our tool+ rh = ResponsesHandler()+ request_payload: Dict[str, Any] = {+ "model": model,+ "input": input_data,+ "agent_kwargs": {**agent_kwargs, "tools": [custom_handler]},+ # Don't need computer_kwargs; agent will use our tool instead+ "env": env_overrides,+ }++ result = await rh.process_request(request_payload)+ return result++if __name__ == "__main__":uvicorn.run(app, host="0.0.0.0", port=8000)diff --git a/libs/python/computer-server/pyproject.toml b/libs/python/computer-server/pyproject.tomlindex 6e9e724066..2790ab5143 100644--- a/libs/python/computer-server/pyproject.toml+++ b/libs/python/computer-server/pyproject.toml@@ -21,7 +21,8 @@ dependencies = ["pillow>=10.2.0","aiohttp>=3.9.1","pyperclip>=1.9.0",- "websockets>=12.0"+ "websockets>=12.0",+ "cua-agent[all]>=0.4.0"][project.optional-dependencies]diff --git a/libs/typescript/agent/README.md b/libs/typescript/agent/README.mdindex 3ca9e054ce..76db192e2a 100644--- a/libs/typescript/agent/README.md+++ b/libs/typescript/agent/README.md@@ -19,9 +19,15 @@ yarn add @trycua/agent```typescriptimport AgentClient from "@trycua/agent";-// Connect to HTTP server+// Connect to local HTTP serverconst client = new AgentClient("https://localhost:8000");+// Connect to a cloud container (port 8443 over HTTPS)+const cloud = new AgentClient(+ "https://m-linux-96lcxd2c2k.containers.cloud.trycua.com:8443",+ { apiKey: process.env.NEXT_PUBLIC_CUA_API_KEY || "" }+);+// Connect to peerconst peerClient = new AgentClient("peer://my-agent-proxy");@@ -64,7 +70,8 @@ const response = await client.responses.create({```typescriptconst client = new AgentClient("https://localhost:8000", {timeout: 60000, // 60 second timeout- retries: 5 // 5 retry attempts+ retries: 5, // 5 retry attempts+ apiKey: "cua_...", // sent as X-API-Key header when using HTTP/HTTPS});const response = await client.responses.create({@@ -181,9 +188,20 @@ The `usage` object provides token counts and cost information for the request.Connect to a CUA agent proxy server:```typescript-const client = new AgentClient("https://my-agent-server.com:8000");+// Local+const client = new AgentClient("https://my-agent-server.com:8000", { apiKey: "cua_..." });++// Cloud container (port 8443)+const cloud = new AgentClient(+ "https://m-linux-96lcxd2c2k.containers.cloud.trycua.com:8443",+ { apiKey: "cua_..." }+);```+Notes:+- The client sends the API key as `X-API-Key` for HTTP/HTTPS connections.+- Cloud containers listen on `:8443` with HTTPS.+### Peer-to-Peer (WebRTC)Connect directly to another peer using WebRTC:diff --git a/libs/typescript/agent/src/client.ts b/libs/typescript/agent/src/client.tsindex 02b3d191ad..d25e698b9d 100644--- a/libs/typescript/agent/src/client.ts+++ b/libs/typescript/agent/src/client.ts@@ -60,11 +60,16 @@ export class AgentClient {);try {+ const headers: Record<string, string> = {+ "Content-Type": "application/json",+ };+ if (this.options.apiKey) {+ headers["X-API-Key"] = this.options.apiKey;+ }+const response = await fetch(`${this.url}/responses`, {method: "POST",- headers: {- "Content-Type": "application/json",- },+ headers,body: JSON.stringify(request),signal: controller.signal,});diff --git a/libs/typescript/agent/src/types.ts b/libs/typescript/agent/src/types.tsindex 24295225f5..85ea7d968a 100644--- a/libs/typescript/agent/src/types.ts+++ b/libs/typescript/agent/src/types.ts@@ -3,6 +3,8 @@ export type ConnectionType = 'http' | 'https' | 'peer';export interface AgentClientOptions {timeout?: number;retries?: number;+ /** Optional CUA API key to send as X-API-Key header for HTTP requests */+ apiKey?: string;}// Request types matching the Python proxy APIexport interface AgentRequest {