Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
added mcp development guide, and vision output to tool calls
Dillon DuPontc4f9da50079f
diff --git a/libs/mcp-server/README.md b/libs/mcp-server/README.mdindex a43076166e..5649cc1984 100644--- a/libs/mcp-server/README.md+++ b/libs/mcp-server/README.md@@ -75,6 +75,41 @@ You can then use the script in your MCP configuration like this:}```+## Development Guide++If you want to develop with the cua-mcp-server directly without installation, you can use this configuration:++```json+{+ "mcpServers": {+ "cua-agent": {+ "command": "/Users/YOURUSERNAME/cua/.venv/bin/python",+ "args": ["-m", "mcp_server.server"],+ "env": {+ "PYTHONPATH": "/Users/YOURUSERNAME/cua/libs/mcp-server:/Users/YOURUSERNAME/cua/libs/agent:/Users/YOURUSERNAME/cua/libs/computer:/Users/YOURUSERNAME/cua/libs/core:/Users/YOURUSERNAME/cua/libs/pylume",+ "CUA_AGENT_LOOP": "UITARS",+ "CUA_MODEL_PROVIDER": "OAICOMPAT",+ "CUA_MODEL_NAME": "ByteDance-Seed/UI-TARS-1.5-7B",+ "CUA_PROVIDER_BASE_URL": "https://****************.us-east-1.aws.endpoints.huggingface.cloud/v1"+ }+ }+ }+}+```++To see the logs:+```+tail -n 20 -f ~/Library/Logs/Claude/mcp*.log+```++This configuration:+- Uses your local Python virtual environment to run the server module directly+- Sets the Python path to include all necessary library dependencies+- Works with Claude Desktop, Cursor, or any other MCP client+- Automatically uses your development code without requiring installation++Just add this to your MCP client's configuration and it will use your local development version of the server.+## Claude Desktop IntegrationTo use with Claude Desktop, add an entry to your Claude Desktop configuration (`claude_desktop_config.json`, typically found in `~/.config/claude-desktop/`):diff --git a/libs/mcp-server/mcp_server/server.py b/libs/mcp-server/mcp_server/server.pyindex f6692f9ac4..f5fb37b286 100644--- a/libs/mcp-server/mcp_server/server.py+++ b/libs/mcp-server/mcp_server/server.py@@ -1,9 +1,10 @@import asyncio+import base64import loggingimport osimport sysimport traceback-from typing import Any, Dict, List, Optional, Union+from typing import Any, Dict, List, Optional, Union, Tuple# Configure logging to output to stderr for debug visibilitylogging.basicConfig(@@ -17,7 +18,7 @@ logger = logging.getLogger("mcp-server")logger.debug("MCP Server module loading...")try:- from mcp.server.fastmcp import Context, FastMCP+ from mcp.server.fastmcp import Context, FastMCP, Imagelogger.debug("Successfully imported FastMCP")except ImportError as e:@@ -49,7 +50,28 @@ def serve() -> FastMCP:server = FastMCP("cua-agent")@server.tool()- async def run_cua_task(ctx: Context, task: str) -> str:+ async def screenshot_cua(ctx: Context) -> Image:+ """+ Take a screenshot of the current screen and return the image.++ Args:+ ctx: The MCP context++ Returns:+ An image resource containing the screenshot+ """+ global global_computer+ if global_computer is None:+ global_computer = Computer(verbosity=logging.INFO)+ await global_computer.run()+ screenshot = await global_computer.interface.screenshot()+ return Image(+ format="png",+ data=screenshot+ )++ @server.tool()+ async def run_cua_task(ctx: Context, task: str) -> Tuple[str, Image]:"""Run a Computer-Use Agent (CUA) task and return the results.@@ -58,7 +80,7 @@ def serve() -> FastMCP:task: The instruction or task for the agent to performReturns:- A string containing the agent's response+ A tuple containing the agent's response and the final screenshot"""global global_computer@@ -76,6 +98,8 @@ def serve() -> FastMCP:loop = AgentLoop.OPENAIelif loop_str == "ANTHROPIC":loop = AgentLoop.ANTHROPIC+ elif loop_str == "UITARS":+ loop = AgentLoop.UITARSelse:loop = AgentLoop.OMNI@@ -107,33 +131,34 @@ def serve() -> FastMCP:full_result = ""async for result in agent.run(task):logger.info(f"Agent step complete: {result.get('id', 'unknown')}")+ ctx.info(f"Agent step complete: {result.get('id', 'unknown')}")# Add response ID to outputfull_result += f"\n[Response ID: {result.get('id', 'unknown')}]\n"-- # Extract and concatenate text responses- if "text" in result:- # Handle both string and dict responses- text_response = result.get("text", "")- if isinstance(text_response, str):- full_result += f"Response: {text_response}\n"- else:- # If it's a dict or other structure, convert to string representation- full_result += f"Response: {str(text_response)}\n"-- # Log detailed information- if "tools" in result:- tools_info = result.get("tools")- logger.debug(f"Tools used: {tools_info}")- full_result += f"\nTools used: {tools_info}\n"++ if "content" in result:+ full_result += f"Response: {result.get('content', '')}\n"# Process output if availableoutputs = result.get("output", [])for output in outputs:output_type = output.get("type")- if output_type == "reasoning":+ if output_type == "message":+ logger.debug(f"Message: {output}")+ content = output.get("content", [])+ for content_part in content:+ if content_part.get("text"):+ full_result += f"\nMessage: {content_part.get('text', '')}\n"+ elif output_type == "reasoning":logger.debug(f"Reasoning: {output}")- full_result += f"\nReasoning: {output.get('content', '')}\n"++ summary_content = output.get("summary", [])+ if summary_content:+ for summary_part in summary_content:+ if summary_part.get("text"):+ full_result += f"\nReasoning: {summary_part.get('text', '')}\n"+ else:+ full_result += f"\nReasoning: {output.get('text', output.get('content', ''))}\n"elif output_type == "computer_call":logger.debug(f"Computer call: {output}")action = output.get("action", "")@@ -144,15 +169,23 @@ def serve() -> FastMCP:full_result += "\n" + "-" * 40 + "\n"logger.info(f"CUA task completed successfully")- return full_result or "Task completed with no text output."+ ctx.info(f"CUA task completed successfully")+ return (+ full_result or "Task completed with no text output.",+ Image(+ format="png",+ data=await global_computer.interface.screenshot()+ )+ )except Exception as e:error_msg = f"Error running CUA task: {str(e)}\n{traceback.format_exc()}"logger.error(error_msg)+ ctx.error(error_msg)return f"Error during task execution: {str(e)}"@server.tool()- async def run_multi_cua_tasks(ctx: Context, tasks: List[str]) -> str:+ async def run_multi_cua_tasks(ctx: Context, tasks: List[str]) -> List:"""Run multiple CUA tasks in sequence and return the combined results.@@ -164,13 +197,15 @@ def serve() -> FastMCP:Combined results from all tasks"""results = []-for i, task in enumerate(tasks):logger.info(f"Running task {i+1}/{len(tasks)}: {task}")- result = await run_cua_task(ctx, task)- results.append(f"Task {i+1}: {task}\nResult: {result}\n")-- return "\n".join(results)+ ctx.info(f"Running task {i+1}/{len(tasks)}: {task}")++ ctx.report_progress(i / len(tasks))+ results.extend(await run_cua_task(ctx, task))+ ctx.report_progress((i + 1) / len(tasks))++ return resultsreturn server