Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

added mcp development guide, and vision output to tool calls

Dillon DuPontc4f9da50079f
diff --git a/libs/mcp-server/README.md b/libs/mcp-server/README.md
index a43076166e..5649cc1984 100644
--- a/libs/mcp-server/README.md
+++ b/libs/mcp-server/README.md
@@ -75,6 +75,41 @@ You can then use the script in your MCP configuration like this:
}
```
+## Development Guide
+
+If you want to develop with the cua-mcp-server directly without installation, you can use this configuration:
+
+```json
+{
+ "mcpServers": {
+ "cua-agent": {
+ "command": "/Users/YOURUSERNAME/cua/.venv/bin/python",
+ "args": ["-m", "mcp_server.server"],
+ "env": {
+ "PYTHONPATH": "/Users/YOURUSERNAME/cua/libs/mcp-server:/Users/YOURUSERNAME/cua/libs/agent:/Users/YOURUSERNAME/cua/libs/computer:/Users/YOURUSERNAME/cua/libs/core:/Users/YOURUSERNAME/cua/libs/pylume",
+ "CUA_AGENT_LOOP": "UITARS",
+ "CUA_MODEL_PROVIDER": "OAICOMPAT",
+ "CUA_MODEL_NAME": "ByteDance-Seed/UI-TARS-1.5-7B",
+ "CUA_PROVIDER_BASE_URL": "https://****************.us-east-1.aws.endpoints.huggingface.cloud/v1"
+ }
+ }
+ }
+}
+```
+
+To see the logs:
+```
+tail -n 20 -f ~/Library/Logs/Claude/mcp*.log
+```
+
+This configuration:
+- Uses your local Python virtual environment to run the server module directly
+- Sets the Python path to include all necessary library dependencies
+- Works with Claude Desktop, Cursor, or any other MCP client
+- Automatically uses your development code without requiring installation
+
+Just add this to your MCP client's configuration and it will use your local development version of the server.
+
## Claude Desktop Integration
To use with Claude Desktop, add an entry to your Claude Desktop configuration (`claude_desktop_config.json`, typically found in `~/.config/claude-desktop/`):
diff --git a/libs/mcp-server/mcp_server/server.py b/libs/mcp-server/mcp_server/server.py
index f6692f9ac4..f5fb37b286 100644
--- a/libs/mcp-server/mcp_server/server.py
+++ b/libs/mcp-server/mcp_server/server.py
@@ -1,9 +1,10 @@
import asyncio
+import base64
import logging
import os
import sys
import traceback
-from typing import Any, Dict, List, Optional, Union
+from typing import Any, Dict, List, Optional, Union, Tuple
# Configure logging to output to stderr for debug visibility
logging.basicConfig(
@@ -17,7 +18,7 @@ logger = logging.getLogger("mcp-server")
logger.debug("MCP Server module loading...")
try:
- from mcp.server.fastmcp import Context, FastMCP
+ from mcp.server.fastmcp import Context, FastMCP, Image
logger.debug("Successfully imported FastMCP")
except ImportError as e:
@@ -49,7 +50,28 @@ def serve() -> FastMCP:
server = FastMCP("cua-agent")
@server.tool()
- async def run_cua_task(ctx: Context, task: str) -> str:
+ async def screenshot_cua(ctx: Context) -> Image:
+ """
+ Take a screenshot of the current screen and return the image.
+
+ Args:
+ ctx: The MCP context
+
+ Returns:
+ An image resource containing the screenshot
+ """
+ global global_computer
+ if global_computer is None:
+ global_computer = Computer(verbosity=logging.INFO)
+ await global_computer.run()
+ screenshot = await global_computer.interface.screenshot()
+ return Image(
+ format="png",
+ data=screenshot
+ )
+
+ @server.tool()
+ async def run_cua_task(ctx: Context, task: str) -> Tuple[str, Image]:
"""
Run a Computer-Use Agent (CUA) task and return the results.
@@ -58,7 +80,7 @@ def serve() -> FastMCP:
task: The instruction or task for the agent to perform
Returns:
- A string containing the agent's response
+ A tuple containing the agent's response and the final screenshot
"""
global global_computer
@@ -76,6 +98,8 @@ def serve() -> FastMCP:
loop = AgentLoop.OPENAI
elif loop_str == "ANTHROPIC":
loop = AgentLoop.ANTHROPIC
+ elif loop_str == "UITARS":
+ loop = AgentLoop.UITARS
else:
loop = AgentLoop.OMNI
@@ -107,33 +131,34 @@ def serve() -> FastMCP:
full_result = ""
async for result in agent.run(task):
logger.info(f"Agent step complete: {result.get('id', 'unknown')}")
+ ctx.info(f"Agent step complete: {result.get('id', 'unknown')}")
# Add response ID to output
full_result += f"\n[Response ID: {result.get('id', 'unknown')}]\n"
-
- # Extract and concatenate text responses
- if "text" in result:
- # Handle both string and dict responses
- text_response = result.get("text", "")
- if isinstance(text_response, str):
- full_result += f"Response: {text_response}\n"
- else:
- # If it's a dict or other structure, convert to string representation
- full_result += f"Response: {str(text_response)}\n"
-
- # Log detailed information
- if "tools" in result:
- tools_info = result.get("tools")
- logger.debug(f"Tools used: {tools_info}")
- full_result += f"\nTools used: {tools_info}\n"
+
+ if "content" in result:
+ full_result += f"Response: {result.get('content', '')}\n"
# Process output if available
outputs = result.get("output", [])
for output in outputs:
output_type = output.get("type")
- if output_type == "reasoning":
+ if output_type == "message":
+ logger.debug(f"Message: {output}")
+ content = output.get("content", [])
+ for content_part in content:
+ if content_part.get("text"):
+ full_result += f"\nMessage: {content_part.get('text', '')}\n"
+ elif output_type == "reasoning":
logger.debug(f"Reasoning: {output}")
- full_result += f"\nReasoning: {output.get('content', '')}\n"
+
+ summary_content = output.get("summary", [])
+ if summary_content:
+ for summary_part in summary_content:
+ if summary_part.get("text"):
+ full_result += f"\nReasoning: {summary_part.get('text', '')}\n"
+ else:
+ full_result += f"\nReasoning: {output.get('text', output.get('content', ''))}\n"
elif output_type == "computer_call":
logger.debug(f"Computer call: {output}")
action = output.get("action", "")
@@ -144,15 +169,23 @@ def serve() -> FastMCP:
full_result += "\n" + "-" * 40 + "\n"
logger.info(f"CUA task completed successfully")
- return full_result or "Task completed with no text output."
+ ctx.info(f"CUA task completed successfully")
+ return (
+ full_result or "Task completed with no text output.",
+ Image(
+ format="png",
+ data=await global_computer.interface.screenshot()
+ )
+ )
except Exception as e:
error_msg = f"Error running CUA task: {str(e)}\n{traceback.format_exc()}"
logger.error(error_msg)
+ ctx.error(error_msg)
return f"Error during task execution: {str(e)}"
@server.tool()
- async def run_multi_cua_tasks(ctx: Context, tasks: List[str]) -> str:
+ async def run_multi_cua_tasks(ctx: Context, tasks: List[str]) -> List:
"""
Run multiple CUA tasks in sequence and return the combined results.
@@ -164,13 +197,15 @@ def serve() -> FastMCP:
Combined results from all tasks
"""
results = []
-
for i, task in enumerate(tasks):
logger.info(f"Running task {i+1}/{len(tasks)}: {task}")
- result = await run_cua_task(ctx, task)
- results.append(f"Task {i+1}: {task}\nResult: {result}\n")
-
- return "\n".join(results)
+ ctx.info(f"Running task {i+1}/{len(tasks)}: {task}")
+
+ ctx.report_progress(i / len(tasks))
+ results.extend(await run_cua_task(ctx, task))
+ ctx.report_progress((i + 1) / len(tasks))
+
+ return results
return server