Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
update mcp server to cua-agent==0.4.x
Dillon DuPont96e4b7586ab4
diff --git a/libs/python/mcp-server/README.md b/libs/python/mcp-server/README.mdindex 3f3c8bbb92..a94da8a7b3 100644--- a/libs/python/mcp-server/README.md+++ b/libs/python/mcp-server/README.md@@ -16,6 +16,21 @@</div>**cua-mcp-server** is a MCP server for the Computer-Use Agent (CUA), allowing you to run CUA through Claude Desktop or other MCP clients.++## LiteLLM Integration++This MCP server features comprehensive liteLLM integration, allowing you to use any supported LLM provider with a simple model string configuration.++- **Unified Configuration**: Use a single `CUA_MODEL_NAME` environment variable with a model string+- **Automatic Provider Detection**: The agent automatically detects the provider and capabilities from the model string+- **Extensive Provider Support**: Works with Anthropic, OpenAI, local models, and any liteLLM-compatible provider++### Model String Examples:+- **Anthropic**: `"anthropic/claude-3-5-sonnet-20241022"`+- **OpenAI**: `"openai/computer-use-preview"`+- **UI-TARS**: `"huggingface-local/ByteDance-Seed/UI-TARS-1.5-7B"`+- **Omni + Any LiteLLM**: `"omniparser+litellm/gpt-4o"`, `"omniparser+litellm/claude-3-haiku"`, `"omniparser+ollama_chat/gemma3"`+### Get started with Agent## Prerequisites@@ -65,10 +80,7 @@ You can then use the script in your MCP configuration like this:"command": "/bin/bash","args": ["~/.cua/start_mcp_server.sh"],"env": {- "CUA_AGENT_LOOP": "OMNI",- "CUA_MODEL_PROVIDER": "ANTHROPIC",- "CUA_MODEL_NAME": "claude-3-7-sonnet-20250219",- "CUA_PROVIDER_API_KEY": "your-api-key"+ "CUA_MODEL_NAME": "anthropic/claude-3-5-sonnet-20241022"}}}@@ -86,11 +98,7 @@ If you want to develop with the cua-mcp-server directly without installation, yo"command": "/bin/bash","args": ["~/cua/libs/python/mcp-server/scripts/start_mcp_server.sh"],"env": {- "CUA_AGENT_LOOP": "UITARS",- "CUA_MODEL_PROVIDER": "OAICOMPAT",- "CUA_MODEL_NAME": "ByteDance-Seed/UI-TARS-1.5-7B",- "CUA_PROVIDER_BASE_URL": "https://****************.us-east-1.aws.endpoints.huggingface.cloud/v1",- "CUA_PROVIDER_API_KEY": "your-api-key"+ "CUA_MODEL_NAME": "huggingface-local/ByteDance-Seed/UI-TARS-1.5-7B"}}}@@ -142,10 +150,7 @@ The server is configured using environment variables (can be set in the Claude D| Variable | Description | Default ||----------|-------------|---------|-| `CUA_AGENT_LOOP` | Agent loop to use (OPENAI, ANTHROPIC, UITARS, OMNI) | OMNI |-| `CUA_MODEL_PROVIDER` | Model provider (ANTHROPIC, OPENAI, OLLAMA, OAICOMPAT) | ANTHROPIC |-| `CUA_MODEL_NAME` | Model name to use | None (provider default) |-| `CUA_PROVIDER_BASE_URL` | Base URL for provider API | None |+| `CUA_MODEL_NAME` | Model string (e.g., "anthropic/claude-3-5-sonnet-20241022", "openai/computer-use-preview", "huggingface-local/ByteDance-Seed/UI-TARS-1.5-7B", "omniparser+litellm/gpt-4o", "omniparser+ollama_chat/gemma3") | anthropic/claude-3-5-sonnet-20241022 || `CUA_MAX_IMAGES` | Maximum number of images to keep in context | 3 |## Available Toolsdiff --git a/libs/python/mcp-server/mcp_server/server.py b/libs/python/mcp-server/mcp_server/server.pyindex 03971cb6ac..73996d5efc 100644--- a/libs/python/mcp-server/mcp_server/server.py+++ b/libs/python/mcp-server/mcp_server/server.py@@ -3,6 +3,7 @@ import base64import loggingimport osimport sys+from tabnanny import verboseimport tracebackfrom typing import Any, Dict, List, Optional, Union, Tuple@@ -28,7 +29,7 @@ except ImportError as e:try:from computer import Computer- from agent import ComputerAgent, LLMProvider, LLM, AgentLoop+ from agent import ComputerAgentlogger.debug("Successfully imported Computer and Agent modules")except ImportError as e:@@ -92,49 +93,27 @@ def serve() -> FastMCP:global_computer = Computer(verbosity=logging.INFO)await global_computer.run()- # Determine which loop to use- loop_str = os.getenv("CUA_AGENT_LOOP", "OMNI")- loop = getattr(AgentLoop, loop_str)-- # Determine provider- provider_str = os.getenv("CUA_MODEL_PROVIDER", "ANTHROPIC")- provider = getattr(LLMProvider, provider_str)-- # Get model name (if specified)- model_name = os.getenv("CUA_MODEL_NAME", None)-- # Get base URL for provider (if needed)- provider_base_url = os.getenv("CUA_PROVIDER_BASE_URL", None)-- # Get api key for provider (if needed)- api_key = os.getenv("CUA_PROVIDER_API_KEY", None)+ # Get model name - this now determines the loop and provider+ model_name = os.getenv("CUA_MODEL_NAME", "anthropic/claude-3-5-sonnet-20241022")++ logger.info(f"Using model: {model_name}")- # Create agent with the specified configuration+ # Create agent with the new v0.4.x APIagent = ComputerAgent(- computer=global_computer,- loop=loop,- model=LLM(- provider=provider,- name=model_name,- provider_base_url=provider_base_url,- ),- api_key=api_key,- save_trajectory=False,+ model=model_name,only_n_most_recent_images=int(os.getenv("CUA_MAX_IMAGES", "3")),verbosity=logging.INFO,+ tools=[global_computer])+ # Create messages in the new v0.4.x format+ messages = [{"role": "user", "content": task}]+# Collect all resultsfull_result = ""- async for result in agent.run(task):- logger.info(f"Agent step complete: {result.get('id', 'unknown')}")- ctx.info(f"Agent step complete: {result.get('id', 'unknown')}")-- # Add response ID to output- full_result += f"\n[Response ID: {result.get('id', 'unknown')}]\n"-- if "content" in result:- full_result += f"Response: {result.get('content', '')}\n"+ async for result in agent.run(messages):+ logger.info(f"Agent processing step")+ ctx.info(f"Agent processing step")# Process output if availableoutputs = result.get("output", [])@@ -145,25 +124,23 @@ def serve() -> FastMCP:content = output.get("content", [])for content_part in content:if content_part.get("text"):- full_result += f"\nMessage: {content_part.get('text', '')}\n"- elif output_type == "reasoning":- logger.debug(f"Reasoning: {output}")-- summary_content = output.get("summary", [])- if summary_content:- for summary_part in summary_content:- if summary_part.get("text"):- full_result += f"\nReasoning: {summary_part.get('text', '')}\n"+ full_result += f"Message: {content_part.get('text', '')}\n"+ elif output_type == "tool_use":+ logger.debug(f"Tool use: {output}")+ tool_name = output.get("name", "")+ full_result += f"Tool: {tool_name}\n"+ elif output_type == "tool_result":+ logger.debug(f"Tool result: {output}")+ result_content = output.get("content", "")+ if isinstance(result_content, list):+ for item in result_content:+ if item.get("type") == "text":+ full_result += f"Result: {item.get('text', '')}\n"else:- full_result += f"\nReasoning: {output.get('text', output.get('content', ''))}\n"- elif output_type == "computer_call":- logger.debug(f"Computer call: {output}")- action = output.get("action", "")- result_value = output.get("result", "")- full_result += f"\nComputer Action: {action}\nResult: {result_value}\n"+ full_result += f"Result: {result_content}\n"# Add separator between steps- full_result += "\n" + "-" * 40 + "\n"+ full_result += "\n" + "-" * 20 + "\n"logger.info(f"CUA task completed successfully")ctx.info(f"CUA task completed successfully")@@ -179,7 +156,21 @@ def serve() -> FastMCP:error_msg = f"Error running CUA task: {str(e)}\n{traceback.format_exc()}"logger.error(error_msg)ctx.error(error_msg)- return f"Error during task execution: {str(e)}"+ # Return tuple with error message and a screenshot if possible+ try:+ if global_computer is not None:+ screenshot = await global_computer.interface.screenshot()+ return (+ f"Error during task execution: {str(e)}",+ Image(format="png", data=screenshot)+ )+ except:+ pass+ # If we can't get a screenshot, return a placeholder+ return (+ f"Error during task execution: {str(e)}",+ Image(format="png", data=b"")+ )@server.tool()async def run_multi_cua_tasks(ctx: Context, tasks: List[str]) -> List:diff --git a/libs/python/mcp-server/pyproject.toml b/libs/python/mcp-server/pyproject.tomlindex ed2ad435a8..f80a1b6bc1 100644--- a/libs/python/mcp-server/pyproject.toml+++ b/libs/python/mcp-server/pyproject.toml@@ -13,8 +13,8 @@ authors = []dependencies = ["mcp>=1.6.0,<2.0.0",- "cua-agent[all]>=0.3.0,<0.4.0",- "cua-computer>=0.3.0,<0.4.0",+ "cua-agent[all]>=0.4.0,<0.5.0",+ "cua-computer>=0.4.0,<0.5.0",][project.scripts]