Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
Updated example, cua-core dep, and added --prompt
Dillon DuPont6db6825516cd
diff --git a/examples/agent_examples.py b/examples/agent_examples.pyindex 6257307750..d6a565bf88 100644--- a/examples/agent_examples.py+++ b/examples/agent_examples.py@@ -8,7 +8,7 @@ import signalfrom computer import Computer, VMProviderType# Import the unified agent class and types-from agent import ComputerAgent, LLMProvider, LLM, AgentLoop+from agent import ComputerAgent# Import utility functionsfrom utils import load_dotenv_files, handle_sigint@@ -19,8 +19,8 @@ logger = logging.getLogger(__name__)async def run_agent_example():- """Run example of using the ComputerAgent with OpenAI and Omni provider."""- print("\n=== Example: ComputerAgent with OpenAI and Omni provider ===")+ """Run example of using the ComputerAgent with different models."""+ print("\n=== Example: ComputerAgent with different models ===")try:# Create a local macOS computer@@ -37,28 +37,37 @@ async def run_agent_example():# provider_type=VMProviderType.CLOUD,# )- # Create Computer instance with async context manager+ # Create ComputerAgent with new APIagent = ComputerAgent(- computer=computer,- loop=AgentLoop.OPENAI,- # loop=AgentLoop.ANTHROPIC,- # loop=AgentLoop.UITARS,- # loop=AgentLoop.OMNI,- model=LLM(provider=LLMProvider.OPENAI), # No model name for Operator CUA- # model=LLM(provider=LLMProvider.OPENAI, name="gpt-4o"),- # model=LLM(provider=LLMProvider.ANTHROPIC, name="claude-3-7-sonnet-20250219"),- # model=LLM(provider=LLMProvider.OLLAMA, name="gemma3:4b-it-q4_K_M"),- # model=LLM(provider=LLMProvider.MLXVLM, name="mlx-community/UI-TARS-1.5-7B-4bit"),- # model=LLM(- # provider=LLMProvider.OAICOMPAT,- # name="gemma-3-12b-it",- # provider_base_url="http://localhost:1234/v1", # LM Studio local endpoint- # ),- save_trajectory=True,+ # Supported models:++ # == OpenAI CUA (computer-use-preview) ==+ model="openai/computer-use-preview",++ # == Anthropic CUA (Claude > 3.5) ==+ # model="anthropic/claude-opus-4-20250514",+ # model="anthropic/claude-sonnet-4-20250514",+ # model="anthropic/claude-3-7-sonnet-20250219",+ # model="anthropic/claude-3-5-sonnet-20240620",++ # == UI-TARS ==+ # model="huggingface-local/ByteDance-Seed/UI-TARS-1.5-7B",+ # model="mlx/mlx-community/UI-TARS-1.5-7B-6bit",+ # model="ollama_chat/0000/ui-tars-1.5-7b",++ # == Omniparser + Any LLM ==+ # model="omniparser+anthropic/claude-opus-4-20250514",+ # model="omniparser+ollama_chat/gemma3:12b-it-q4_K_M",++ tools=[computer],only_n_most_recent_images=3,verbosity=logging.DEBUG,+ trajectory_dir="trajectories",+ use_prompt_caching=True,+ max_trajectory_budget=1.0,)+ # Example tasks to demonstrate the agenttasks = ["Look for a repository named trycua/cua on GitHub.","Check the open issues, open the most recent one and read it.",@@ -68,43 +77,35 @@ async def run_agent_example():"Focus on the Composer text area, then write and submit a task to help resolve the GitHub issue.",]+ # Use message-based conversation history+ history = []+for i, task in enumerate(tasks):- print(f"\nExecuting task {i}/{len(tasks)}: {task}")- async for result in agent.run(task):- print("Response ID: ", result.get("id"))-- # Print detailed usage information- usage = result.get("usage")- if usage:- print("\nUsage Details:")- print(f" Input Tokens: {usage.get('input_tokens')}")- if "input_tokens_details" in usage:- print(f" Input Tokens Details: {usage.get('input_tokens_details')}")- print(f" Output Tokens: {usage.get('output_tokens')}")- if "output_tokens_details" in usage:- print(f" Output Tokens Details: {usage.get('output_tokens_details')}")- print(f" Total Tokens: {usage.get('total_tokens')}")-- print("Response Text: ", result.get("text"))-- # Print tools information- tools = result.get("tools")- if tools:- print("\nTools:")- print(tools)-- # Print reasoning and tool call outputs- outputs = result.get("output", [])- for output in outputs:- output_type = output.get("type")- if output_type == "reasoning":- print("\nReasoning Output:")- print(output)- elif output_type == "computer_call":- print("\nTool Call Output:")- print(output)-- print(f"\n✅ Task {i+1}/{len(tasks)} completed: {task}")+ print(f"\nExecuting task {i+1}/{len(tasks)}: {task}")++ # Add user message to history+ history.append({"role": "user", "content": task})++ # Run agent with conversation history+ async for result in agent.run(history, stream=False):+ # Add agent outputs to history+ history += result.get("output", [])++ # Print output for debugging+ for item in result.get("output", []):+ if item.get("type") == "message":+ content = item.get("content", [])+ for content_part in content:+ if content_part.get("text"):+ print(f"Agent: {content_part.get('text')}")+ elif item.get("type") == "computer_call":+ action = item.get("action", {})+ action_type = action.get("type", "")+ print(f"Computer Action: {action_type}({action})")+ elif item.get("type") == "computer_call_output":+ print("Computer Output: [Screenshot/Result]")++ print(f"✅ Task {i+1}/{len(tasks)} completed: {task}")except Exception as e:logger.error(f"Error in run_agent_example: {e}")diff --git a/libs/python/agent/agent/cli.py b/libs/python/agent/agent/cli.pyindex 8656e86f85..9b9aded88a 100644--- a/libs/python/agent/agent/cli.py+++ b/libs/python/agent/agent/cli.py@@ -92,26 +92,30 @@ def print_welcome(model: str, agent_loop: str, container_name: str):async def ainput(prompt: str = ""):return await asyncio.to_thread(input, prompt)-async def chat_loop(agent, model: str, container_name: str):+async def chat_loop(agent, model: str, container_name: str, initial_prompt: str = ""):"""Main chat loop with the agent."""print_welcome(model, agent.agent_loop.__name__, container_name)history = []+ if initial_prompt:+ history.append({"role": "user", "content": initial_prompt})+while True:- # Get user input with prompt- print_colored("> ", end="")- user_input = await ainput()-- if user_input.lower() in ['exit', 'quit', 'q']:- print_colored("\n👋 Goodbye!")- break+ if history[-1].get("role") != "user":+ # Get user input with prompt+ print_colored("> ", end="")+ user_input = await ainput()- if not user_input:- continue-- # Add user message to history- history.append({"role": "user", "content": user_input})+ if user_input.lower() in ['exit', 'quit', 'q']:+ print_colored("\n👋 Goodbye!")+ break++ if not user_input:+ continue++ # Add user message to history+ history.append({"role": "user", "content": user_input})# Stream responses from the agent with spinnerwith yaspin(text="Thinking...", spinner="line", attrs=["dark"]) as spinner:@@ -204,6 +208,12 @@ Examples:action="store_true",help="Enable verbose logging")++ parser.add_argument(+ "-p", "--prompt",+ type=str,+ help="Initial prompt to send to the agent. Leave blank for interactive mode."+ )args = parser.parse_args()@@ -269,9 +279,11 @@ Examples:agent_kwargs = {"model": args.model,"tools": [computer],- "only_n_most_recent_images": args.images,"verbosity": 20 if args.verbose else 30, # DEBUG vs WARNING}++ if args.images > 0:+ agent_kwargs["only_n_most_recent_images"] = args.imagesif args.trajectory:agent_kwargs["trajectory_dir"] = "trajectories"@@ -286,7 +298,7 @@ Examples:agent = ComputerAgent(**agent_kwargs)# Start chat loop- await chat_loop(agent, args.model, container_name)+ await chat_loop(agent, args.model, container_name, args.prompt)diff --git a/libs/python/agent/pyproject.toml b/libs/python/agent/pyproject.tomlindex 683d036c84..be10f72921 100644--- a/libs/python/agent/pyproject.toml+++ b/libs/python/agent/pyproject.toml@@ -20,7 +20,7 @@ dependencies = ["rich>=13.7.1","python-dotenv>=1.0.1","cua-computer>=0.3.0,<0.5.0",- "cua-core>=0.1.0,<0.2.0",+ "cua-core>=0.1.8,<0.2.0","certifi>=2024.2.2","litellm>=1.74.8"]