Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
Feature/agent loop test (#528)
Adamb5e71efcc9b0
diff --git a/.github/workflows/test-cua-models.yml b/.github/workflows/test-cua-models.ymlnew file mode 100644index 0000000000..646bf7ecd0--- /dev/null+++ b/.github/workflows/test-cua-models.yml@@ -0,0 +1,118 @@+name: Test CUA Supporting Models++# This workflow tests all supported CUA models with API keys+# Run manually using workflow_dispatch with test_models=true++on:+ pull_request:+ branches: [ main, master ]+ workflow_dispatch:+ inputs:+ test_models:+ description: "Test all supported models (requires API keys)"+ required: false+ default: "true"+ type: boolean++jobs:+ # Test all CUA models - runs on PRs or when manually triggered+ test-all-models:+ if: ${{ github.event_name == 'pull_request' || inputs.test_models == 'true' }}+ runs-on: ubuntu-latest+ strategy:+ fail-fast: false+ matrix:+ model:+ # Anthropic Claude Models+ # - anthropic/claude-3-5-sonnet-20241022+ # - anthropic/claude-3-7-sonnet-20250219+ # - anthropic/claude-opus-4-20250514+ # - anthropic/claude-sonnet-4-20250514+ # - anthropic/claude-opus-4-1-20250805+ - anthropic/claude-sonnet-4-5-20250929+ # - anthropic/claude-haiku-4-5-20251001++ # OpenAI Models+ # - openai/computer-use-preview++ # Gemini Models+ # - gemini-2.5-computer-use-preview-10-2025++ # GLM-4.5V Models+ # - openrouter/z-ai/glm-4.5v++ # UI-TARS Models+ # - huggingface-local/ByteDance-Seed/UI-TARS-1.5-7B++ # OpenCUA Models+ # - huggingface-local/xlangai/OpenCUA-7B+ # - huggingface-local/xlangai/OpenCUA-32B++ # GTA1 Family Models+ # - huggingface-local/HelloKKMe/GTA1-7B+ # - huggingface-local/HelloKKMe/GTA1-32B+ # - huggingface-local/HelloKKMe/GTA1-72B++ # Holo 1.5 Family Models+ # - huggingface-local/Hcompany/Holo1.5-3B+ # - huggingface-local/Hcompany/Holo1.5-7B+ # - huggingface-local/Hcompany/Holo1.5-72B++ # InternVL 3.5 Family Models+ # - huggingface-local/OpenGVLab/InternVL3_5-1B+ # - huggingface-local/OpenGVLab/InternVL3_5-2B+ # - huggingface-local/OpenGVLab/InternVL3_5-4B+ # - huggingface-local/OpenGVLab/InternVL3_5-8B++ # GLM-4.5V Local+ # - huggingface-local/zai-org/GLM-4.5V++ # Composed Models (Grounding + Planning)+ # - omniparser+anthropic/claude-3-5-sonnet-20241022+ # - omniparser+openai/gpt-4o-mini+ # - moondream3+anthropic/claude-3-5-sonnet-20241022+ # - moondream3+openai/gpt-4o-mini++ steps:+ - name: Checkout repository+ uses: actions/checkout@v4++ - name: Set up Python+ uses: actions/setup-python@v5+ with:+ python-version: "3.12"++ - name: Install system dependencies+ run: |+ sudo apt-get update+ sudo apt-get install -y libgl1-mesa-dri libglib2.0-0++ - name: Install CUA dependencies+ run: |+ pip install --upgrade pip+ pip install -e libs/python/agent -e libs/python/computer+ pip install -e libs/python/core+ pip install "cua-agent[uitars-hf]"+ pip install pytest++ - name: Set up environment variables+ run: |+ echo "ANTHROPIC_API_KEY=${{ secrets.ANTHROPIC_API_KEY }}" >> $GITHUB_ENV+ echo "OPENAI_API_KEY=${{ secrets.OPENAI_API_KEY }}" >> $GITHUB_ENV+ echo "GOOGLE_API_KEY=${{ secrets.GOOGLE_API_KEY }}" >> $GITHUB_ENV+ echo "OPENROUTER_API_KEY=${{ secrets.OPENROUTER_API_KEY }}" >> $GITHUB_ENV++ - name: Test model with agent loop+ run: |+ cd tests/agent_loop_testing+ python agent_test.py --model "${{ matrix.model }}"++ - name: Upload test results+ if: always()+ uses: actions/upload-artifact@v4+ with:+ name: test-results-${{ matrix.model }}+ path: |+ tests/agent_loop_testing/test_images/+ *.log+ retention-days: 7diff --git a/package-lock.json b/package-lock.jsonnew file mode 100644index 0000000000..2df654cb9e--- /dev/null+++ b/package-lock.json@@ -0,0 +1,28 @@+{+ "name": "cua",+ "lockfileVersion": 3,+ "requires": true,+ "packages": {+ "": {+ "devDependencies": {+ "prettier": "^3.6.2"+ }+ },+ "node_modules/prettier": {+ "version": "3.6.2",+ "resolved": "https://registry.npmjs.org/prettier/-/prettier-3.6.2.tgz",+ "integrity": "sha512-I7AIg5boAr5R0FFtJ6rCfD+LFsWHp81dolrFD8S79U9tb8Az2nGrJncnMSnys+bpQJfRUzqs9hnA81OAA3hCuQ==",+ "dev": true,+ "license": "MIT",+ "bin": {+ "prettier": "bin/prettier.cjs"+ },+ "engines": {+ "node": ">=14"+ },+ "funding": {+ "url": "https://github.com/prettier/prettier?sponsor=1"+ }+ }+ }+}diff --git a/tests/agent_loop_testing/README.md b/tests/agent_loop_testing/README.mdnew file mode 100644index 0000000000..48712d901a--- /dev/null+++ b/tests/agent_loop_testing/README.md@@ -0,0 +1,70 @@+# CUA Agent Test++Simple test for CUA ComputerAgent SDK with mock computer.++## Run Test++```bash+python tests/agent_loop_testing/agent_test.py --model anthropic/claude-sonnet-4-20250514+```++## What It Does++- Tests real CUA ComputerAgent SDK+- Uses mock computer (only screenshots, no real actions)+- Agent tries to "Open Safari browser"+- Runs up to 5 iterations+- Shows agent responses and tool calls++## What Passes ✅++- Agent initializes+- Takes screenshots+- Analyzes images+- Makes tool calls+- Runs multiple iterations++## What Fails ❌++- Missing dependencies+- Invalid API keys+- Agent crashes+- Import errors++## Install++```bash+pip install -e libs/python/agent -e libs/python/computer+export ANTHROPIC_API_KEY="your-key"+```++## Example Output++```+🤖 Testing CUA Agent: anthropic/claude-sonnet-4-20250514+==================================================+✅ CUA Agent created+✅ Mock computer ready+🚀 Running agent...++Iteration 1:+ Agent: I'll click on Safari to open it.+ Tool: click {'x': 125, 'y': 975}++Iteration 2:+ Agent: Safari didn't open, let me try again.+ Tool: click {'x': 125, 'y': 975}++Iteration 3:+ Agent: This appears to be a static test environment.++🏁 Stopping after 5 iterations (safety limit)++==================================================+🎉 TEST COMPLETE!+==================================================+✅ Model: anthropic/claude-sonnet-4-20250514+✅ Iterations: 3+✅ Screenshots: 3+✅ Agent executed successfully+```\ No newline at end of filediff --git a/tests/agent_loop_testing/agent_test.py b/tests/agent_loop_testing/agent_test.pynew file mode 100644index 0000000000..b31c824985--- /dev/null+++ b/tests/agent_loop_testing/agent_test.py@@ -0,0 +1,192 @@+#!/usr/bin/env python3+"""+Simple CUA Agent Test++Tests the actual CUA ComputerAgent SDK with a mock computer.+Only provides screenshot functionality - no complex computer actions.+"""++import asyncio+import base64+import sys+from io import BytesIO+from pathlib import Path++from PIL import Image, ImageDraw++# Add project root to path+project_root = Path(__file__).parent.parent.parent+sys.path.insert(0, str(project_root))+++class MockComputer:+ """Mock computer that only provides screenshots."""++ def __init__(self):+ self.action_count = 0+ self._image = self._create_image()++ def _create_image(self) -> str:+ """Create a simple desktop image."""+ img = Image.new("RGB", (1920, 1080), color="lightblue")+ draw = ImageDraw.Draw(img)++ # Draw Safari icon+ draw.rectangle([100, 950, 150, 1000], fill="blue", outline="black", width=2)+ draw.text((110, 960), "Safari", fill="white")++ # Draw Terminal icon+ draw.rectangle([200, 950, 250, 1000], fill="green", outline="black", width=2)+ draw.text((210, 960), "Terminal", fill="white")++ # Convert to base64+ img_bytes = BytesIO()+ img.save(img_bytes, format="PNG")+ return base64.b64encode(img_bytes.getvalue()).decode("utf-8")++ async def screenshot(self) -> str:+ self.action_count += 1+ return self._image++ async def get_dimensions(self) -> tuple[int, int]:+ return (1920, 1080)++ # All other methods are no-ops (required by CUA interface)+ async def click(self, x: int, y: int, button: str = "left") -> None:+ await asyncio.sleep(0.1)++ async def double_click(self, x: int, y: int) -> None:+ await asyncio.sleep(0.1)++ async def scroll(self, x: int, y: int, scroll_x: int, scroll_y: int) -> None:+ await asyncio.sleep(0.1)++ async def type(self, text: str) -> None:+ await asyncio.sleep(0.1)++ async def wait(self, ms: int = 1000) -> None:+ await asyncio.sleep(ms / 1000.0)++ async def move(self, x: int, y: int) -> None:+ await asyncio.sleep(0.1)++ async def keypress(self, keys) -> None:+ await asyncio.sleep(0.1)++ async def drag(self, path) -> None:+ await asyncio.sleep(0.1)++ async def get_current_url(self) -> str:+ return "desktop://mock"++ async def get_environment(self) -> str:+ return "mac"++ # Required abstract methods+ async def left_mouse_down(self, x: int = 0, y: int = 0) -> None:+ await asyncio.sleep(0.1)++ async def left_mouse_up(self, x: int = 0, y: int = 0) -> None:+ await asyncio.sleep(0.1)++ async def right_mouse_down(self, x: int = 0, y: int = 0) -> None:+ await asyncio.sleep(0.1)++ async def right_mouse_up(self, x: int = 0, y: int = 0) -> None:+ await asyncio.sleep(0.1)++ async def mouse_move(self, x: int, y: int) -> None:+ await asyncio.sleep(0.1)++ async def key_down(self, key: str) -> None:+ await asyncio.sleep(0.1)++ async def key_up(self, key: str) -> None:+ await asyncio.sleep(0.1)++ async def type_text(self, text: str) -> None:+ await asyncio.sleep(0.1)+++async def test_cua_agent(model_name: str):+ """Test CUA agent with mock computer."""+ print(f"🤖 Testing CUA Agent: {model_name}")+ print("=" * 50)++ try:+ # Import the real CUA agent+ from agent import ComputerAgent++ # Create mock computer+ mock_computer = MockComputer()++ # Create the real CUA ComputerAgent+ agent = ComputerAgent(model=model_name, tools=[mock_computer], max_trajectory_budget=5.0)++ print("✅ CUA Agent created")+ print("✅ Mock computer ready")+ print("🚀 Running agent...")+ print()++ # Run the agent with a specific task+ message = "Open Safari browser"++ iteration = 0+ async for result in agent.run([{"role": "user", "content": message}]):+ iteration += 1+ print(f"Iteration {iteration}:")++ # Print agent output+ output_items = result.get("output", [])+ if not output_items:+ print(" (No output from agent)")+ else:+ for item in output_items:+ if item["type"] == "message":+ print(f" Agent: {item['content'][0]['text']}")+ elif item["type"] == "tool_call":+ print(f" Tool: {item.get('tool_name')} {item.get('arguments')}")+ else:+ print(f" Unknown output type: {item}")++ # Debug: print full result for empty iterations+ if not output_items:+ print(f" Debug - Full result: {result}")++ # Let the agent decide when to stop (it should try to complete the task)+ # Only stop after 5 iterations to prevent infinite loops+ if iteration >= 5:+ print("🏁 Stopping after 5 iterations (safety limit)")+ break++ print()+ print("=" * 50)+ print("🎉 TEST COMPLETE!")+ print("=" * 50)+ print(f"✅ Model: {model_name}")+ print(f"✅ Iterations: {iteration}")+ print(f"✅ Screenshots: {mock_computer.action_count}")+ print("✅ Agent executed successfully")++ return True++ except ImportError as e:+ print(f"❌ Import error: {e}")+ print("💡 Install CUA: pip install -e libs/python/agent -e libs/python/computer")+ return False+ except Exception as e:+ print(f"❌ Test failed: {e}")+ return False+++if __name__ == "__main__":+ import argparse++ parser = argparse.ArgumentParser(description="Test CUA Agent with mock computer")+ parser.add_argument(+ "--model", default="anthropic/claude-sonnet-4-20250514", help="CUA model to test"+ )+ args = parser.parse_args()++ success = asyncio.run(test_cua_agent(args.model))+ sys.exit(0 if success else 1)