Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
Add nested virtualization to cuabot, add `agent-browser` and `agent-device` to Dockerfile (#1022)
ddupont964127337606
diff --git a/README.md b/README.mdindex d8ae95a0cc..43e5fbd96e 100644--- a/README.md+++ b/README.md@@ -40,6 +40,8 @@ cuabot --type "hello"cuabot --click <x> <y> [button]```+Built-in support for `agent-browser` and `agent-device` (iOS, Android) out of the box.+<div align="center">**[Get Started](https://cua.ai/docs/cuabot/cuabot)** | **[Installation](https://cua.ai/docs/cuabot/install)** | *First spotted at [ClawCon](https://www.claw-con.com/)*diff --git a/libs/cuabot/Dockerfile b/libs/cuabot/Dockerfileindex b264ff622b..2bc919c84e 100644--- a/libs/cuabot/Dockerfile+++ b/libs/cuabot/Dockerfile@@ -19,7 +19,8 @@ RUN apt-get update && apt-get install -y \python3-pip \lsb-release \netcat-openbsd \- git+ git \+ dos2unix# Install xpra from official repo (must be v6+)RUN apt-get update \@@ -31,10 +32,10 @@ RUN apt-get update \&& XPRA_VERSION=$(xpra --version | grep -oP 'v\K[0-9]+' | head -1) \&& if [ "$XPRA_VERSION" -lt 5 ]; then echo "ERROR: xpra version must be >= 5, got v$XPRA_VERSION"; exit 1; fi-# Install Node.js 20.x+# Install Node.js 22.x (agent-device requires >=22)RUN mkdir -p /etc/apt/keyrings \&& curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | gpg --dearmor | tee /etc/apt/keyrings/nodesource.gpg > /dev/null \- && echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_20.x nodistro main" | tee /etc/apt/sources.list.d/nodesource.list \+ && echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_22.x nodistro main" | tee /etc/apt/sources.list.d/nodesource.list \&& apt-get update && apt-get install -y nodejs# Install remaining packages@@ -48,6 +49,48 @@ RUN apt-get install -y \&& DEBIAN_FRONTEND=noninteractive npx playwright install-deps \&& apt-get clean && rm -rf /var/lib/apt/lists/*+# Install Android SDK dependencies for agent-device+RUN apt-get update && apt-get install -y --no-install-recommends \+ openjdk-17-jdk-headless \+ unzip \+ libpulse0 \+ libgl1-mesa-glx \+ libnss3 \+ libxcomposite1 \+ libxcursor1 \+ libxi6 \+ libxtst6 \+ libxrandr2 \+ libasound2 \+ wmctrl \+ && apt-get clean && rm -rf /var/lib/apt/lists/*++# Install agent-device globally+RUN npm install -g agent-device++# Set Android SDK environment variables+ENV ANDROID_SDK_ROOT=/home/user/android-sdk+ENV JAVA_HOME=/usr/lib/jvm/java-17-openjdk-amd64+ENV PATH="${ANDROID_SDK_ROOT}/cmdline-tools/latest/bin:${ANDROID_SDK_ROOT}/platform-tools:${ANDROID_SDK_ROOT}/emulator:${PATH}"++# Install Android SDK command-line tools+RUN mkdir -p "${ANDROID_SDK_ROOT}/cmdline-tools" \+ && cd /tmp \+ && wget -q "https://dl.google.com/android/repository/commandlinetools-linux-11076708_latest.zip" -O cmdline-tools.zip \+ && unzip -q cmdline-tools.zip -d "${ANDROID_SDK_ROOT}/cmdline-tools" \+ && mv "${ANDROID_SDK_ROOT}/cmdline-tools/cmdline-tools" "${ANDROID_SDK_ROOT}/cmdline-tools/latest" \+ && rm cmdline-tools.zip \+ && chown -R user:user "${ANDROID_SDK_ROOT}"++# Install platform-tools, emulator, and platforms (NOT the system image - too large)+RUN yes | sdkmanager --licenses 2>/dev/null || true \+ && sdkmanager "platform-tools" "emulator" "platforms;android-35" \+ && chown -R user:user "${ANDROID_SDK_ROOT}"++# Symlink adb/emulator into PATH+RUN ln -sf /home/user/android-sdk/platform-tools/adb /usr/local/bin/adb \+ && ln -sf /home/user/android-sdk/emulator/emulator /usr/local/bin/emulator+# Install Chromium from xtradeb PPA (avoids snap requirement on Ubuntu 22.04)RUN add-apt-repository ppa:xtradeb/apps -y \&& apt-get update \@@ -82,8 +125,6 @@ RUN agent-browser install# Set Chromium as default browser (needed for some agent CLI onboardings)RUN xdg-settings set default-web-browser chromium.desktop 2>/dev/null || true-## ==== preinstalled apps for the agent ====-# Python plotting librariesRUN pip3 install --no-cache-dir matplotlib numpy pandas seaborn plotly@@ -106,6 +147,14 @@ RUN mkdir -p ~/.local/bin && \printf '#!/bin/bash\nnpx @google/gemini-cli "$@"' > ~/.local/bin/gemini && \chmod +x ~/.local/bin/codex ~/.local/bin/aider ~/.local/bin/openclaw ~/.local/bin/vibe ~/.local/bin/gemini+# Wrapper for agent-device (handles install command for system image download)+COPY --chown=user:user src/scripts/agent-device-wrapper.sh /home/user/.local/bin/agent-device+RUN dos2unix /home/user/.local/bin/agent-device && chmod +x /home/user/.local/bin/agent-device++# Copy entrypoint script+COPY --chown=user:user src/scripts/entrypoint.sh /home/user/.local/bin/entrypoint.sh+RUN dos2unix /home/user/.local/bin/entrypoint.sh && chmod +x /home/user/.local/bin/entrypoint.sh+ENV DISPLAY=:100ENV XDG_RUNTIME_DIR=/tmp/runtime-userENV AGENT_BROWSER_HEADED=1@@ -122,4 +171,4 @@ EXPOSE 10000USER user# Note: dbus-daemon runs as session bus (not system) to avoid needing root-CMD ["sh", "-c", "xpra --version && dbus-daemon --session --fork --address=unix:path=/tmp/dbus-session 2>/dev/null || true && xpra seamless :100 --sharing=yes --no-daemon --bind-tcp=0.0.0.0:10000 --html=on --dpi=96"]+CMD ["/home/user/.local/bin/entrypoint.sh"]diff --git a/libs/cuabot/src/cuabotd.ts b/libs/cuabot/src/cuabotd.tsindex c81fd5d6ee..f09b3f4bb4 100644--- a/libs/cuabot/src/cuabotd.ts+++ b/libs/cuabot/src/cuabotd.ts@@ -34,6 +34,7 @@ const execAsync = promisify(exec);const DEFAULT_PORT = 7842;const CONFIG_DIR = join(homedir(), ".cuabot");const CLAUDE_CONFIG_DIR = join(CONFIG_DIR, "user", ".claude");+const ANDROID_SDK_IMAGES_DIR = join(CONFIG_DIR, "user", "android-sdk", "system-images");const IMAGE_NAME = "trycua/cuabot:latest";const BASE_CONTAINER_NAME = "cuabot-xpra";@@ -49,6 +50,7 @@ export function getSessionName(): string | null {return sessionName;}+// Find an available HTTP server portasync function findAvailableServerPort(preferredPort: number): Promise<number> {const net = await import("net");@@ -121,6 +123,10 @@ function ensureConfigDir() {if (!existsSync(CLAUDE_CONFIG_DIR)) {mkdirSync(CLAUDE_CONFIG_DIR, { recursive: true });}+ // Ensure Android SDK system-images directory exists for volume mount+ if (!existsSync(ANDROID_SDK_IMAGES_DIR)) {+ mkdirSync(ANDROID_SDK_IMAGES_DIR, { recursive: true });+ }}// Docker helpers@@ -214,6 +220,7 @@ async function ensureContainer(): Promise<number> {// Convert Windows path to Docker-compatible path for volume mountconst toDockerPath = (p: string) => p.replace(/\\/g, "/").replace(/^([A-Z]):/, (_, letter) => `/${letter.toLowerCase()}`);const claudeConfigPath = toDockerPath(CLAUDE_CONFIG_DIR);+ const androidSdkImagesPath = toDockerPath(ANDROID_SDK_IMAGES_DIR);const telemetryEnabled = getTelemetryEnabled();const cuabotName = sessionName || "cuabot";@@ -230,9 +237,17 @@ async function ensureContainer(): Promise<number> {"-e", `CUABOT_COLOR=${cuabotColor}`,"--add-host=host.docker.internal:host-gateway","-v", `${claudeConfigPath}:/home/user/.claude`,- IMAGE_NAME,+ "-v", `${androidSdkImagesPath}:/home/user/android-sdk/system-images`,];+ // Add KVM device passthrough for nested virtualization+ if (process.env.CUABOT_NESTED_VIRT === "1") {+ createCmd.push("--device=/dev/kvm");+ console.log("[server] Nested virtualization enabled (--device=/dev/kvm)");+ }++ createCmd.push(IMAGE_NAME);+await execAsync(createCmd.join(" "));await execAsync(`docker start ${getContainerName()}`);@@ -1160,6 +1175,16 @@ export async function stopServer(): Promise<boolean> {} catch { }}+ // Stop and remove Docker container+ try {+ await execAsync(`docker stop ${getContainerName()}`);+ console.log(`Stopped container ${getContainerName()}`);+ } catch { }+ try {+ await execAsync(`docker rm ${getContainerName()}`);+ console.log(`Removed container ${getContainerName()}`);+ } catch { }+// Clean up filestry { unlinkSync(getPidFile()); } catch { }try { unlinkSync(getPortFile()); } catch { }diff --git a/libs/cuabot/src/prompts/SYSTEM.md b/libs/cuabot/src/prompts/SYSTEM.mdindex 74645e151d..676c25c5ef 100644--- a/libs/cuabot/src/prompts/SYSTEM.md+++ b/libs/cuabot/src/prompts/SYSTEM.md@@ -1,175 +1,19 @@You are a computer-use agent with your own Ubuntu 22.04 sandbox (xpra). User sees your desktop in real-time. You have sudo privileges.## Environment-- Node.js 20.x, Python 3.10, pip3, uv (Python package manager)-- Currently installed: chromium, agent-browser, feh, claude-code, x11-apps+- Node.js 22.x, Python 3.10, pip3, uv (Python package manager)+- Currently installed: chromium, agent-browser, agent-device (adb), feh, claude-code, x11-apps- Python libs: matplotlib, numpy, pandas, seaborn, plotly- uv is available for running self-contained Python scripts with inline dependencies+## Agent Tools+- **computer-use**: Desktop automation MCP. Use for all screenshot tasks or desktop app automation.+- **agent-browser**: Browser automation CLI. Use for all web browsing tasks.+- **agent-device**: Android device/emulator automation CLI. Use for Android app testing and automation.+## Guidelines- Keep responses brief- Use agent-browser for browser automation (not chromium-browser)-- Use bash tool to run agent-browser commands+- Use agent-device for Android emulator/device automation+- Use bash tool to run agent-browser and agent-device commands- If asked to open an app that's not installed, install it first using `sudo apt-get install`--## agent-browser--Browser automation CLI for AI agents. Always use `agent-browser` when doing browser tasks, NEVER use `chromium-browser` or `firefox` unless specifically asked.--### Quick Start--```bash-agent-browser open example.com-agent-browser snapshot # Get accessibility tree with refs-agent-browser click @e2 # Click by ref from snapshot-agent-browser fill @e3 "[email protected]" # Fill by ref-agent-browser get text @e1 # Get text by ref-agent-browser screenshot page.png-agent-browser close-```--### Core Commands--```bash-agent-browser open <url> --headed # Navigate to URL (use --headed to show browser)-agent-browser click <sel> # Click element-agent-browser dblclick <sel> # Double-click element-agent-browser type <sel> <text> # Type into element-agent-browser fill <sel> <text> # Clear and fill-agent-browser press <key> # Press key (Enter, Tab, Control+a)-agent-browser keydown <key> # Hold key down-agent-browser keyup <key> # Release key-agent-browser hover <sel> # Hover element-agent-browser select <sel> <val> # Select dropdown option-agent-browser check <sel> # Check checkbox-agent-browser uncheck <sel> # Uncheck checkbox-agent-browser scroll <dir> [px] # Scroll (up/down/left/right)-agent-browser drag <src> <tgt> # Drag and drop-agent-browser screenshot [path] # Take screenshot-agent-browser snapshot # Accessibility tree with refs (best for AI)-agent-browser close # Close browser-```--### Get Info--```bash-agent-browser get text <sel> # Get text content-agent-browser get html <sel> # Get innerHTML-agent-browser get value <sel> # Get input value-agent-browser get attr <sel> <attr> # Get attribute-agent-browser get title # Get page title-agent-browser get url # Get current URL-agent-browser get count <sel> # Count matching elements-```--### Check State--```bash-agent-browser is visible <sel> # Check if visible-agent-browser is enabled <sel> # Check if enabled-agent-browser is checked <sel> # Check if checked-```--### Find Elements (Semantic Locators)--```bash-agent-browser find role <role> <action> [value] # By ARIA role-agent-browser find text <text> <action> # By text content-agent-browser find label <label> <action> [value] # By label-agent-browser find placeholder <ph> <action> [value] # By placeholder-agent-browser find testid <id> <action> [value] # By data-testid-```--**Actions:** `click`, `fill`, `check`, `hover`, `text`--**Examples:**-```bash-agent-browser find role button click --name "Submit"-agent-browser find text "Sign In" click-agent-browser find label "Email" fill "[email protected]"-```--### Wait--```bash-agent-browser wait <selector> # Wait for element to be visible-agent-browser wait <ms> # Wait for time (milliseconds)-agent-browser wait --text "Welcome" # Wait for text to appear-agent-browser wait --url "**/dash" # Wait for URL pattern-agent-browser wait --load networkidle # Wait for load state-```--### Mouse Control--```bash-agent-browser mouse move <x> <y> # Move mouse-agent-browser mouse down [button] # Press button (left/right/middle)-agent-browser mouse up [button] # Release button-agent-browser mouse wheel <dy> [dx] # Scroll wheel-```--### Tabs & Navigation--```bash-agent-browser tab # List tabs-agent-browser tab new [url] # New tab-agent-browser tab <n> # Switch to tab n-agent-browser tab close [n] # Close tab-agent-browser back # Go back-agent-browser forward # Go forward-agent-browser reload # Reload page-```--### Snapshot Options--```bash-agent-browser snapshot # Full accessibility tree-agent-browser snapshot -i # Interactive elements only-agent-browser snapshot -c # Compact (remove empty elements)-agent-browser snapshot -d 3 # Limit depth to 3 levels-agent-browser snapshot -s "#main" # Scope to CSS selector-agent-browser snapshot -i -c -d 5 # Combine options-```--### Selectors--#### Refs (Recommended)--Refs provide deterministic element selection from snapshots:--```bash-# 1. Get snapshot with refs-agent-browser snapshot-# Output:-# - heading "Example Domain" [ref=e1] [level=1]-# - button "Submit" [ref=e2]-# - textbox "Email" [ref=e3]--# 2. Use refs to interact-agent-browser click @e2-agent-browser fill @e3 "[email protected]"-agent-browser get text @e1-```--#### CSS Selectors--```bash-agent-browser click "#id"-agent-browser click ".class"-agent-browser click "div > button"-```--#### Text & XPath--```bash-agent-browser click "text=Submit"-agent-browser click "xpath=//button"-```--### Optimal AI Workflow--1. Open page: `agent-browser open example.com`-2. Get snapshot: `agent-browser snapshot -i`-3. Identify target refs from snapshot-4. Execute actions using refs: `agent-browser click @e2`-5. Re-snapshot after page changesdiff --git a/libs/cuabot/src/scripts/agent-device-wrapper.sh b/libs/cuabot/src/scripts/agent-device-wrapper.shnew file mode 100644index 0000000000..0a3e0f9ab0--- /dev/null+++ b/libs/cuabot/src/scripts/agent-device-wrapper.sh@@ -0,0 +1,179 @@+#!/bin/bash+# Wrapper for agent-device+# System images are downloaded on-demand via "agent-device install"++export ANDROID_SDK_ROOT=/home/user/android-sdk+export JAVA_HOME=/usr/lib/jvm/java-17-openjdk-amd64+export PATH="${ANDROID_SDK_ROOT}/cmdline-tools/latest/bin:${ANDROID_SDK_ROOT}/platform-tools:${ANDROID_SDK_ROOT}/emulator:${PATH}"++# Check if nested virtualization is enabled+check_kvm() {+ if [ ! -e /dev/kvm ]; then+ echo "=============================================="+ echo "WARNING: Nested virtualization is not enabled!"+ echo "=============================================="+ echo ""+ echo "agent-device requires KVM for Android emulation."+ echo "Please restart cuabot with nested virtualization enabled:"+ echo ""+ echo " CUABOT_NESTED_VIRT=1 cuabot"+ echo ""+ echo "=============================================="+ exit 1+ fi+}++# Download and install the system image+install_system_image() {+ check_kvm++ echo "[agent-device] Downloading Android system image..."+ echo "[agent-device] This may take up to 15 minutes depending on your connection."+ echo ""++ # Accept licenses+ yes | sdkmanager --licenses 2>/dev/null || true++ # Download system image+ echo "[agent-device] Downloading system-images;android-35;google_apis_playstore;x86_64..."+ sdkmanager "system-images;android-35;google_apis_playstore;x86_64"++ if [ $? -ne 0 ]; then+ echo ""+ echo "ERROR: Failed to download system image"+ exit 1+ fi++ # Create AVD+ echo ""+ echo "[agent-device] Creating Android Virtual Device..."+ echo "no" | avdmanager create avd -n device \+ -k "system-images;android-35;google_apis_playstore;x86_64" -d "pixel_6" 2>/dev/null || true++ # Configure AVD for container use+ mkdir -p ~/.android/avd/device.avd+ cat > ~/.android/avd/device.avd/config.ini << 'AVDEOF'+avd.ini.encoding=UTF-8+AvdId=device+PlayStore.enabled=true+abi.type=x86_64+disk.dataPartition.size=4G+hw.accelerator.isAccelerated=yes+hw.cpu.arch=x86_64+hw.cpu.ncore=4+hw.gpu.enabled=yes+hw.gpu.mode=swiftshader_indirect+hw.keyboard=yes+hw.lcd.density=420+hw.lcd.height=2400+hw.lcd.width=1080+hw.ramSize=4096+image.sysdir.1=system-images/android-35/google_apis_playstore/x86_64/+tag.display=Google Play+tag.id=google_apis_playstore+vm.heapSize=576+AVDEOF++ echo ""+ echo "[agent-device] Installation complete!"+ echo "[agent-device] You can now use agent-device commands."+ exit 0+}++# Handle install command+if [ "$1" = "install" ]; then+ install_system_image+fi++# Check if nested virtualization is enabled+check_kvm++# Check if Android system images exist+if ! ls /home/user/android-sdk/system-images/android-* 1>/dev/null 2>&1; then+ echo "=============================================="+ echo "ERROR: Android system images not found!"+ echo "=============================================="+ echo ""+ echo "Run the following command to download the system image:"+ echo ""+ echo " agent-device install"+ echo ""+ echo "(This may take up to 15 minutes)"+ echo "=============================================="+ exit 1+fi++# Create AVD if it doesn't exist+if [ ! -d ~/.android/avd/device.avd ]; then+ echo "[agent-device] Creating Android Virtual Device..."++ # Find the installed system image+ SYSTEM_IMAGE=$(ls -d ${ANDROID_SDK_ROOT}/system-images/android-*/google_apis_playstore/x86_64 2>/dev/null | head -1)+ if [ -z "$SYSTEM_IMAGE" ]; then+ SYSTEM_IMAGE=$(ls -d ${ANDROID_SDK_ROOT}/system-images/android-*/google_apis/x86_64 2>/dev/null | head -1)+ fi+ if [ -z "$SYSTEM_IMAGE" ]; then+ SYSTEM_IMAGE=$(ls -d ${ANDROID_SDK_ROOT}/system-images/android-*/default/x86_64 2>/dev/null | head -1)+ fi++ if [ -z "$SYSTEM_IMAGE" ]; then+ echo "ERROR: Could not find a valid system image in ${ANDROID_SDK_ROOT}/system-images/"+ exit 1+ fi++ # Extract the image identifier+ ANDROID_VER=$(echo "$SYSTEM_IMAGE" | grep -oP 'android-\d+' | head -1)+ IMAGE_TYPE=$(basename $(dirname "$SYSTEM_IMAGE"))+ IMAGE_ID="system-images;${ANDROID_VER};${IMAGE_TYPE};x86_64"++ echo "[agent-device] Using system image: $IMAGE_ID"+ echo "no" | avdmanager create avd -n device -k "$IMAGE_ID" -d "pixel_6" 2>/dev/null || true++ # Configure AVD for container use+ mkdir -p ~/.android/avd/device.avd+ IMAGE_SYSDIR=$(echo "$SYSTEM_IMAGE" | sed "s|${ANDROID_SDK_ROOT}/||")++ cat > ~/.android/avd/device.avd/config.ini << AVDEOF+avd.ini.encoding=UTF-8+AvdId=device+PlayStore.enabled=true+abi.type=x86_64+disk.dataPartition.size=4G+hw.accelerator.isAccelerated=yes+hw.cpu.arch=x86_64+hw.cpu.ncore=4+hw.gpu.enabled=yes+hw.gpu.mode=swiftshader_indirect+hw.keyboard=yes+hw.lcd.density=420+hw.lcd.height=2400+hw.lcd.width=1080+hw.ramSize=4096+image.sysdir.1=${IMAGE_SYSDIR}/+tag.display=Google Play+tag.id=${IMAGE_TYPE}+vm.heapSize=576+AVDEOF++ echo "[agent-device] AVD created!"+fi++# Fix KVM permissions if needed+if [ -e /dev/kvm ] && [ ! -w /dev/kvm ]; then+ sudo chmod 666 /dev/kvm 2>/dev/null || true+fi++# Auto-start emulator if no Android devices are connected+if ! ${ANDROID_SDK_ROOT}/platform-tools/adb devices 2>/dev/null | grep -q "emulator\|device$"; then+ echo "[agent-device] Starting Android emulator..."+ nohup ${ANDROID_SDK_ROOT}/emulator/emulator -avd device -no-audio -gpu swiftshader_indirect -no-snapshot -no-boot-anim &>/tmp/emulator.log &+ ${ANDROID_SDK_ROOT}/platform-tools/adb wait-for-device+ echo "[agent-device] Waiting for boot..."+ while [ "$(${ANDROID_SDK_ROOT}/platform-tools/adb shell getprop sys.boot_completed 2>/dev/null)" != "1" ]; do+ sleep 2+ done+ echo "[agent-device] Emulator ready!"+fi++# Run the real agent-device binary+exec /usr/lib/node_modules/agent-device/bin/agent-device.mjs "$@"diff --git a/libs/cuabot/src/scripts/entrypoint.sh b/libs/cuabot/src/scripts/entrypoint.shnew file mode 100644index 0000000000..ba48b496f9--- /dev/null+++ b/libs/cuabot/src/scripts/entrypoint.sh@@ -0,0 +1,18 @@+#!/bin/bash+# Cuabot container entrypoint+# Adds skills on startup (they go to /home/user/.claude which is mounted)++# Add skills in background if not already present+if [ ! -f /home/user/.claude/.skills-installed ]; then+ (+ echo "[entrypoint] Installing skills in background..."+ npx skills add callstackincubator/agent-device --all 2>/dev/null || true+ npx skills add vercel-labs/agent-browser --all 2>/dev/null || true+ touch /home/user/.claude/.skills-installed+ echo "[entrypoint] Skills installed"+ ) &+fi++# Start dbus and xpra+dbus-daemon --session --fork --address=unix:path=/tmp/dbus-session 2>/dev/null || true+exec xpra seamless :100 --sharing=yes --no-daemon --bind-tcp=0.0.0.0:10000 --html=on --dpi=96