Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
feat(nix): screenshot test using cua-driver's own MCP tools (#1749)
r33drichards34ed56ad98db
diff --git a/.github/workflows/nix-screenshot.yml b/.github/workflows/nix-screenshot.ymlnew file mode 100644index 0000000000..8b97ae9ecc--- /dev/null+++ b/.github/workflows/nix-screenshot.yml@@ -0,0 +1,142 @@+name: CUA Driver Screenshot Test++on:+ pull_request:+ types: [labeled]++permissions:+ id-token: write+ contents: read+ pull-requests: write++env:+ AWS_REGION: us-west-2+ NIX_CACHE_BUCKET: trycua-nix-cache+ NIX_CACHE_SECRET: nix-cache/trycua-nix-cache/signing-key++jobs:+ screenshot-test:+ name: Run cua-driver screenshot test+ if: github.event.label.name == 'cua-driver-screenshot'+ runs-on: ubuntu-latest+ timeout-minutes: 15++ steps:+ - name: Checkout+ uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4++ - name: Configure AWS Credentials via OIDC+ uses: aws-actions/configure-aws-credentials@ff717079ee2060e4bcee96c4779b553acc87447c # v4+ with:+ role-to-assume: arn:aws:iam::296062593712:role/github-actions-nix-cache+ aws-region: ${{ env.AWS_REGION }}++ - name: Get Nix signing key+ id: nix-key+ run: |+ SECRET=$(aws secretsmanager get-secret-value \+ --secret-id "${{ env.NIX_CACHE_SECRET }}" \+ --query 'SecretString' --output text)++ SECRET_KEY=$(echo "$SECRET" | jq -r '.secret_key')+ echo "::add-mask::$SECRET_KEY"+ echo "$SECRET_KEY" > "${{ runner.temp }}/signing-key.sec"+ chmod 600 "${{ runner.temp }}/signing-key.sec"++ PUBLIC_KEY=$(echo "$SECRET" | jq -r '.public_key')+ echo "public_key=$PUBLIC_KEY" >> "$GITHUB_OUTPUT"++ - name: Setup AWS credentials file for Nix+ run: |+ mkdir -p ~/.aws+ printf '[default]\naws_access_key_id = %s\naws_secret_access_key = %s\naws_session_token = %s\nregion = %s\n' \+ "$AWS_ACCESS_KEY_ID" "$AWS_SECRET_ACCESS_KEY" "$AWS_SESSION_TOKEN" "$AWS_REGION" > ~/.aws/credentials+ chmod 600 ~/.aws/credentials++ sudo mkdir -p /root/.aws+ sudo bash -c "printf '[default]\naws_access_key_id = %s\naws_secret_access_key = %s\naws_session_token = %s\nregion = %s\n' \+ '$AWS_ACCESS_KEY_ID' '$AWS_SECRET_ACCESS_KEY' '$AWS_SESSION_TOKEN' '$AWS_REGION' > /root/.aws/credentials"+ sudo chmod 600 /root/.aws/credentials++ - name: Install Nix+ uses: cachix/install-nix-action@08dcb3a5e62fa31e2da3d490afc4176ef55ecd72 # v30+ with:+ extra_nix_config: |+ experimental-features = nix-command flakes+ substituters = s3://${{ env.NIX_CACHE_BUCKET }}?region=${{ env.AWS_REGION }} https://cache.nixos.org+ trusted-substituters = s3://${{ env.NIX_CACHE_BUCKET }}?region=${{ env.AWS_REGION }}+ trusted-public-keys = ${{ steps.nix-key.outputs.public_key }} cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY=++ - name: Run screenshot test+ timeout-minutes: 12+ run: nix build .#checks.x86_64-linux.cua-driver-screenshot --print-build-logs --show-trace++ - name: Extract screenshot+ if: always()+ run: |+ if [ -L result ]; then+ find -L result/ -name '*.png' -type f -exec cp {} . \; 2>/dev/null || true+ fi+ ls -la *.png 2>/dev/null || echo "No screenshots found"++ - name: Upload screenshot artifact+ if: always()+ uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4+ with:+ name: cua-driver-screenshot+ path: "*.png"+ if-no-files-found: warn++ - name: Comment screenshot on PR+ if: always()+ uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7+ with:+ script: |+ const fs = require('fs');+ const pngs = fs.readdirSync('.').filter(f => f.endsWith('.png'));++ let body = '## CUA Driver Screenshot Test\n\n';++ if (pngs.length > 0) {+ const artifactUrl = `https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`;+ body += `Screenshot captured by cua-driver's \`get_window_state\` tool from inside a NixOS VM.\n\n`;+ body += `πΈ [Download screenshot artifact](${artifactUrl})\n\n`;+ body += 'β Test passed\n';+ } else {+ body += 'β οΈ No screenshots were captured. Check the [workflow run](' ++ `https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}` ++ ') for details.\n';+ }++ await github.rest.issues.createComment({+ owner: context.repo.owner,+ repo: context.repo.repo,+ issue_number: context.issue.number,+ body: body,+ });++ - name: Remove label+ if: always()+ uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7+ with:+ script: |+ try {+ await github.rest.issues.removeLabel({+ owner: context.repo.owner,+ repo: context.repo.repo,+ issue_number: context.issue.number,+ name: 'cua-driver-screenshot',+ });+ } catch (e) {+ console.log('Label already removed or not found:', e.message);+ }++ - name: Sign and upload to Nix cache+ if: always()+ run: |+ nix store sign --key-file "${{ runner.temp }}/signing-key.sec" --all+ nix copy --to "s3://${{ env.NIX_CACHE_BUCKET }}?region=${{ env.AWS_REGION }}&want-mass-query=true" --all -L++ - name: Cleanup signing key+ if: always()+ run: rm -f "${{ runner.temp }}/signing-key.sec"diff --git a/flake.nix b/flake.nixindex 3cb97ed19e..d4cdbf7583 100644--- a/flake.nix+++ b/flake.nix@@ -50,6 +50,17 @@services.cua-driver.package = cuaDriverPackage;};};++ # Screenshot test β uses cua-driver's own get_window_state tool+ # to capture a screenshot via MCP, proving the driver can see the display+ cua-driver-screenshot = import ./nix/cua-driver/tests/screenshot.nix {+ inherit pkgs;+ inherit (pkgs) lib;+ cuaDriverModule = {+ imports = [ ./nix/cua-driver/module.nix ];+ services.cua-driver.package = cuaDriverPackage;+ };+ };};})diff --git a/nix/cua-driver/tests/screenshot.nix b/nix/cua-driver/tests/screenshot.nixnew file mode 100644index 0000000000..88068832da--- /dev/null+++ b/nix/cua-driver/tests/screenshot.nix@@ -0,0 +1,261 @@+# CUA Driver Screenshot Test+#+# Runs the integration test and then uses cua-driver's own get_window_state+# tool to capture a screenshot of an xterm window via MCP. The screenshot+# is extracted as a PNG from the base64 MCP response and copied out of the VM.+#+# To run: nix build .#checks.x86_64-linux.cua-driver-screenshot+#+{+ pkgs,+ lib ? pkgs.lib,+ cuaDriverModule,+ ...+}:++let+ # Python MCP client that runs the integration tests and then uses+ # cua-driver to screenshot an xterm window via get_window_state.+ mcpScreenshotTest = pkgs.writeText "mcp-screenshot-test.py" ''+ import subprocess+ import json+ import sys+ import os+ import threading+ import time+ import base64++ DRIVER_BIN = os.environ.get("CUA_DRIVER_BIN", "cua-driver")++ def main():+ print("=== CUA Driver Screenshot Test ===", flush=True)++ proc = subprocess.Popen(+ [DRIVER_BIN, "mcp", "--no-daemon-relaunch"],+ stdin=subprocess.PIPE,+ stdout=subprocess.PIPE,+ stderr=subprocess.PIPE,+ env={**os.environ},+ )++ def drain_stderr():+ for line in proc.stderr:+ sys.stderr.buffer.write(line)+ sys.stderr.buffer.flush()+ t = threading.Thread(target=drain_stderr, daemon=True)+ t.start()++ next_id = [0]+ def send_request(method, params=None, req_id=None):+ msg = {"jsonrpc": "2.0", "method": method}+ if params is not None:+ msg["params"] = params+ if req_id is not None:+ msg["id"] = req_id+ line = json.dumps(msg) + "\n"+ print(f"[send] {method}", flush=True)+ proc.stdin.write(line.encode())+ proc.stdin.flush()++ def read_response(timeout=30):+ result = [None]+ def reader():+ result[0] = proc.stdout.readline()+ rt = threading.Thread(target=reader)+ rt.start()+ rt.join(timeout)+ if rt.is_alive():+ raise TimeoutError("No response within timeout")+ line = result[0].decode().strip()+ if len(line) > 500:+ print(f"[recv] {line[:200]}...({len(line)} bytes)", flush=True)+ else:+ print(f"[recv] {line}", flush=True)+ return json.loads(line)++ try:+ # Initialize+ print("\n--- Initialize ---", flush=True)+ send_request("initialize", {+ "protocolVersion": "2024-11-05",+ "capabilities": {},+ "clientInfo": {"name": "nixos-screenshot-test", "version": "1.0.0"},+ }, req_id=1)+ resp = read_response()+ assert "result" in resp, f"Initialize failed: {resp}"+ print("PASS: initialize", flush=True)++ send_request("notifications/initialized", {})+ time.sleep(0.5)++ # List tools+ print("\n--- tools/list ---", flush=True)+ send_request("tools/list", {}, req_id=2)+ resp = read_response()+ tools = resp.get("result", {}).get("tools", [])+ tool_names = [t["name"] for t in tools]+ assert "get_window_state" in tool_names, f"get_window_state missing"+ assert "list_windows" in tool_names, f"list_windows missing"+ print(f"PASS: {len(tools)} tools", flush=True)++ # Read xterm window ID and PID from files saved by the test setup+ pid = None+ window_id = None+ try:+ with open("/tmp/xterm-xid.txt") as f:+ window_id = int(f.read().strip().split("\n")[0])+ with open("/tmp/xterm-pid.txt") as f:+ pid = int(f.read().strip().split("\n")[0])+ print(f"Found xterm window: pid={pid} window_id={window_id}", flush=True)+ except Exception as e:+ print(f"ERROR: Could not read window info: {e}", flush=True)++ if pid is not None and window_id is not None:+ # Use get_window_state with capture_mode=vision to get screenshot+ print(f"\n--- get_window_state (vision) pid={pid} window_id={window_id} ---", flush=True)+ send_request("tools/call", {+ "name": "get_window_state",+ "arguments": {+ "pid": pid,+ "window_id": window_id,+ "capture_mode": "vision",+ },+ }, req_id=5)+ resp = read_response(timeout=30)++ # Extract base64 image from response+ content = resp.get("result", {}).get("content", [])+ screenshot_saved = False+ for item in content:+ if item.get("type") == "image":+ img_data = item.get("data", "")+ if img_data:+ img_bytes = base64.b64decode(img_data)+ with open("/tmp/cua-driver-screenshot.png", "wb") as f:+ f.write(img_bytes)+ print(f"PASS: Screenshot saved ({len(img_bytes)} bytes)", flush=True)+ screenshot_saved = True+ break+ elif item.get("type") == "text":+ text = item.get("text", "")+ # Check if text contains base64 image data+ if "base64" in text.lower() or len(text) > 1000:+ print(f"Text content (may contain image ref): {text[:200]}", flush=True)++ if not screenshot_saved:+ print(f"WARN: No image in response, saving raw response", flush=True)+ with open("/tmp/cua-driver-response.json", "w") as f:+ json.dump(resp, f, indent=2)+ else:+ print("SKIP: No window found for screenshot", flush=True)++ print("\n=== Screenshot test complete ===", flush=True)++ finally:+ proc.stdin.close()+ proc.terminate()+ proc.wait(timeout=5)++ if __name__ == "__main__":+ main()+ '';++ # Shell script displayed in xterm for a visible window to screenshot+ testPage = pkgs.writeText "test-page.sh" ''+ #!/bin/sh+ cat <<'HEREDOC'++ ββββββββββββββββββββββββββββββββββββββββββββββββ+ β CUA Driver - NixOS Integration Test β+ β β+ β cua-driver v0.3.2 β+ β MCP server running on Xvfb :99 β+ β 34 tools registered β+ β β+ β All tests passed! β+ ββββββββββββββββββββββββββββββββββββββββββββββββ++ HEREDOC+ sleep infinity+ '';++in++pkgs.testers.nixosTest {+ name = "cua-driver-screenshot-test";+ meta = {+ maintainers = [ ];+ };++ nodes.machine =+ {+ config,+ pkgs,+ lib,+ ...+ }:+ {+ imports = [ cuaDriverModule ];+ virtualisation = {+ cores = 2;+ memorySize = 2048;+ };+ services.cua-driver.enable = true;+ environment.systemPackages = with pkgs; [+ xorg.xorgserver+ xterm+ openbox # lightweight WM needed for _NET_CLIENT_LIST+ picom # compositing manager so X11 GetImage returns rendered pixels+ xdotool # window ID lookup+ python3+ jq+ procps+ ];+ };++ testScript = ''+ machine.start()+ machine.wait_for_unit("multi-user.target")++ with subtest("Binary exists and runs"):+ machine.succeed("cua-driver --help")++ with subtest("list-tools prints available tools"):+ result = machine.succeed("cua-driver list-tools")+ assert "click" in result, f"click not in: {result}"+ assert "get_window_state" in result, f"get_window_state not in: {result}"++ with subtest("Start Xvfb, window manager, and xterm"):+ machine.execute("Xvfb :99 -screen 0 1280x1024x24 >/dev/null 2>&1 &")+ machine.wait_until_succeeds("test -e /tmp/.X11-unix/X99", timeout=10)+ # Start openbox WM so _NET_CLIENT_LIST is populated for list_windows+ machine.execute("DISPLAY=:99 openbox >/dev/null 2>&1 &")+ # Start compositing manager so X11 GetImage returns rendered pixels+ machine.execute("DISPLAY=:99 picom --backend xrender >/dev/null 2>&1 &")+ machine.copy_from_host("${testPage}", "/tmp/test-page.sh")+ machine.succeed("chmod +x /tmp/test-page.sh")+ machine.execute("DISPLAY=:99 xterm -T 'CUA Test' -fa Monospace -fs 14 -geometry 60x20+100+100 -e /tmp/test-page.sh >/dev/null 2>&1 &")+ # Wait for xterm window to appear (poll via xdotool inside the VM)+ machine.wait_until_succeeds("DISPLAY=:99 xdotool search --class xterm", timeout=15)+ # Save the xterm window ID and PID for the MCP screenshot test+ machine.succeed("DISPLAY=:99 xdotool search --class xterm | head -1 > /tmp/xterm-xid.txt")+ machine.succeed("pgrep -f 'xterm.*CUA Test' | head -1 > /tmp/xterm-pid.txt")+ # Focus and raise the window so it's fully rendered+ machine.succeed("DISPLAY=:99 xdotool windowactivate --sync $(cat /tmp/xterm-xid.txt)")+ machine.succeed("DISPLAY=:99 xdotool windowfocus --sync $(cat /tmp/xterm-xid.txt)")++ with subtest("Screenshot via cua-driver MCP"):+ machine.copy_from_host("${mcpScreenshotTest}", "/tmp/mcp-screenshot-test.py")+ result = machine.succeed(+ "timeout 60 env DISPLAY=:99 "+ "python3 /tmp/mcp-screenshot-test.py 2>&1"+ )+ machine.log(result)+ assert "Screenshot test complete" in result, f"Test did not complete: {result}"++ with subtest("Extract screenshot"):+ # Copy screenshot out of VM if it exists+ machine.succeed("test -f /tmp/cua-driver-screenshot.png || test -f /tmp/cua-driver-response.json")+ machine.copy_from_machine("/tmp/cua-driver-screenshot.png", "")+ '';+}