Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

feat(nix): screenshot test using cua-driver's own MCP tools (#1749)

r33drichards34ed56ad98db
diff --git a/.github/workflows/nix-screenshot.yml b/.github/workflows/nix-screenshot.yml
new file mode 100644
index 0000000000..8b97ae9ecc
--- /dev/null
+++ b/.github/workflows/nix-screenshot.yml
@@ -0,0 +1,142 @@
+name: CUA Driver Screenshot Test
+
+on:
+ pull_request:
+ types: [labeled]
+
+permissions:
+ id-token: write
+ contents: read
+ pull-requests: write
+
+env:
+ AWS_REGION: us-west-2
+ NIX_CACHE_BUCKET: trycua-nix-cache
+ NIX_CACHE_SECRET: nix-cache/trycua-nix-cache/signing-key
+
+jobs:
+ screenshot-test:
+ name: Run cua-driver screenshot test
+ if: github.event.label.name == 'cua-driver-screenshot'
+ runs-on: ubuntu-latest
+ timeout-minutes: 15
+
+ steps:
+ - name: Checkout
+ uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
+
+ - name: Configure AWS Credentials via OIDC
+ uses: aws-actions/configure-aws-credentials@ff717079ee2060e4bcee96c4779b553acc87447c # v4
+ with:
+ role-to-assume: arn:aws:iam::296062593712:role/github-actions-nix-cache
+ aws-region: ${{ env.AWS_REGION }}
+
+ - name: Get Nix signing key
+ id: nix-key
+ run: |
+ SECRET=$(aws secretsmanager get-secret-value \
+ --secret-id "${{ env.NIX_CACHE_SECRET }}" \
+ --query 'SecretString' --output text)
+
+ SECRET_KEY=$(echo "$SECRET" | jq -r '.secret_key')
+ echo "::add-mask::$SECRET_KEY"
+ echo "$SECRET_KEY" > "${{ runner.temp }}/signing-key.sec"
+ chmod 600 "${{ runner.temp }}/signing-key.sec"
+
+ PUBLIC_KEY=$(echo "$SECRET" | jq -r '.public_key')
+ echo "public_key=$PUBLIC_KEY" >> "$GITHUB_OUTPUT"
+
+ - name: Setup AWS credentials file for Nix
+ run: |
+ mkdir -p ~/.aws
+ printf '[default]\naws_access_key_id = %s\naws_secret_access_key = %s\naws_session_token = %s\nregion = %s\n' \
+ "$AWS_ACCESS_KEY_ID" "$AWS_SECRET_ACCESS_KEY" "$AWS_SESSION_TOKEN" "$AWS_REGION" > ~/.aws/credentials
+ chmod 600 ~/.aws/credentials
+
+ sudo mkdir -p /root/.aws
+ sudo bash -c "printf '[default]\naws_access_key_id = %s\naws_secret_access_key = %s\naws_session_token = %s\nregion = %s\n' \
+ '$AWS_ACCESS_KEY_ID' '$AWS_SECRET_ACCESS_KEY' '$AWS_SESSION_TOKEN' '$AWS_REGION' > /root/.aws/credentials"
+ sudo chmod 600 /root/.aws/credentials
+
+ - name: Install Nix
+ uses: cachix/install-nix-action@08dcb3a5e62fa31e2da3d490afc4176ef55ecd72 # v30
+ with:
+ extra_nix_config: |
+ experimental-features = nix-command flakes
+ substituters = s3://${{ env.NIX_CACHE_BUCKET }}?region=${{ env.AWS_REGION }} https://cache.nixos.org
+ trusted-substituters = s3://${{ env.NIX_CACHE_BUCKET }}?region=${{ env.AWS_REGION }}
+ trusted-public-keys = ${{ steps.nix-key.outputs.public_key }} cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY=
+
+ - name: Run screenshot test
+ timeout-minutes: 12
+ run: nix build .#checks.x86_64-linux.cua-driver-screenshot --print-build-logs --show-trace
+
+ - name: Extract screenshot
+ if: always()
+ run: |
+ if [ -L result ]; then
+ find -L result/ -name '*.png' -type f -exec cp {} . \; 2>/dev/null || true
+ fi
+ ls -la *.png 2>/dev/null || echo "No screenshots found"
+
+ - name: Upload screenshot artifact
+ if: always()
+ uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
+ with:
+ name: cua-driver-screenshot
+ path: "*.png"
+ if-no-files-found: warn
+
+ - name: Comment screenshot on PR
+ if: always()
+ uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7
+ with:
+ script: |
+ const fs = require('fs');
+ const pngs = fs.readdirSync('.').filter(f => f.endsWith('.png'));
+
+ let body = '## CUA Driver Screenshot Test\n\n';
+
+ if (pngs.length > 0) {
+ const artifactUrl = `https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`;
+ body += `Screenshot captured by cua-driver's \`get_window_state\` tool from inside a NixOS VM.\n\n`;
+ body += `πŸ“Έ [Download screenshot artifact](${artifactUrl})\n\n`;
+ body += 'βœ… Test passed\n';
+ } else {
+ body += '⚠️ No screenshots were captured. Check the [workflow run](' +
+ `https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}` +
+ ') for details.\n';
+ }
+
+ await github.rest.issues.createComment({
+ owner: context.repo.owner,
+ repo: context.repo.repo,
+ issue_number: context.issue.number,
+ body: body,
+ });
+
+ - name: Remove label
+ if: always()
+ uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7
+ with:
+ script: |
+ try {
+ await github.rest.issues.removeLabel({
+ owner: context.repo.owner,
+ repo: context.repo.repo,
+ issue_number: context.issue.number,
+ name: 'cua-driver-screenshot',
+ });
+ } catch (e) {
+ console.log('Label already removed or not found:', e.message);
+ }
+
+ - name: Sign and upload to Nix cache
+ if: always()
+ run: |
+ nix store sign --key-file "${{ runner.temp }}/signing-key.sec" --all
+ nix copy --to "s3://${{ env.NIX_CACHE_BUCKET }}?region=${{ env.AWS_REGION }}&want-mass-query=true" --all -L
+
+ - name: Cleanup signing key
+ if: always()
+ run: rm -f "${{ runner.temp }}/signing-key.sec"
diff --git a/flake.nix b/flake.nix
index 3cb97ed19e..d4cdbf7583 100644
--- a/flake.nix
+++ b/flake.nix
@@ -50,6 +50,17 @@
services.cua-driver.package = cuaDriverPackage;
};
};
+
+ # Screenshot test β€” uses cua-driver's own get_window_state tool
+ # to capture a screenshot via MCP, proving the driver can see the display
+ cua-driver-screenshot = import ./nix/cua-driver/tests/screenshot.nix {
+ inherit pkgs;
+ inherit (pkgs) lib;
+ cuaDriverModule = {
+ imports = [ ./nix/cua-driver/module.nix ];
+ services.cua-driver.package = cuaDriverPackage;
+ };
+ };
};
}
)
diff --git a/nix/cua-driver/tests/screenshot.nix b/nix/cua-driver/tests/screenshot.nix
new file mode 100644
index 0000000000..88068832da
--- /dev/null
+++ b/nix/cua-driver/tests/screenshot.nix
@@ -0,0 +1,261 @@
+# CUA Driver Screenshot Test
+#
+# Runs the integration test and then uses cua-driver's own get_window_state
+# tool to capture a screenshot of an xterm window via MCP. The screenshot
+# is extracted as a PNG from the base64 MCP response and copied out of the VM.
+#
+# To run: nix build .#checks.x86_64-linux.cua-driver-screenshot
+#
+{
+ pkgs,
+ lib ? pkgs.lib,
+ cuaDriverModule,
+ ...
+}:
+
+let
+ # Python MCP client that runs the integration tests and then uses
+ # cua-driver to screenshot an xterm window via get_window_state.
+ mcpScreenshotTest = pkgs.writeText "mcp-screenshot-test.py" ''
+ import subprocess
+ import json
+ import sys
+ import os
+ import threading
+ import time
+ import base64
+
+ DRIVER_BIN = os.environ.get("CUA_DRIVER_BIN", "cua-driver")
+
+ def main():
+ print("=== CUA Driver Screenshot Test ===", flush=True)
+
+ proc = subprocess.Popen(
+ [DRIVER_BIN, "mcp", "--no-daemon-relaunch"],
+ stdin=subprocess.PIPE,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE,
+ env={**os.environ},
+ )
+
+ def drain_stderr():
+ for line in proc.stderr:
+ sys.stderr.buffer.write(line)
+ sys.stderr.buffer.flush()
+ t = threading.Thread(target=drain_stderr, daemon=True)
+ t.start()
+
+ next_id = [0]
+ def send_request(method, params=None, req_id=None):
+ msg = {"jsonrpc": "2.0", "method": method}
+ if params is not None:
+ msg["params"] = params
+ if req_id is not None:
+ msg["id"] = req_id
+ line = json.dumps(msg) + "\n"
+ print(f"[send] {method}", flush=True)
+ proc.stdin.write(line.encode())
+ proc.stdin.flush()
+
+ def read_response(timeout=30):
+ result = [None]
+ def reader():
+ result[0] = proc.stdout.readline()
+ rt = threading.Thread(target=reader)
+ rt.start()
+ rt.join(timeout)
+ if rt.is_alive():
+ raise TimeoutError("No response within timeout")
+ line = result[0].decode().strip()
+ if len(line) > 500:
+ print(f"[recv] {line[:200]}...({len(line)} bytes)", flush=True)
+ else:
+ print(f"[recv] {line}", flush=True)
+ return json.loads(line)
+
+ try:
+ # Initialize
+ print("\n--- Initialize ---", flush=True)
+ send_request("initialize", {
+ "protocolVersion": "2024-11-05",
+ "capabilities": {},
+ "clientInfo": {"name": "nixos-screenshot-test", "version": "1.0.0"},
+ }, req_id=1)
+ resp = read_response()
+ assert "result" in resp, f"Initialize failed: {resp}"
+ print("PASS: initialize", flush=True)
+
+ send_request("notifications/initialized", {})
+ time.sleep(0.5)
+
+ # List tools
+ print("\n--- tools/list ---", flush=True)
+ send_request("tools/list", {}, req_id=2)
+ resp = read_response()
+ tools = resp.get("result", {}).get("tools", [])
+ tool_names = [t["name"] for t in tools]
+ assert "get_window_state" in tool_names, f"get_window_state missing"
+ assert "list_windows" in tool_names, f"list_windows missing"
+ print(f"PASS: {len(tools)} tools", flush=True)
+
+ # Read xterm window ID and PID from files saved by the test setup
+ pid = None
+ window_id = None
+ try:
+ with open("/tmp/xterm-xid.txt") as f:
+ window_id = int(f.read().strip().split("\n")[0])
+ with open("/tmp/xterm-pid.txt") as f:
+ pid = int(f.read().strip().split("\n")[0])
+ print(f"Found xterm window: pid={pid} window_id={window_id}", flush=True)
+ except Exception as e:
+ print(f"ERROR: Could not read window info: {e}", flush=True)
+
+ if pid is not None and window_id is not None:
+ # Use get_window_state with capture_mode=vision to get screenshot
+ print(f"\n--- get_window_state (vision) pid={pid} window_id={window_id} ---", flush=True)
+ send_request("tools/call", {
+ "name": "get_window_state",
+ "arguments": {
+ "pid": pid,
+ "window_id": window_id,
+ "capture_mode": "vision",
+ },
+ }, req_id=5)
+ resp = read_response(timeout=30)
+
+ # Extract base64 image from response
+ content = resp.get("result", {}).get("content", [])
+ screenshot_saved = False
+ for item in content:
+ if item.get("type") == "image":
+ img_data = item.get("data", "")
+ if img_data:
+ img_bytes = base64.b64decode(img_data)
+ with open("/tmp/cua-driver-screenshot.png", "wb") as f:
+ f.write(img_bytes)
+ print(f"PASS: Screenshot saved ({len(img_bytes)} bytes)", flush=True)
+ screenshot_saved = True
+ break
+ elif item.get("type") == "text":
+ text = item.get("text", "")
+ # Check if text contains base64 image data
+ if "base64" in text.lower() or len(text) > 1000:
+ print(f"Text content (may contain image ref): {text[:200]}", flush=True)
+
+ if not screenshot_saved:
+ print(f"WARN: No image in response, saving raw response", flush=True)
+ with open("/tmp/cua-driver-response.json", "w") as f:
+ json.dump(resp, f, indent=2)
+ else:
+ print("SKIP: No window found for screenshot", flush=True)
+
+ print("\n=== Screenshot test complete ===", flush=True)
+
+ finally:
+ proc.stdin.close()
+ proc.terminate()
+ proc.wait(timeout=5)
+
+ if __name__ == "__main__":
+ main()
+ '';
+
+ # Shell script displayed in xterm for a visible window to screenshot
+ testPage = pkgs.writeText "test-page.sh" ''
+ #!/bin/sh
+ cat <<'HEREDOC'
+
+ ╔══════════════════════════════════════════════╗
+ β•‘ CUA Driver - NixOS Integration Test β•‘
+ β•‘ β•‘
+ β•‘ cua-driver v0.3.2 β•‘
+ β•‘ MCP server running on Xvfb :99 β•‘
+ β•‘ 34 tools registered β•‘
+ β•‘ β•‘
+ β•‘ All tests passed! β•‘
+ β•šβ•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•
+
+ HEREDOC
+ sleep infinity
+ '';
+
+in
+
+pkgs.testers.nixosTest {
+ name = "cua-driver-screenshot-test";
+ meta = {
+ maintainers = [ ];
+ };
+
+ nodes.machine =
+ {
+ config,
+ pkgs,
+ lib,
+ ...
+ }:
+ {
+ imports = [ cuaDriverModule ];
+ virtualisation = {
+ cores = 2;
+ memorySize = 2048;
+ };
+ services.cua-driver.enable = true;
+ environment.systemPackages = with pkgs; [
+ xorg.xorgserver
+ xterm
+ openbox # lightweight WM needed for _NET_CLIENT_LIST
+ picom # compositing manager so X11 GetImage returns rendered pixels
+ xdotool # window ID lookup
+ python3
+ jq
+ procps
+ ];
+ };
+
+ testScript = ''
+ machine.start()
+ machine.wait_for_unit("multi-user.target")
+
+ with subtest("Binary exists and runs"):
+ machine.succeed("cua-driver --help")
+
+ with subtest("list-tools prints available tools"):
+ result = machine.succeed("cua-driver list-tools")
+ assert "click" in result, f"click not in: {result}"
+ assert "get_window_state" in result, f"get_window_state not in: {result}"
+
+ with subtest("Start Xvfb, window manager, and xterm"):
+ machine.execute("Xvfb :99 -screen 0 1280x1024x24 >/dev/null 2>&1 &")
+ machine.wait_until_succeeds("test -e /tmp/.X11-unix/X99", timeout=10)
+ # Start openbox WM so _NET_CLIENT_LIST is populated for list_windows
+ machine.execute("DISPLAY=:99 openbox >/dev/null 2>&1 &")
+ # Start compositing manager so X11 GetImage returns rendered pixels
+ machine.execute("DISPLAY=:99 picom --backend xrender >/dev/null 2>&1 &")
+ machine.copy_from_host("${testPage}", "/tmp/test-page.sh")
+ machine.succeed("chmod +x /tmp/test-page.sh")
+ machine.execute("DISPLAY=:99 xterm -T 'CUA Test' -fa Monospace -fs 14 -geometry 60x20+100+100 -e /tmp/test-page.sh >/dev/null 2>&1 &")
+ # Wait for xterm window to appear (poll via xdotool inside the VM)
+ machine.wait_until_succeeds("DISPLAY=:99 xdotool search --class xterm", timeout=15)
+ # Save the xterm window ID and PID for the MCP screenshot test
+ machine.succeed("DISPLAY=:99 xdotool search --class xterm | head -1 > /tmp/xterm-xid.txt")
+ machine.succeed("pgrep -f 'xterm.*CUA Test' | head -1 > /tmp/xterm-pid.txt")
+ # Focus and raise the window so it's fully rendered
+ machine.succeed("DISPLAY=:99 xdotool windowactivate --sync $(cat /tmp/xterm-xid.txt)")
+ machine.succeed("DISPLAY=:99 xdotool windowfocus --sync $(cat /tmp/xterm-xid.txt)")
+
+ with subtest("Screenshot via cua-driver MCP"):
+ machine.copy_from_host("${mcpScreenshotTest}", "/tmp/mcp-screenshot-test.py")
+ result = machine.succeed(
+ "timeout 60 env DISPLAY=:99 "
+ "python3 /tmp/mcp-screenshot-test.py 2>&1"
+ )
+ machine.log(result)
+ assert "Screenshot test complete" in result, f"Test did not complete: {result}"
+
+ with subtest("Extract screenshot"):
+ # Copy screenshot out of VM if it exists
+ machine.succeed("test -f /tmp/cua-driver-screenshot.png || test -f /tmp/cua-driver-response.json")
+ machine.copy_from_machine("/tmp/cua-driver-screenshot.png", "")
+ '';
+}