Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
feat(xfce-cua): add multi-architecture Cua Driver desktop (#2544)
Francesco Bonacci2d7203ce69f6
diff --git a/.github/workflows/cd-container-xfce-cua.yml b/.github/workflows/cd-container-xfce-cua.ymlnew file mode 100644index 0000000000..1c9b25a6d2--- /dev/null+++ b/.github/workflows/cd-container-xfce-cua.yml@@ -0,0 +1,18 @@+name: "CD: Container XFCE Cua"++on:+ push:+ tags:+ - "docker-xfce-cua-v*.*.*"+ workflow_dispatch:++jobs:+ publish:+ uses: ./.github/workflows/docker-reusable-publish.yml+ with:+ image_name: xfce-cua+ context_dir: libs/xfce-cua+ tag_prefix: docker-xfce-cua-v+ docker_hub_org: trycua+ secrets:+ DOCKER_HUB_TOKEN: ${{ secrets.DOCKER_HUB_TOKEN }}diff --git a/.github/workflows/ci-container-xfce-cua.yml b/.github/workflows/ci-container-xfce-cua.ymlnew file mode 100644index 0000000000..f449c7433b--- /dev/null+++ b/.github/workflows/ci-container-xfce-cua.yml@@ -0,0 +1,103 @@+name: "CI: Container XFCE Cua"++on:+ pull_request:+ paths:+ - "libs/xfce-cua/**"+ - ".github/workflows/ci-container-xfce-cua.yml"++jobs:+ build-and-smoke:+ name: build-and-smoke (${{ matrix.platform }})+ runs-on: ${{ matrix.platform == 'linux/arm64' && 'ubuntu-24.04-arm' || 'ubuntu-latest' }}+ strategy:+ fail-fast: false+ matrix:+ platform:+ - linux/amd64+ - linux/arm64+ steps:+ - name: Checkout+ uses: actions/checkout@v4++ - name: Set expected Debian architecture+ id: architecture+ run: |+ case "${{ matrix.platform }}" in+ linux/amd64) echo "debian=amd64" >> "$GITHUB_OUTPUT" ;;+ linux/arm64) echo "debian=arm64" >> "$GITHUB_OUTPUT" ;;+ esac++ - name: Set up Docker Buildx+ uses: docker/setup-buildx-action@v3++ - name: Build native image+ uses: docker/build-push-action@v5+ with:+ context: ./libs/xfce-cua+ file: ./libs/xfce-cua/Dockerfile+ load: true+ platforms: ${{ matrix.platform }}+ tags: xfce-cua:ci++ - name: Verify native packages+ env:+ EXPECTED_ARCH: ${{ steps.architecture.outputs.debian }}+ run: |+ docker run --rm \+ --user cua \+ -e HOME=/home/cua \+ -e EXPECTED_ARCH \+ xfce-cua:ci \+ bash -lc '+ test "$(dpkg --print-architecture)" = "$EXPECTED_ARCH"+ /opt/venv/bin/cua-driver --version+ /opt/venv/bin/python -c "import importlib.util; from cua_driver import CuaDriver, get_binary_path; assert importlib.util.find_spec(\"computer_server\") is None; print(get_binary_path())"+ ffmpeg -version | head -1+ '++ - name: Verify desktop services+ run: |+ docker run --rm --detach \+ --name xfce-cua-ci \+ -e VNC_PW=ci-test \+ xfce-cua:ci+ trap 'docker stop xfce-cua-ci >/dev/null 2>&1 || true' EXIT++ for attempt in $(seq 1 60); do+ status=$(docker inspect --format '{{.State.Health.Status}}' xfce-cua-ci)+ if [ "$status" = "healthy" ]; then+ break+ fi+ if [ "$status" = "unhealthy" ] || [ "$attempt" -eq 60 ]; then+ docker logs xfce-cua-ci+ exit 1+ fi+ sleep 2+ done++ docker exec xfce-cua-ci curl --fail --silent \+ http://127.0.0.1:6901/vnc.html >/dev/null+ docker exec xfce-cua-ci supervisorctl \+ -c /etc/supervisor/supervisord.conf status++ docker exec --detach --user cua xfce-cua-ci \+ /opt/venv/bin/cua-driver serve --dangerously-bypass-approvals+ for attempt in $(seq 1 30); do+ if docker exec --user cua xfce-cua-ci \+ /opt/venv/bin/cua-driver status >/dev/null 2>&1; then+ break+ fi+ if [ "$attempt" -eq 30 ]; then+ docker logs xfce-cua-ci+ exit 1+ fi+ sleep 1+ done++ docker exec --user cua xfce-cua-ci bash -lc '+ /opt/venv/bin/cua-driver doctor --json |+ /opt/venv/bin/python -c "import json, sys; report = json.load(sys.stdin); at_spi = next(probe for probe in report[\"probes\"] if probe[\"label\"] == \"AT-SPI\"); assert report[\"ok\"]; assert at_spi[\"status\"] == \"ok\", at_spi"+ '+ docker stop xfce-cua-ci+ trap - EXITdiff --git a/.github/workflows/ci-release-reminder.yml b/.github/workflows/ci-release-reminder.ymlindex 9116f7bb01..7966a44f97 100644--- a/.github/workflows/ci-release-reminder.yml+++ b/.github/workflows/ci-release-reminder.yml@@ -41,6 +41,7 @@ jobs:["libs/typescript/core/"]="npm/core"["libs/typescript/playground/"]="npm/playground"["libs/cuabot/"]="npm/cuabot"+ ["libs/xfce-cua/"]="docker/xfce-cua"["libs/xfce/"]="docker/xfce"["libs/kasm/"]="docker/kasm"["libs/lumier/"]="docker/lumier"diff --git a/.github/workflows/release-bump-version.yml b/.github/workflows/release-bump-version.ymlindex d026dd1f40..f1b504ad98 100644--- a/.github/workflows/release-bump-version.yml+++ b/.github/workflows/release-bump-version.yml@@ -32,6 +32,7 @@ on:- npm/cuabot- docker/cuabot- docker/kasm+ - docker/xfce-cua- docker/xfce- docker/lumier- docker/qemu-android@@ -180,6 +181,10 @@ jobs:echo "directory=libs/kasm" >> $GITHUB_OUTPUTecho "type=docker" >> $GITHUB_OUTPUT;;+ "docker/xfce-cua")+ echo "directory=libs/xfce-cua" >> $GITHUB_OUTPUT+ echo "type=docker" >> $GITHUB_OUTPUT+ ;;"docker/xfce")echo "directory=libs/xfce" >> $GITHUB_OUTPUTecho "type=docker" >> $GITHUB_OUTPUTdiff --git a/.github/workflows/release-on-merge.yml b/.github/workflows/release-on-merge.ymlindex f14d6879a1..f6f9b63deb 100644--- a/.github/workflows/release-on-merge.yml+++ b/.github/workflows/release-on-merge.yml@@ -102,6 +102,7 @@ jobs:["libs/typescript/fleet/"]="npm/fleet"["libs/typescript/playground/"]="npm/playground"["libs/cuabot/"]="npm/cuabot"+ ["libs/xfce-cua/"]="docker/xfce-cua"["libs/xfce/"]="docker/xfce"["libs/kasm/"]="docker/kasm"["libs/lumier/"]="docker/lumier"diff --git a/.github/workflows/release-unreleased-digest.yml b/.github/workflows/release-unreleased-digest.ymlindex 4fc2cf4dc4..c880165a5d 100644--- a/.github/workflows/release-unreleased-digest.yml+++ b/.github/workflows/release-unreleased-digest.yml@@ -36,6 +36,7 @@ jobs:SERVICE_TAG_DIR["npm/core"]="npm-core-v|libs/typescript/core/"SERVICE_TAG_DIR["npm/playground"]="npm-playground-v|libs/typescript/playground/"SERVICE_TAG_DIR["npm/cuabot"]="cuabot-v|libs/cuabot/"+ SERVICE_TAG_DIR["docker/xfce-cua"]="docker-xfce-cua-v|libs/xfce-cua/"SERVICE_TAG_DIR["docker/xfce"]="docker-xfce-v|libs/xfce/"SERVICE_TAG_DIR["docker/kasm"]="docker-kasm-v|libs/kasm/"SERVICE_TAG_DIR["docker/lumier"]="docker-lumier-v|libs/lumier/"diff --git a/libs/xfce-cua/.bumpversion.cfg b/libs/xfce-cua/.bumpversion.cfgnew file mode 100644index 0000000000..933222d8b7--- /dev/null+++ b/libs/xfce-cua/.bumpversion.cfg@@ -0,0 +1,8 @@+[bumpversion]+current_version = 0.0.0+commit = True+tag = True+tag_name = docker-xfce-cua-v{new_version}+message = chore: bump xfce-cua container to {new_version}++[bumpversion:file:VERSION]diff --git a/libs/xfce-cua/.dockerignore b/libs/xfce-cua/.dockerignorenew file mode 100644index 0000000000..07e1bdfe32--- /dev/null+++ b/libs/xfce-cua/.dockerignore@@ -0,0 +1,3 @@+.git+**/__pycache__+*.pycdiff --git a/libs/xfce-cua/Dockerfile b/libs/xfce-cua/Dockerfilenew file mode 100644index 0000000000..24b954e155--- /dev/null+++ b/libs/xfce-cua/Dockerfile@@ -0,0 +1,97 @@+# Vanilla Cua XFCE desktop.+#+# Keep this image application-neutral. Application-specific images should build+# on top of it instead of making the desktop runtime inherit their architecture+# constraints.+FROM python:3.13-slim-bookworm++LABEL org.opencontainers.image.title="Cua XFCE"+LABEL org.opencontainers.image.description="Multi-architecture XFCE desktop with Cua Driver"+LABEL org.opencontainers.image.source="https://github.com/trycua/cua"++ENV DEBIAN_FRONTEND=noninteractive \+ HOME=/home/cua \+ DISPLAY=:1 \+ DBUS_SESSION_BUS_ADDRESS=unix:path=/run/user/1000/bus \+ VNC_PORT=5901 \+ NOVNC_PORT=6901 \+ VNC_RESOLUTION=1280x900 \+ VNC_COL_DEPTH=24 \+ PATH=/opt/venv/bin:$PATH++USER root++RUN apt-get update && apt-get install -y --no-install-recommends \+ at-spi2-core \+ ca-certificates \+ curl \+ dbus-x11 \+ ffmpeg \+ firefox-esr \+ fonts-dejavu-core \+ libglib2.0-bin \+ netcat-openbsd \+ novnc \+ procps \+ sudo \+ supervisor \+ tigervnc-standalone-server \+ tigervnc-tools \+ websockify \+ x11-xserver-utils \+ xdg-utils \+ xfce4 \+ xfce4-terminal \+ && apt-get clean \+ && find /var/lib/apt/lists -mindepth 1 -delete++RUN useradd --create-home --shell /bin/bash --groups sudo cua \+ && echo "cua ALL=(ALL) NOPASSWD:ALL" > /etc/sudoers.d/cua \+ && chmod 0440 /etc/sudoers.d/cua++COPY requirements.txt /tmp/requirements.txt++RUN python -m venv /opt/venv \+ && /opt/venv/bin/python -m pip install --no-cache-dir --upgrade pip \+ && /opt/venv/bin/python -m pip install --no-cache-dir \+ -r /tmp/requirements.txt++RUN /opt/venv/bin/python -c \+ "from pathlib import Path; from cua_driver import CuaDriver, get_binary_path; assert Path(get_binary_path()).is_file()" \+ && chown cua:cua "$(/opt/venv/bin/python -c 'from cua_driver import get_binary_path; print(get_binary_path())')" \+ && su -s /bin/bash -c \+ "HOME=/home/cua /opt/venv/bin/cua-driver --version" cua++RUN printf '%s\n' \+ 'pref("datareporting.policy.firstRunURL", "");' \+ 'pref("datareporting.policy.dataSubmissionEnabled", false);' \+ 'pref("datareporting.healthreport.uploadEnabled", false);' \+ 'pref("browser.aboutwelcome.enabled", false);' \+ > /etc/firefox-esr/cua.js++COPY src/supervisor/supervisord.conf /etc/supervisor/supervisord.conf+COPY src/scripts/ /usr/local/bin/+COPY --chown=cua:cua src/xfce/ /home/cua/.config/xfce4/++RUN chmod +x /usr/local/bin/*.sh \+ && mkdir -p \+ /home/cua/.cache/dconf \+ /home/cua/.config/autostart \+ /home/cua/.mozilla/firefox \+ /home/cua/.vnc \+ /home/cua/shared \+ /home/cua/storage \+ /run/user/1000 \+ && printf '[Desktop Entry]\nHidden=true\n' > /home/cua/.config/autostart/xfce4-power-manager.desktop \+ && printf '[Desktop Entry]\nHidden=true\n' > /home/cua/.config/autostart/xfce4-screensaver.desktop \+ && chown -R cua:cua /home/cua /run/user/1000 \+ && chmod 0700 /run/user/1000++WORKDIR /home/cua++EXPOSE 5901 6901++HEALTHCHECK --interval=10s --timeout=5s --start-period=30s --retries=6 \+ CMD curl --fail --silent "http://127.0.0.1:${NOVNC_PORT}/vnc.html" >/dev/null || exit 1++CMD ["/usr/bin/supervisord", "-c", "/etc/supervisor/supervisord.conf"]diff --git a/libs/xfce-cua/README.md b/libs/xfce-cua/README.mdnew file mode 100644index 0000000000..d78614eee2--- /dev/null+++ b/libs/xfce-cua/README.md@@ -0,0 +1,58 @@+# xfce-cua++`trycua/xfce-cua` is Cua's application-neutral Linux desktop image. It is+built directly on Debian and published for both `linux/amd64` and+`linux/arm64`.++It includes:++- XFCE on an X11 VNC session+- browser access through noVNC+- Firefox ESR+- Cua Driver and its Python SDK+- ffmpeg for Cua Driver trajectory video recording++It intentionally does not include KiCad or another application suite.+Application-specific images can build on top of this image without constraining+the base desktop to one CPU architecture.++## Run++```bash+docker run --rm \+ -p 6901:6901 \+ -e VNC_PW=cua \+ trycua/xfce-cua:latest+```++Open <http://localhost:6901/vnc.html> and use the password supplied through+`VNC_PW`.++## Use Cua Driver inside the desktop++The installed executable and Python SDK use the container's native CPU+architecture:++```bash+docker exec -u cua <container> cua-driver --version+docker exec -u cua <container> \+ python -c "from cua_driver import CuaDriver; print(CuaDriver)"+```++Cua Driver is the only computer-use runtime included. It does not auto-start+because its lifecycle belongs to the client running inside the sandbox. Start+it against the XFCE session when needed:++```bash+docker exec -d -u cua \+ -e HOME=/home/cua \+ -e DISPLAY=:1 \+ <container> \+ cua-driver serve --dangerously-bypass-approvals+```++## Build++```bash+docker buildx build --platform linux/amd64,linux/arm64 libs/xfce-cua+```diff --git a/libs/xfce-cua/VERSION b/libs/xfce-cua/VERSIONnew file mode 100644index 0000000000..77d6f4ca23--- /dev/null+++ b/libs/xfce-cua/VERSION@@ -0,0 +1 @@+0.0.0diff --git a/libs/xfce-cua/requirements.txt b/libs/xfce-cua/requirements.txtnew file mode 100644index 0000000000..30a7d23bd1--- /dev/null+++ b/libs/xfce-cua/requirements.txt@@ -0,0 +1,2 @@+# Pin released dependencies so the container can be reproduced from its tag.+cua-driver==0.12.4diff --git a/libs/xfce-cua/src/scripts/start-novnc.sh b/libs/xfce-cua/src/scripts/start-novnc.shnew file mode 100644index 0000000000..be1c0c45ab--- /dev/null+++ b/libs/xfce-cua/src/scripts/start-novnc.sh@@ -0,0 +1,11 @@+#!/bin/bash+set -euo pipefail++while ! nc -z 127.0.0.1 "${VNC_PORT:-5901}"; do+ sleep 1+done++exec websockify \+ --web=/usr/share/novnc \+ "${NOVNC_PORT:-6901}" \+ "127.0.0.1:${VNC_PORT:-5901}"diff --git a/libs/xfce-cua/src/scripts/start-vnc.sh b/libs/xfce-cua/src/scripts/start-vnc.shnew file mode 100644index 0000000000..3659550dee--- /dev/null+++ b/libs/xfce-cua/src/scripts/start-vnc.sh@@ -0,0 +1,28 @@+#!/bin/bash+set -euo pipefail++rm -f /tmp/.X1-lock /tmp/.X11-unix/X1++security_args=(-SecurityTypes None --I-KNOW-THIS-IS-INSECURE)+if [[ -n "${VNC_PW:-}" ]]; then+ mkdir -p /home/cua/.vnc+ printf '%s\n' "$VNC_PW" | vncpasswd -f > /home/cua/.vnc/passwd+ chmod 0600 /home/cua/.vnc/passwd+ security_args=(-SecurityTypes VncAuth -rfbauth /home/cua/.vnc/passwd)+ unset VNC_PW+fi++vncserver :1 \+ -geometry "${VNC_RESOLUTION:-1280x900}" \+ -depth "${VNC_COL_DEPTH:-24}" \+ -rfbport "${VNC_PORT:-5901}" \+ -localhost no \+ "${security_args[@]}" \+ -AlwaysShared \+ -AcceptPointerEvents \+ -AcceptKeyEvents \+ -AcceptCutText \+ -SendCutText \+ -xstartup /usr/local/bin/xstartup.sh++exec tail -F /home/cua/.vnc/*.logdiff --git a/libs/xfce-cua/src/scripts/xstartup.sh b/libs/xfce-cua/src/scripts/xstartup.shnew file mode 100644index 0000000000..fcc99fdac8--- /dev/null+++ b/libs/xfce-cua/src/scripts/xstartup.sh@@ -0,0 +1,25 @@+#!/bin/bash+set -euo pipefail++export XDG_CURRENT_DESKTOP=XFCE+export XDG_SESSION_DESKTOP=xfce+export XDG_SESSION_TYPE=x11++if [[ -z "${DBUS_SESSION_BUS_ADDRESS:-}" ]]; then+ eval "$(dbus-launch --sh-syntax --exit-with-session)"+fi++startxfce4 &+xfce_pid=$!++for _ in $(seq 1 20); do+ if xset q >/dev/null 2>&1; then+ xset s off+ xset -dpms+ xset s noblank+ break+ fi+ sleep 1+done++wait "$xfce_pid"diff --git a/libs/xfce-cua/src/supervisor/supervisord.conf b/libs/xfce-cua/src/supervisor/supervisord.confnew file mode 100644index 0000000000..f99db7ed4c--- /dev/null+++ b/libs/xfce-cua/src/supervisor/supervisord.conf@@ -0,0 +1,46 @@+[supervisord]+nodaemon=true+user=root+logfile=/var/log/supervisor/supervisord.log+pidfile=/var/run/supervisord.pid+childlogdir=/var/log/supervisor++[unix_http_server]+file=/var/run/supervisor.sock+chmod=0700++[rpcinterface:supervisor]+supervisor.rpcinterface_factory=supervisor.rpcinterface:make_main_rpcinterface++[supervisorctl]+serverurl=unix:///var/run/supervisor.sock++[program:dbus]+command=/usr/bin/dbus-daemon --session --nofork --address=unix:path=/run/user/1000/bus+user=cua+environment=HOME="/home/cua",USER="cua"+autorestart=true+startsecs=1+stdout_logfile=/var/log/supervisor/dbus.log+stderr_logfile=/var/log/supervisor/dbus.error.log+priority=5++[program:vncserver]+command=/usr/local/bin/start-vnc.sh+user=cua+environment=HOME="/home/cua",USER="cua",DISPLAY=":1"+autorestart=true+startsecs=2+stdout_logfile=/var/log/supervisor/vncserver.log+stderr_logfile=/var/log/supervisor/vncserver.error.log+priority=10++[program:novnc]+command=/usr/local/bin/start-novnc.sh+user=cua+environment=HOME="/home/cua",USER="cua"+autorestart=true+startsecs=2+stdout_logfile=/var/log/supervisor/novnc.log+stderr_logfile=/var/log/supervisor/novnc.error.log+priority=20diff --git a/libs/xfce-cua/src/xfce/helpers.rc b/libs/xfce-cua/src/xfce/helpers.rcnew file mode 100644index 0000000000..2cce9cb048--- /dev/null+++ b/libs/xfce-cua/src/xfce/helpers.rc@@ -0,0 +1,2 @@+WebBrowser=firefox-esr+TerminalEmulator=xfce4-terminal