Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
feat(cua-driver): prepare ClawHub skill release (#2265)
Francesco Bonacci393c984190b2
diff --git a/.github/workflows/clawhub-cua-driver.yml b/.github/workflows/clawhub-cua-driver.ymlnew file mode 100644index 0000000000..e80964a32b--- /dev/null+++ b/.github/workflows/clawhub-cua-driver.yml@@ -0,0 +1,143 @@+name: "ClawHub: cua-driver skill"++on:+ pull_request:+ paths:+ - ".github/workflows/clawhub-cua-driver.yml"+ - "libs/cua-driver/rust/Cargo.toml"+ - "libs/cua-driver/rust/Skills/cua-driver/**"+ - "libs/cua-driver/rust/crates/cua-driver/src/skills.rs"+ workflow_dispatch:+ inputs:+ version:+ description: "ClawHub version; must match Cargo.toml and SKILL.md"+ required: true+ type: string+ owner:+ description: "ClawHub publisher handle"+ required: true+ default: "f-trycua"+ type: string+ confirm_mit0:+ description: "Cua AI has authority to publish every bundled file under MIT-0"+ required: true+ default: false+ type: boolean++permissions:+ contents: read+ id-token: write++env:+ CLAWHUB_CLI_VERSION: "0.23.1"+ SKILL_PATH: "libs/cua-driver/rust/Skills/cua-driver"++jobs:+ dry-run:+ name: Validate publish bundle+ if: github.event_name == 'pull_request'+ uses: openclaw/clawhub/.github/workflows/skill-publish.yml@b2f06028cff845974cfc9e528317ef879663935b+ with:+ skill_path: "libs/cua-driver/rust/Skills/cua-driver"+ owner: "f-trycua"+ dry_run: true+ ref: ${{ github.event.pull_request.head.sha }}++ publish:+ name: Publish ${{ inputs.version }} as @${{ inputs.owner }}/cua-driver+ if: github.event_name == 'workflow_dispatch'+ runs-on: ubuntu-latest+ permissions:+ contents: read+ steps:+ - uses: actions/checkout@v4++ - name: Validate release inputs+ env:+ CONFIRM_MIT0: ${{ inputs.confirm_mit0 }}+ REQUESTED_VERSION: ${{ inputs.version }}+ run: |+ python3 - <<'PY'+ import os+ import re+ import tomllib+ from pathlib import Path++ if os.environ["CONFIRM_MIT0"] != "true":+ raise SystemExit("confirm_mit0 must be checked before a public release")+ if os.environ["GITHUB_REF"] != "refs/heads/main":+ raise SystemExit("ClawHub releases must be dispatched from the main branch")++ requested = os.environ["REQUESTED_VERSION"].strip()+ cargo = tomllib.loads(Path("libs/cua-driver/rust/Cargo.toml").read_text())+ cargo_version = cargo["workspace"]["package"]["version"]+ skill_text = Path(os.environ["SKILL_PATH"], "SKILL.md").read_text()+ frontmatter = skill_text.split("---", 2)[1]+ match = re.search(r"^version:\s*([^\s#]+)", frontmatter, re.MULTILINE)+ if not match:+ raise SystemExit("SKILL.md frontmatter has no version")+ skill_version = match.group(1)++ if len({requested, cargo_version, skill_version}) != 1:+ raise SystemExit(+ "version mismatch: "+ f"requested={requested}, Cargo.toml={cargo_version}, SKILL.md={skill_version}"+ )+ PY++ - name: Configure ClawHub authentication+ env:+ CLAWHUB_TOKEN: ${{ secrets.CLAWHUB_TOKEN }}+ run: |+ python3 - <<'PY'+ import json+ import os+ from pathlib import Path++ token = os.environ.get("CLAWHUB_TOKEN", "").strip()+ if not token:+ raise SystemExit("CLAWHUB_TOKEN is not configured")+ path = Path(os.environ["RUNNER_TEMP"]) / "clawhub-config.json"+ path.write_text(json.dumps({"registry": "https://clawhub.ai", "token": token}))+ path.chmod(0o600)+ with Path(os.environ["GITHUB_ENV"]).open("a") as env_file:+ env_file.write(f"CLAWHUB_CONFIG_PATH={path}\n")+ PY++ - name: Publish cua-driver skill+ env:+ OWNER: ${{ inputs.owner }}+ VERSION: ${{ inputs.version }}+ run: |+ npx --yes "clawhub@${CLAWHUB_CLI_VERSION}" skill publish "${SKILL_PATH}" \+ --slug cua-driver \+ --name "CUA Driver" \+ --owner "${OWNER}" \+ --version "${VERSION}" \+ --changelog "cua-driver ${VERSION}" \+ --tags latest \+ --source-repo "${GITHUB_REPOSITORY}" \+ --source-commit "${GITHUB_SHA}" \+ --source-ref "${GITHUB_REF}" \+ --source-path "${SKILL_PATH}" \+ --json | tee "${RUNNER_TEMP}/clawhub-publish.json"++ - name: Summarize release+ if: always()+ run: |+ if [[ -f "${RUNNER_TEMP}/clawhub-publish.json" ]]; then+ {+ echo '## ClawHub publish result'+ echo '```json'+ cat "${RUNNER_TEMP}/clawhub-publish.json"+ echo '```'+ } >> "${GITHUB_STEP_SUMMARY}"+ fi++ - name: Upload publish result+ if: always()+ uses: actions/upload-artifact@v4+ with:+ name: clawhub-cua-driver-publish-${{ inputs.version }}+ path: ${{ runner.temp }}/clawhub-publish.json+ if-no-files-found: ignorediff --git a/libs/cua-driver/README.md b/libs/cua-driver/README.mdindex aec31a136b..c947e24267 100644--- a/libs/cua-driver/README.md+++ b/libs/cua-driver/README.md@@ -6,14 +6,14 @@ Background computer-use driver for any agents. Speaks MCP over stdio; drives nat## Repository Layout-| Path | Purpose |-| --- | --- |-| `rust/` | Cargo workspace for the driver daemon, platform crates, and Rust tests |-| `python/` | Python package wrapper and package tests |-| `tests/fixtures/` | Source-built GUI harness apps and shared fixtures |-| `rust/crates/cua-driver/tests/` | Rust integration tests for the driver and GUI harnesses |-| `scripts/` | Install, uninstall, local build, and VM sync helpers |-| `docs/` | Small repo-local specs that are not part of the hosted docs site |+| Path | Purpose |+| ------------------------------- | ---------------------------------------------------------------------- |+| `rust/` | Cargo workspace for the driver daemon, platform crates, and Rust tests |+| `python/` | Python package wrapper and package tests |+| `tests/fixtures/` | Source-built GUI harness apps and shared fixtures |+| `rust/crates/cua-driver/tests/` | Rust integration tests for the driver and GUI harnesses |+| `scripts/` | Install, uninstall, local build, and VM sync helpers |+| `docs/` | Small repo-local specs that are not part of the hosted docs site |Start with `rust/README.md`, `rust/crates/cua-driver/tests/README.md`, and`tests/fixtures/README.md` when changing driver behavior or tests.@@ -52,3 +52,48 @@ macOS attributes Accessibility and Screen Recording grants to a responsible app- **Embedded:** have the macOS app that owns the grants spawn the driver directly with `CUA_DRIVER_EMBEDDED=1` (or `--embedded`). This keeps the driver in that app's responsibility chain, so it inherits the app's grants. A gateway, terminal, or unrelated helper must not spawn it on the app's behalf.Directly spawning a raw `cua-driver` binary outside `CuaDriver.app` without embedded mode is unsupported: it has no stable bundle identity for TCC attribution. Do not grant permissions to arbitrary binary paths or rely on that configuration in production. See [`rust/Skills/cua-driver/EMBEDDING.md`](rust/Skills/cua-driver/EMBEDDING.md) for the embedding contract and examples.++## Publishing the agent skill to ClawHub++The canonical skill source is `rust/Skills/cua-driver`. It is published as one+cross-platform ClawHub skill at `@f-trycua/cua-driver`; the bundle includes the+macOS, Windows, and Linux documents. Direct installs through `cua-driver skills+install` still keep only the host OS document unless `--all-platforms` is used.++ClawHub releases have their own explicit license boundary. The repository stays+under MIT, while every skill copy published through ClawHub is distributed+under MIT-0. Before a release, retain an internal record that Cua AI has the+right to distribute every bundled file under MIT-0.++Pull requests that change the skill run a publish dry-run through ClawHub's+pinned reusable workflow. A real release is available only through the+`ClawHub: cua-driver skill` workflow's manual dispatch. The publish job requires+all of the following:++1. Dispatch the workflow from `main`.+2. Enter a version that matches both `rust/Cargo.toml` and the `version` field+ in `rust/Skills/cua-driver/SKILL.md`.+3. Confirm the MIT-0 rights check in the workflow form.+4. Configure a repository Actions secret named `CLAWHUB_TOKEN` for a publisher+ that can release under the selected owner. The default owner is `f-trycua`.++The workflow pins the ClawHub CLI, records the source repository, commit, ref,+and path, and uploads the JSON publish result as an Actions artifact. When a+`trycua` organization publisher becomes available, transfer the existing skill+instead of creating a second listing, then change the workflow and this section+in the same pull request.++After publishing, inspect and scan the exact version, then install it into an+empty work directory:++```bash+npx --yes [email protected] inspect @f-trycua/cua-driver --version 0.8.3 --files+npx --yes [email protected] scan --slug cua-driver --version 0.8.3 --update+npx --yes [email protected] --workdir /tmp/cua-driver-clawhub-smoke \+ install @f-trycua/cua-driver+```++Confirm that `MACOS.md`, `WINDOWS.md`, and `LINUX.md` are present, run+`cua-driver doctor`, and perform a read-only `list_apps` call through OpenClaw.+If a release is faulty, publish the last known-good content as a new patch+version. Do not delete the current latest version before a replacement exists.diff --git a/libs/cua-driver/rust/Skills/cua-driver/.clawhubignore b/libs/cua-driver/rust/Skills/cua-driver/.clawhubignorenew file mode 100644index 0000000000..d725f76449--- /dev/null+++ b/libs/cua-driver/rust/Skills/cua-driver/.clawhubignore@@ -0,0 +1,2 @@+# Repository-only end-to-end test instructions are not required at runtime.+TESTS.mddiff --git a/libs/cua-driver/rust/Skills/cua-driver/README.md b/libs/cua-driver/rust/Skills/cua-driver/README.mdindex d75012c9ca..c9f7055f5a 100644--- a/libs/cua-driver/rust/Skills/cua-driver/README.md+++ b/libs/cua-driver/rust/Skills/cua-driver/README.md@@ -1,9 +1,9 @@-# cua-driver-rs — Claude Code skill+# cua-driver agent skill-A [Claude Code](https://code.claude.com) skill that teaches Claude to-drive native GUI apps on **macOS, Windows, and Linux** via the+A cross-agent skill that teaches AI agents to drive native GUI apps on+**macOS, Windows, and Linux** via the[`cua-driver`](https://github.com/trycua/cua/tree/main/libs/cua-driver/rust)-CLI — snapshot an app's accessibility tree (AX on macOS, UIA on+CLI. It covers how to snapshot an app's accessibility tree (AX on macOS, UIA onWindows, AT-SPI on Linux), click/type/scroll by `element_index` orpixel coords, and verify via re-snapshot. Backgrounded-first on everyplatform: no focus steal, no cursor warp.@@ -72,9 +72,11 @@ forbidden-list / launch / click details.3. **TCC grants on `CuaDriver.app`** — **Accessibility** and**Screen Recording** in System Settings → Privacy & Security.Verify with:+```bashcua-driver check_permissions```+Both fields must be `true`. If not, the app appears in therelevant panes of System Settings after first use; toggle it onthere.@@ -109,9 +111,9 @@ forbidden-list / launch / click details.### Linux-The full tool surface is supported on X11 (background input via AT-SPI-+ XSendEvent, screenshots, recording); native Wayland input/capture is-opt-in and still preview. See `LINUX.md`. Install:+The full tool surface is supported on X11 (background input through AT-SPI and+XSendEvent, screenshots, and recording); native Wayland input/capture is opt-in+and still preview. See `LINUX.md`. Install:```bash/bin/bash -c "$(curl -fsSL https://cua.ai/driver/install.sh)"@@ -124,6 +126,33 @@ installs the Rust port — no flag needed.)The skill is a drop-in directory. Same shape on every platform.+**OpenClaw via ClawHub:**++```bash+clawhub install @f-trycua/cua-driver+```++ClawHub installs the complete cross-platform bundle. The agent reads the shared+core and the document for the current host OS. Install `cua-driver` separately+before using the skill:++```bash+/bin/bash -c "$(curl -fsSL https://cua.ai/driver/install.sh)" # macOS or Linux+```++```powershell+irm https://cua.ai/driver/install.ps1 | iex # Windows+```++**Direct installation for supported agents:**++```bash+cua-driver skills install+```++This detects installed agent skill directories and keeps only the current host+OS document by default. Pass `--all-platforms` to retain all three OS documents.+**Personal scope** (all Claude Code sessions on your machine):```bash@@ -144,13 +173,6 @@ mkdir -p .claude/skillscp -R /path/to/cua/libs/cua-driver/rust/Skills/cua-driver .claude/skills/```-Or run the verb that does this automatically + fetches the matching-release version from GitHub:--```bash-cua-driver skills install-```-See `cua-driver skills --help` for the full subcommand list(`install` / `update` / `uninstall` / `status` / `path`).@@ -162,7 +184,7 @@ Numbers", "open Calculator and compute 17×23", "navigate totrycua.com in Edge". You can also invoke it explicitly:```-/cua-driver-rs+/cua-driver```## Claude Code MCP compatibility mode@@ -195,7 +217,8 @@ Use MCP for this Claude Code vision/computer-use-style path. CLI screenshots sticoverage lives in `WINDOWS.md`'s "Web apps on Windows" section.- `RECORDING.md` — trajectory recording / replay (macOS and LinuxX11 today; Wayland video capture not yet supported).-- `TESTS.md` — manual test scripts for end-to-end skill verification.+- `TESTS.md` — manual end-to-end verification scripts for repository+ contributors. ClawHub release bundles intentionally exclude this file.## Troubleshooting@@ -210,7 +233,7 @@ Use MCP for this Claude Code vision/computer-use-style path. CLI screenshots sti— `get_window_state` always walks the tree now. Act by pixel off thescreenshot already in the same response (an element px action).Tiny screenshot → likely a stale window capture. See the perception- + element ax/px action notes in SKILL.md.+ and element ax/px action notes in SKILL.md.- System-alert beep when pressing Return on a minimized Chromeomnibox → the keyboard-commit-on-minimized limitation. Use`set_value` on the field instead, or AX-click a Go/Submit button.@@ -233,4 +256,7 @@ cp -R libs/cua-driver/rust/Skills/cua-driver ~/.claude/skills/## License-MIT. Same license as the parent `trycua/cua` repo.+The source files in the `trycua/cua` repository remain licensed under MIT.+Copies published through ClawHub are distributed under MIT-0, as required for+all ClawHub skills. MIT-0 permits use, modification, and redistribution without+an attribution requirement.diff --git a/libs/cua-driver/rust/Skills/cua-driver/SKILL.md b/libs/cua-driver/rust/Skills/cua-driver/SKILL.mdindex 720a99b42b..98f36dcd0c 100644--- a/libs/cua-driver/rust/Skills/cua-driver/SKILL.md+++ b/libs/cua-driver/rust/Skills/cua-driver/SKILL.md@@ -1,9 +1,35 @@----name: cua-driver-rs-description: Drive a native GUI app (macOS, Windows, Linux) via the cua-driver CLI (default) or MCP server — snapshot its accessibility tree, click/type/scroll by element_index or pixel coords, verify via re-snapshot, all without bringing the target to the foreground. Use when the user asks you to operate, drive, automate, or perform a GUI task in a real application on the host.+name: cua-driver+description: Drive a native GUI app (macOS, Windows, Linux) via the cua-driver CLI (default) or MCP server; snapshot its accessibility tree, click/type/scroll by element_index or pixel coordinates, and verify via re-snapshot without bringing the target to the foreground. Use when the user asks you to operate, drive, automate, or perform a GUI task in a real application on the host.+version: 0.8.3+metadata:+ openclaw:+ requires:+ bins:+ - cua-driver+ envVars:+ - name: CUA_DRIVER_CDP_PORT+ required: false+ description: Optional Chromium DevTools Protocol port for browser automation.+ - name: CUA_DRIVER_EMBEDDED+ required: false+ description: Set to 1 when a macOS host app launches the driver in embedded mode.+ - name: CUA_DRIVER_HOST_BUNDLE_ID+ required: false+ description: Bundle identifier of the macOS host app in embedded mode.+ - name: CUA_DRIVER_PATH+ required: false+ description: Optional path to a cua-driver binary used by an embedding host.+ - name: CUA_DRIVER_RS_ENABLE_WAYLAND+ required: false+ description: Set to 1 to enable the preview native Wayland backend.+ - name: CUA_DRIVER_RS_MCP_HTTP_PORT+ required: false+ description: Optional port for the local MCP HTTP endpoint.+ homepage: https://cua.ai/docs/cua-driver----# cua-driver-rs+# cua-driverOrchestrates cross-platform app automation via `cua-driver`. Whenevera user asks to drive a native app, follow the loop in this skill@@ -34,7 +60,8 @@ Cross-cutting topics also have their own files:browser / Electron / Tauri fallbacks (sparse AX trees, omnibox navigation,minimized windows, and tabs-vs-windows guidance).- `RECORDING.md` — session recording + `replay_trajectory`.-- `TESTS.md` — internal test surface.+- `TESTS.md` in the source repository contains the internal test surface. It is+ intentionally excluded from ClawHub release bundles.Use whichever combination matches the host. When in doubt, run`cua-driver doctor` — it reports the platform and the right entrydiff --git a/libs/cua-driver/rust/crates/cua-driver/src/skills.rs b/libs/cua-driver/rust/crates/cua-driver/src/skills.rsindex 899a46d80c..ff43f919ba 100644--- a/libs/cua-driver/rust/crates/cua-driver/src/skills.rs+++ b/libs/cua-driver/rust/crates/cua-driver/src/skills.rs@@ -83,6 +83,7 @@ const SKILL_FILES: &[&str] = &["WEB_APPS.md","RECORDING.md","TESTS.md",+ "EMBEDDING.md",];/// Per-host filter: returns the platform-specific docs that should NOT@@ -698,7 +699,8 @@ fn print_path() -> Result<()> {#[cfg(test)]mod tests {- use super::extract_tar_gz;+ use super::{extract_tar_gz, SKILL_FILES};+ use std::path::PathBuf;use tempfile::tempdir;/// Build a gzipped tarball with the entries given as@@ -720,6 +722,27 @@ mod tests {gz_buf}+ #[test]+ fn from_main_manifest_matches_canonical_markdown_files() {+ let skill_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR"))+ .join("../../Skills/cua-driver");+ let mut canonical = std::fs::read_dir(&skill_dir)+ .unwrap_or_else(|error| panic!("failed to read {}: {error}", skill_dir.display()))+ .map(|entry| entry.unwrap().path())+ .filter(|path| path.extension().and_then(|extension| extension.to_str()) == Some("md"))+ .map(|path| path.file_name().unwrap().to_string_lossy().into_owned())+ .collect::<Vec<_>>();+ canonical.sort();++ let mut manifest = SKILL_FILES.iter()+ .map(|file| (*file).to_owned())+ .collect::<Vec<_>>();+ manifest.sort();++ assert_eq!(manifest, canonical,+ "SKILL_FILES must include every canonical Markdown file");+ }+#[test]fn extract_flat_tarball_v_0_2_20_plus() {// Post-fix shape: one wrapper dir, files directly under it.@@ -804,11 +827,12 @@ mod tests {("cua-driver-rs-v0.2.20-skills/RECORDING.md",b"R"),("cua-driver-rs-v0.2.20-skills/WEB_APPS.md", b"W"),("cua-driver-rs-v0.2.20-skills/TESTS.md", b"T"),+ ("cua-driver-rs-v0.2.20-skills/EMBEDDING.md",b"E"),]);let dest = tempdir().unwrap();extract_tar_gz(&bytes, dest.path(), /*all_platforms=*/ false).unwrap();// README + SKILL + cross-platform docs ALWAYS present.- for f in ["README.md", "SKILL.md", "RECORDING.md", "WEB_APPS.md", "TESTS.md"] {+ for f in ["README.md", "SKILL.md", "RECORDING.md", "WEB_APPS.md", "TESTS.md", "EMBEDDING.md"] {assert!(dest.path().join(f).exists(),"{f} should be present after per-host extraction");}