Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
ci(cua-driver): alert when the canonical installer breaks (#4152)
Francesco Bonaccib5203864d6ee
diff --git a/.github/scripts/tests/test_cua_driver_installer_canary.py b/.github/scripts/tests/test_cua_driver_installer_canary.pynew file mode 100644index 0000000000..4e214eb5bc--- /dev/null+++ b/.github/scripts/tests/test_cua_driver_installer_canary.py@@ -0,0 +1,157 @@+"""The canonical-installer canary runs what users run and files one issue on failure."""++from __future__ import annotations++import os+from pathlib import Path+import subprocess+import textwrap++import pytest+import yaml+++REPO_ROOT = Path(__file__).resolve().parents[3]+WORKFLOW = REPO_ROOT / ".github/workflows/monitor-branded-installers.yml"++[email protected](scope="module")+def workflow() -> dict:+ return yaml.safe_load(WORKFLOW.read_text(encoding="utf-8"))+++def step(job: dict, name: str) -> dict:+ matches = [entry for entry in job["steps"] if entry.get("name") == name]+ assert len(matches) == 1, name+ return matches[0]+++def test_canary_runs_often_and_after_every_installer_change(workflow: dict) -> None:+ triggers = workflow[True] # PyYAML parses `on` as True+ assert triggers["schedule"] == [{"cron": "15 */6 * * *"}]+ assert triggers["release"] == {"types": ["published"]}+ assert triggers["workflow_run"] == {+ "workflows": ["CD: Cua Driver (cross-platform)"],+ "types": ["completed"],+ }+ paths = triggers["push"]["paths"]+ assert triggers["push"]["branches"] == ["main"]+ for required in (+ ".github/release-state/cua-driver-rs-*",+ "libs/cua-driver/scripts/_install-rust.sh",+ "libs/cua-driver/scripts/install.sh",+ "libs/cua-driver/scripts/install.ps1",+ ):+ assert required in paths+ assert "pull_request" not in triggers and "pull_request_target" not in triggers+ assert workflow["permissions"] == {"contents": "read", "issues": "write"}+++def test_canary_uses_default_resolution_on_every_platform(workflow: dict) -> None:+ canary = workflow["jobs"]["canary"]+ assert canary["strategy"]["matrix"]["os"] == ["ubuntu-latest", "macos-26", "windows-latest"]+ assert canary["env"]["INSTALLER_SOURCE"] == "${{ github.event_name == 'push' && 'checkout' || 'cua.ai' }}"+ assert canary["env"]["CUA_DRIVER_RS_TELEMETRY_ENABLED"] == "false"+ # The only pin is the manual diagnostics input; scheduled runs never pin.+ assert canary["env"]["CANARY_PIN"] == "${{ inputs.version }}"++ unix = step(canary, "Run the canonical macOS/Linux installer with default resolution")["run"]+ assert '/bin/bash -c "$(curl -fsSL https://cua.ai/driver/install.sh)"' in unix+ assert "unset CUA_DRIVER_RS_VERSION CUA_DRIVER_VERSION GH_TOKEN GITHUB_TOKEN" in unix+ assert unix.index("unset CUA_DRIVER_RS_VERSION") < unix.index('export CUA_DRIVER_RS_VERSION="$CANARY_PIN"')+ for isolation in ('export HOME="$CANARY/home"', 'export CUA_DRIVER_RS_HOME=', 'export CUA_DRIVER_RS_INSTALL_DIR="$CANARY/bin"'):+ assert isolation in unix+ assert 'grep -Fqx "source=Notarized Developer ID"' in unix+ assert "\\(YCK386LBJ7\\)$" in unix+ assert "codesign --verify --deep --strict" in unix++ windows = step(canary, "Run the canonical Windows installer with default resolution")["run"]+ assert "[scriptblock]::Create((Invoke-RestMethod https://cua.ai/driver/install.ps1))) -NoAutoStart -NoPathUpdate" in windows+ assert "& ./libs/cua-driver/scripts/install.ps1 -NoAutoStart -NoPathUpdate" in windows+ assert "Get-AuthenticodeSignature" in windows+ assert "$env:CUA_DRIVER_RS_HOME = Join-Path $canary" in windows+++def test_failures_open_one_bug_issue_per_canary(workflow: dict) -> None:+ report = workflow["jobs"]["report"]+ assert set(report["needs"]) == {"check", "canary"}+ assert report["if"] == "always() && (needs.check.result == 'failure' || needs.canary.result == 'failure')"+ run = step(report, "Open or update monitoring issues")["run"]+ assert "--label bug" in run+ assert "Cua Driver canonical installer canary is failing" in run+ assert "Branded installer endpoint check is failing" in run+ assert "actions/runs/${GITHUB_RUN_ID}" in run+++def _run_report(+ tmp_path: Path, workflow: dict, *, open_issues: str, env: dict[str, str]+) -> tuple[list[list[str]], str]:+ run = step(workflow["jobs"]["report"], "Open or update monitoring issues")["run"]+ calls = tmp_path / "gh-calls"+ script = tmp_path / "report.sh"+ script.write_text(+ textwrap.dedent(+ f"""\+ gh() {{+ printf '%s\\037' "$@" >> "{calls.as_posix()}"+ printf '\\036' >> "{calls.as_posix()}"+ if [[ "$1 $2" == "issue list" ]]; then+ printf '%s' "$OPEN_ISSUE"+ fi+ }}+ """+ )+ + run,+ encoding="utf-8",+ )+ result = subprocess.run(+ ["bash", str(script)],+ capture_output=True,+ text=True,+ env={+ "PATH": os.environ["PATH"],+ "GITHUB_REPOSITORY": "trycua/cua",+ "GITHUB_RUN_ID": "42",+ "GITHUB_SHA": "abc123",+ "TRIGGER": "schedule",+ "DRY_RUN": "false",+ "ENDPOINT_RESULT": "success",+ "ENDPOINT_FAILURES": "",+ "CANARY_RESULT": "success",+ "OPEN_ISSUE": open_issues,+ **env,+ },+ )+ assert result.returncode == 0, result.stderr+ records = calls.read_text().split("\x1e") if calls.exists() else []+ return [record.strip("\x1f").split("\x1f") for record in records if record], result.stdout+++def test_report_creates_then_comments_instead_of_duplicating(tmp_path: Path, workflow: dict) -> None:+ calls, _ = _run_report(tmp_path, workflow, open_issues="", env={"CANARY_RESULT": "failure"})+ assert calls[0][:5] == ["issue", "list", "--repo", "trycua/cua", "--state"]+ assert calls[1][:8] == [+ "issue", "create", "--repo", "trycua/cua",+ "--title", "Cua Driver canonical installer canary is failing",+ "--label", "bug",+ ]+ assert calls[1][8] == "--body"+ assert "Workflow run: https://github.com/trycua/cua/actions/runs/42" in calls[1][9]+ assert len(calls) == 2++ (tmp_path / "gh-calls").unlink()+ calls, _ = _run_report(tmp_path, workflow, open_issues="4200", env={"CANARY_RESULT": "failure"})+ assert [call[:3] for call in calls if call[1] != "list"] == [["issue", "comment", "4200"]]+++def test_report_dry_run_files_nothing(tmp_path: Path, workflow: dict) -> None:+ calls, stdout = _run_report(+ tmp_path,+ workflow,+ open_issues="",+ env={"CANARY_RESULT": "failure", "ENDPOINT_RESULT": "failure", "ENDPOINT_FAILURES": "x returned HTTP 500", "DRY_RUN": "true"},+ )+ assert calls == []+ assert 'would open or update the issue titled "Cua Driver canonical installer canary is failing"' in stdout+ assert 'would open or update the issue titled "Branded installer endpoint check is failing"' in stdout+ assert "x returned HTTP 500" in stdoutdiff --git a/.github/workflows/monitor-branded-installers.yml b/.github/workflows/monitor-branded-installers.ymlindex 8d80265ff6..ab792af901 100644--- a/.github/workflows/monitor-branded-installers.yml+++ b/.github/workflows/monitor-branded-installers.yml@@ -1,22 +1,74 @@name: Monitor branded installer endpoints+# Two canaries for what users actually run:+#+# check the branded endpoints are served with the expected content.+# canary the canonical Cua Driver installers install a working, signed+# driver with DEFAULT version resolution (no pin) on macOS, Linux,+# and Windows, exactly as the documented one-liners do:+# /bin/bash -c "$(curl -fsSL https://cua.ai/driver/install.sh)"+# & ([scriptblock]::Create((irm https://cua.ai/driver/install.ps1))) -NoAutoStart -NoPathUpdate+#+# A failure opens (or comments on) one deduplicated `bug` issue per canary so+# a broken default install is noticed within hours, not when a user reports+# it (#4109: the installers baked a release whose macOS app was unsigned).+#+# The canary installs into temporary HOME, package-home, and bin directories.+# On macOS the installer also writes /Applications/CuaDriver.app, which is+# acceptable only because hosted runners are ephemeral. Windows runs with+# -NoAutoStart -NoPathUpdate so no scheduled task or PATH entry is created.+on:schedule:- # Run daily; the first scheduled run will be the next day after merge.- - cron: '15 14 * * *'+ - cron: "15 */6 * * *"+ release:+ types: [published]+ # The installer bake commit uses [skip ci]; follow the CD run instead so a+ # newly baked version is exercised as soon as it lands.+ workflow_run:+ workflows: ["CD: Cua Driver (cross-platform)"]+ types: [completed]+ push:+ branches: [main]+ paths:+ - ".github/release-state/cua-driver-rs-*"+ - ".github/workflows/monitor-branded-installers.yml"+ - "libs/cua-driver/scripts/_install-common.psm1"+ - "libs/cua-driver/scripts/_install-common.sh"+ - "libs/cua-driver/scripts/_install-rust.sh"+ - "libs/cua-driver/scripts/install.ps1"+ - "libs/cua-driver/scripts/install.sh"workflow_dispatch:+ inputs:+ version:+ description: "Diagnostics only: pin this x.y.z instead of default resolution"+ required: false+ default: ""+ dry_run:+ description: "Print the issue that would be filed instead of filing it"+ required: false+ type: boolean+ default: falsepermissions:contents: readissues: writeconcurrency:- group: monitor-branded-installers+ # Scheduled and event runs supersede each other; manual diagnostics never+ # cancel one another.+ group: monitor-branded-installers-${{ github.event_name }}-${{ github.ref }}${{ github.event_name == 'workflow_dispatch' && format('-{0}', github.run_id) || '' }}cancel-in-progress: truejobs:check:+ if: >-+ github.event_name != 'release' ||+ startsWith(github.event.release.tag_name, 'cua-driver-rs-v') ||+ startsWith(github.event.release.tag_name, 'lume-v')runs-on: ubuntu-latest+ outputs:+ failures: ${{ steps.endpoints.outputs.failures }}steps:- name: Check public installer endpointsid: endpoints@@ -70,31 +122,190 @@ jobs:echo 'failures=' >> "$GITHUB_OUTPUT"- - name: Open or update monitoring issue- if: failure() && steps.endpoints.outputs.failures != ''+ canary:+ name: "canonical installer / ${{ matrix.os }} (${{ github.event_name == 'push' && 'pushed scripts' || 'cua.ai' }})"+ if: >-+ github.event_name != 'release' ||+ startsWith(github.event.release.tag_name, 'cua-driver-rs-v')+ strategy:+ fail-fast: false+ matrix:+ os: [ubuntu-latest, macos-26, windows-latest]+ runs-on: ${{ matrix.os }}+ timeout-minutes: 20+ env:+ # Pushes exercise the pushed installer and release state directly, so a+ # change is tested before the branded endpoints finish propagating it.+ # Every other trigger runs the public branded endpoints users run.+ INSTALLER_SOURCE: ${{ github.event_name == 'push' && 'checkout' || 'cua.ai' }}+ CANARY_PIN: ${{ inputs.version }}+ CUA_DRIVER_RS_TELEMETRY_ENABLED: "false"+ CUA_TELEMETRY_ENABLED: "false"+ steps:+ - uses: actions/checkout@v4+ with:+ persist-credentials: false+ sparse-checkout: |+ .github/release-state+ libs/cua-driver/scripts++ - name: Install Linux runtime libraries+ if: runner.os == 'Linux'+ run: |+ sudo apt-get update+ sudo apt-get install -y --no-install-recommends \+ libx11-6 libxi6 libxtst6 libxext6 libwayland-client0 libxkbcommon0++ - name: Run the canonical macOS/Linux installer with default resolution+ if: runner.os != 'Windows'+ shell: bash+ run: |+ set -euo pipefail+ CANARY="$RUNNER_TEMP/cua-driver-canary"+ rm -rf "$CANARY"+ mkdir -p "$CANARY/home"+ # Isolate every per-user location the installer writes. No version+ # pin and no GitHub token: resolution is exactly what users get.+ export HOME="$CANARY/home"+ export CUA_DRIVER_RS_HOME="$CANARY/home/.cua-driver"+ export CUA_DRIVER_RS_INSTALL_DIR="$CANARY/bin"+ export CUA_DRIVER_RS_NO_MODIFY_PATH=1+ unset CUA_DRIVER_RS_VERSION CUA_DRIVER_VERSION GH_TOKEN GITHUB_TOKEN+ if [[ -n "${CANARY_PIN:-}" ]]; then+ echo "::notice::Diagnostics run pinned to ${CANARY_PIN}; this is not default resolution."+ export CUA_DRIVER_RS_VERSION="$CANARY_PIN"+ fi+ if [[ "$INSTALLER_SOURCE" == "checkout" ]]; then+ /bin/bash libs/cua-driver/scripts/install.sh 2>&1 | tee "$CANARY/install.log"+ else+ /bin/bash -c "$(curl -fsSL https://cua.ai/driver/install.sh)" 2>&1 | tee "$CANARY/install.log"+ fi++ INSTALLED="$CUA_DRIVER_RS_INSTALL_DIR/cua-driver"+ test -x "$INSTALLED"+ VERSION_OUTPUT=$("$INSTALLED" --version)+ echo "Installed: $VERSION_OUTPUT"+ PUBLISHED=$(tr -d '[:space:]' < .github/release-state/cua-driver-rs-published-version)+ PUBLISHED="${CANARY_PIN:-$PUBLISHED}"+ if ! grep -Fq "$PUBLISHED" <<< "$VERSION_OUTPUT"; then+ echo "::warning::Installed '$VERSION_OUTPUT', but main publishes $PUBLISHED (the branded endpoint may still be propagating)."+ fi+ if [[ "$RUNNER_OS" == "macOS" ]]; then+ APP=/Applications/CuaDriver.app+ test -x "$APP/Contents/MacOS/cua-driver"+ /usr/libexec/PlistBuddy -c Print:CFBundleIdentifier "$APP/Contents/Info.plist" |+ grep -Fx com.trycua.driver+ codesign --verify --deep --strict --verbose=2 "$APP"+ # Gatekeeper is what blocks a user's first launch; require the+ # notarized Developer ID assessment, not just a valid signature.+ ASSESSMENT=$(spctl -a -vvv -t exec "$APP" 2>&1)+ echo "$ASSESSMENT"+ grep -Fqx "source=Notarized Developer ID" <<< "$ASSESSMENT"+ grep -Eq '^origin=Developer ID Application: .+ \(YCK386LBJ7\)$' <<< "$ASSESSMENT"+ fi++ - name: Run the canonical Windows installer with default resolution+ if: runner.os == 'Windows'+ shell: pwsh+ run: |+ $ErrorActionPreference = 'Stop'+ $canary = Join-Path $env:RUNNER_TEMP "cua-driver-canary"+ Remove-Item -LiteralPath $canary -Recurse -Force -ErrorAction SilentlyContinue+ $env:CUA_DRIVER_RS_HOME = Join-Path $canary "home"+ $env:CUA_DRIVER_RS_INSTALL_DIR = Join-Path $canary "bin"+ foreach ($name in 'CUA_DRIVER_RS_VERSION', 'GH_TOKEN', 'GITHUB_TOKEN') {+ Remove-Item "Env:$name" -ErrorAction SilentlyContinue+ }+ if ($env:CANARY_PIN) {+ Write-Host "::notice::Diagnostics run pinned to $($env:CANARY_PIN); this is not default resolution."+ $env:CUA_DRIVER_RS_VERSION = $env:CANARY_PIN+ }+ if ($env:INSTALLER_SOURCE -eq 'checkout') {+ & ./libs/cua-driver/scripts/install.ps1 -NoAutoStart -NoPathUpdate+ }+ else {+ & ([scriptblock]::Create((Invoke-RestMethod https://cua.ai/driver/install.ps1))) -NoAutoStart -NoPathUpdate+ }+ $binary = Join-Path $env:CUA_DRIVER_RS_INSTALL_DIR "cua-driver.exe"+ if (-not (Test-Path -LiteralPath $binary)) {+ throw "installer did not create $binary"+ }+ $versionOutput = & $binary --version+ if ($LASTEXITCODE -ne 0) {+ throw "installed binary exited with $LASTEXITCODE"+ }+ Write-Host "Installed: $versionOutput"+ $published = (Get-Content -Raw .github/release-state/cua-driver-rs-published-version).Trim()+ if ($env:CANARY_PIN) { $published = $env:CANARY_PIN }+ if ($versionOutput -notmatch [regex]::Escape($published)) {+ Write-Host "::warning::Installed '$versionOutput', but main publishes $published (the branded endpoint may still be propagating)."+ }+ $signature = Get-AuthenticodeSignature -LiteralPath $binary+ $signature | Format-List Status, StatusMessage, SignerCertificate+ if ($signature.Status -ne 'Valid' -or $signature.SignerCertificate.Subject -notmatch 'CN="?Cua AI, Inc\.') {+ throw "installed cua-driver.exe is not validly signed by Cua AI, Inc.: $($signature.StatusMessage)"+ }++ report:+ name: Report installer canary failures+ needs: [check, canary]+ if: always() && (needs.check.result == 'failure' || needs.canary.result == 'failure')+ runs-on: ubuntu-latest+ steps:+ - name: Open or update monitoring issuesenv:GH_TOKEN: ${{ github.token }}- FAILURES: ${{ steps.endpoints.outputs.failures }}+ ENDPOINT_RESULT: ${{ needs.check.result }}+ ENDPOINT_FAILURES: ${{ needs.check.outputs.failures }}+ CANARY_RESULT: ${{ needs.canary.result }}+ TRIGGER: ${{ github.event_name }}+ DRY_RUN: ${{ inputs.dry_run == true }}shell: bashrun: |set -euo pipefail- title='Branded installer endpoint check is failing'- body=$(cat <<EOF- The daily branded installer monitor found one or more failures.+ RUN_URL="https://github.com/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"++ # One open issue per title; later failures comment on it.+ report() {+ local title="$1" body="$2" issue_number+ if [[ "$DRY_RUN" == "true" ]]; then+ printf 'Dry run: would open or update the issue titled "%s" (label bug) with:\n%s\n' "$title" "$body"+ return+ fi+ issue_number=$(gh issue list --repo "$GITHUB_REPOSITORY" --state open \+ --search "\"$title\" in:title" --json number,title \+ --jq "map(select(.title == \"$title\")) | .[0].number // empty")+ if [[ -n "$issue_number" ]]; then+ gh issue comment "$issue_number" --repo "$GITHUB_REPOSITORY" --body "$body"+ else+ gh issue create --repo "$GITHUB_REPOSITORY" --title "$title" --label bug --body "$body"+ fi+ }++ if [[ "$ENDPOINT_RESULT" == "failure" ]]; then+ report "Branded installer endpoint check is failing" "$(cat <<EOF+ The branded installer monitor found one or more failures.- ```- ${FAILURES}- ```+ \`\`\`+ ${ENDPOINT_FAILURES:-see the workflow run}+ \`\`\`- Workflow run: https://github.com/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}+ Workflow run: ${RUN_URL}+ Trigger: ${TRIGGER}Checked at: ${GITHUB_SHA}EOF- )+ )"+ fi- issue_number=$(gh issue list --repo "$GITHUB_REPOSITORY" --state open \- --search "$title in:title" --json number --jq '.[0].number // empty')- if [[ -n "$issue_number" ]]; then- gh issue comment "$issue_number" --repo "$GITHUB_REPOSITORY" --body "$body"- else- gh issue create --repo "$GITHUB_REPOSITORY" --title "$title" --body "$body"+ if [[ "$CANARY_RESULT" == "failure" ]]; then+ report "Cua Driver canonical installer canary is failing" "$(cat <<EOF+ The canonical Cua Driver installer failed with default version resolution on at least one platform. Fresh installs are likely broken for users on that platform.++ Workflow run: ${RUN_URL}+ Trigger: ${TRIGGER}+ Checked at: ${GITHUB_SHA}++ To roll the installers back to the last good release, follow the release safety runbook in scripts/ci/README.md.+ EOF+ )"fi