Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
ci(rust): build Linux tests once, run in 3 shards (CUA-3560, audit cut 5) (#6595)
Francesco Bonacci2393194bd5bb
diff --git a/.github/actions/nightly-issue/action.yml b/.github/actions/nightly-issue/action.ymlnew file mode 100644index 0000000000..34e67d5626--- /dev/null+++ b/.github/actions/nightly-issue/action.yml@@ -0,0 +1,43 @@+name: "Open or update a nightly failure issue"+description: >-+ A job that moved off pull requests to the nightly schedule (or to a path+ filter on main) must still be heard when it fails. Call this from a final+ job with `if: failure()` and `permissions: issues: write`: it opens ONE issue+ per title, or comments on the open one, with the run link. The issue carries+ the `flaky` and `ci-nightly` labels so the weekly "Flaky tests this week"+ digest and the people who watch CI issues see it. Needs actions/checkout of+ this directory first (sparse-checkout: .github/actions/nightly-issue).++inputs:+ title:+ description: "Issue title; one open issue per title."+ required: true+ detail:+ description: "One or two plain sentences on what failed and where to look."+ required: true++runs:+ using: composite+ steps:+ - name: Open or update the issue+ shell: bash+ env:+ GH_TOKEN: ${{ github.token }}+ TITLE: ${{ inputs.title }}+ DETAIL: ${{ inputs.detail }}+ RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}+ SHA: ${{ github.sha }}+ run: |+ set -euo pipefail+ gh label create ci-nightly --repo "$GITHUB_REPOSITORY" --color FBCA04 \+ --description "A nightly or main-only CI job failed" 2>/dev/null || true+ body="$DETAIL"$'\n\n'"Run: $RUN_URL"$'\n'"Commit: $SHA"+ issue=$(gh issue list --repo "$GITHUB_REPOSITORY" --state open --label ci-nightly \+ --search "\"$TITLE\" in:title" --json number,title \+ --jq "map(select(.title == \"$TITLE\")) | .[0].number // empty")+ if [[ -n "$issue" ]]; then+ gh issue comment "$issue" --repo "$GITHUB_REPOSITORY" --body "$body"+ else+ gh issue create --repo "$GITHUB_REPOSITORY" --title "$TITLE" --body "$body" \+ --label ci-nightly --label flaky+ fidiff --git a/.github/scripts/tests/test_ci_cua_workspace_tests.py b/.github/scripts/tests/test_ci_cua_workspace_tests.pyindex 326c25c3fa..2a89f06f85 100644--- a/.github/scripts/tests/test_ci_cua_workspace_tests.py+++ b/.github/scripts/tests/test_ci_cua_workspace_tests.py@@ -1,6 +1,6 @@"""Every libs/cua crate that has tests runs them in GitHub CI, or says why not.-The Linux Rust lane (ci-cua-sdk.yml, `rust-linux`) lists its crates by hand+The Linux Rust lane (ci-cua-sdk.yml, `rust-linux-build`) lists its crates by handwith `-p`. A crate missing from the list is never tested there, and a pathfilter cannot see that: cua-auth's tests were skipped this way, so threebroken flows tests merged green (CUA-3363). This test makes the list and the@@ -65,18 +65,19 @@ def tested_packages():for step in job.get("steps") or []:run = (step.get("run") or "").replace("\\\n", " ")for cmd in re.split(r"\n|&&|;", run):- if re.search(r"cargo (nextest run|test)\b", cmd):+ if re.search(r"cargo (nextest (run|archive)|test)\b", cmd):found |= set(re.findall(r"(?:^|\s)(?:-p|--package)[ =](\S+)", cmd))return frozenset(found)def nextest_packages():jobs = yaml.safe_load(WORKFLOW.read_text())["jobs"]- for step in jobs["rust-linux"]["steps"]:- run = step.get("run", "")- if "cargo nextest run" in run:+ # The test binaries are built once, by `cargo nextest archive`, and the shards run the archive.+ for step in jobs["rust-linux-build"]["steps"]:+ run = step.get("run", "").replace("\\\n", " ")+ if "cargo nextest archive" in run:return set(re.findall(r"(?:^|\s)-p\s+(\S+)", run))- raise AssertionError("rust-linux has no cargo nextest step")+ raise AssertionError("rust-linux-build has no cargo nextest archive step")TESTED = sorted(package_name(m) for m in members() if has_tests(m))@@ -86,7 +87,7 @@ TESTED = sorted(package_name(m) for m in members() if has_tests(m))def test_a_crate_with_tests_runs_them_in_the_linux_lane_or_says_why_not(name):assert name in tested_packages() or name in ELSEWHERE, (f"{name} has tests but is not named with `-p` on any `cargo test` or `cargo nextest` step in "- f".github/workflows: its tests never run on GitHub. Add it to the rust-linux nextest list in ci-cua-sdk.yml, or to ELSEWHERE with the reason."+ f".github/workflows: its tests never run on GitHub. Add it to the rust-linux-build nextest archive list in ci-cua-sdk.yml, or to ELSEWHERE with the reason.")diff --git a/.github/scripts/tests/test_ci_reuse_pr_result.py b/.github/scripts/tests/test_ci_reuse_pr_result.pyindex 23d861bcf2..cffa6b1492 100644--- a/.github/scripts/tests/test_ci_reuse_pr_result.py+++ b/.github/scripts/tests/test_ci_reuse_pr_result.py@@ -436,6 +436,37 @@ def test_cua_sdk_rust_linux_leg_is_sharded_behind_the_reused_check_name():assert "ci:full" in job["if"]+def test_cua_sdk_rust_tests_are_built_once_and_run_from_the_archive():+ wf = load("ci-cua-sdk.yml")+ build, shards = wf["jobs"]["rust-linux-build"], wf["jobs"]["rust-linux"]+ assert build["needs"] == "reuse" and shards["needs"] == ["reuse", "rust-linux-build"]+ # Same conditions: the build runs exactly when the shards do.+ assert build["if"] == shards["if"]+ build_text, shard_text = str(build["steps"]), str(shards["steps"])+ assert "cargo nextest archive" in build_text and "rust-linux-tests" in build_text+ assert "cargo nextest run" in shard_text and "--archive-file" in shard_text and "--workspace-remap" in shard_text+ # Integration tests bake CARGO_BIN_EXE_* and CARGO_TARGET_TMPDIR in at compile time: the archive must be+ # unpacked into the build's target directory and target/tmp must exist, or every cua-cli test fails to spawn.+ assert "--extract-to ." in shard_text and "--extract-overwrite" in shard_text and "mkdir -p target/tmp" in shard_text+ # The shards never compile the nextest packages again: they only restore the cache.+ assert "cargo nextest run --locked" not in shard_text+ assert any(s.get("with", {}).get("save-if") == "false" for s in shards["steps"] if "rust-cache" in str(s.get("uses", "")))+ assert not any("save-if" in s.get("with", {}) for s in build["steps"] if "rust-cache" in str(s.get("uses", "")))+++def test_cua_sdk_pool_model_checks_leave_pull_requests_but_run_on_main_and_nightly():+ text = (ROOT / ".github/workflows/ci-cua-sdk.yml").read_text()+ wf = load("ci-cua-sdk.yml")+ assert "schedule" in (wf.get("on") or wf[True])+ run = next(s["run"] for s in wf["jobs"]["rust-linux"]["steps"] if "cargo nextest run" in s.get("run", ""))+ # Excluded only on pull requests without `ci:full`; main, the nightly schedule and `ci:full` run them.+ assert "github.event_name == 'pull_request' && !contains(github.event.pull_request.labels.*.name, 'ci:full')" in run+ for name in ("pool_protocol_is_safe", "mutation_no_epoch_check_breaks_single_writer", "finding_stale_holder_cancels_current_attempt"):+ assert f"test(={name})" in run+ assert f"fn {name}()" in (ROOT / "libs/cua/crates/cua-spaces-cluster/tests/pool_model.rs").read_text()+ assert "pool_model" in text++def test_cua_sdk_jobs_in_the_subset_skip_only_when_it_is_reused():wf = load("ci-cua-sdk.yml")skip = "needs.reuse.outputs.pr_subset != 'true'"diff --git a/.github/workflows/README.md b/.github/workflows/README.mdindex 4d18b41b71..ea289814f3 100644--- a/.github/workflows/README.md+++ b/.github/workflows/README.md@@ -89,6 +89,16 @@ workflow.* sccache hashes the rustc version into every object key, so a toolchain bumpsimply misses until main re-seeds, and the cap ages the old objects out.+## Rust tests: build once, run in shards++`CI: cua SDK` compiles the Linux test binaries once (`rust linux build`, a+`cargo nextest archive` uploaded as the `rust-linux-tests` artifact) and the three+`rust linux shard N/3` jobs run `cargo nextest run --archive-file ... --partition+hash:N/3`. The three `pool_model` model checks (about 254, 112 and 62 s) are left out+of pull request runs and run on every push to main, in the nightly schedule and on a+pull request with the `ci:full` label. Linux links with rust-lld (rustc's default+since 1.90); CI test builds already have debug info off, so the archive stays small.+## Known flaky tests and quarantineSee [`FLAKY.md`](FLAKY.md). A test goes on the list with an owner and a fixdiff --git a/.github/workflows/ci-cua-sdk.yml b/.github/workflows/ci-cua-sdk.ymlindex 29abb90e7a..f47ff07ce6 100644--- a/.github/workflows/ci-cua-sdk.yml+++ b/.github/workflows/ci-cua-sdk.yml@@ -192,15 +192,85 @@ jobs:exit 1fi- # The Linux leg of `rust`, in three shards. Each shard builds (one cache- # family, saved by shard 1 only) and runs a third of the tests with- # cargo-nextest partitions. Doctests (not run by nextest) and the budget- # tests that need to run alone stay in cargo test on shards 1 and 2.+ # The Linux leg of `rust`: build the test binaries once, run them in three+ # shards. Compiling and linking the workspace's 176 test binaries took 11 to+ # 19 minutes, and every shard did it again for 2 to 5 minutes of tests.+ # `rust-linux-build` compiles them once into a cargo-nextest archive (and+ # warms the sccache entries the shards' own `cargo test` steps use); each+ # shard downloads the archive and runs a third of the tests from it with+ # nextest partitions. Doctests (not run by nextest) and the budget tests that+ # need to run alone stay in cargo test on shards 1 and 2.+ #+ # Linker: x86_64 Linux builds link with rust-lld, the default of every rustc+ # since 1.90 (the toolchain is pinned to 1.97.1), so there is no mold or+ # apt lld to install. Debug info is already off for CI tests+ # (CARGO_PROFILE_DEV_DEBUG=0 above, lighter than `line-tables-only`, which+ # libs/cua/Cargo.toml keeps for local builds): that keeps the archive small.+ # macOS and Windows keep their platform linkers; swapping them is not safe+ # without a run on each OS.+ rust-linux-build:+ if: ${{ !cancelled() && (github.event_name != 'pull_request' || !github.event.pull_request.draft) && !startsWith(github.head_ref, 'release-please--branches--') && !((github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'ci:full')) && needs.reuse.outputs.pr_subset == 'true') }}+ name: rust linux build+ runs-on: ubuntu-latest+ needs: reuse+ timeout-minutes: 60+ defaults:+ run:+ working-directory: libs/cua+ shell: bash+ steps:+ - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4+ # Its debug test binaries outgrow the default cleanup's 43 GB.+ - uses: ./.github/actions/linux-rust-prep+ with:+ free-more: "true"+ - name: Install protoc+ uses: ./.github/actions/setup-protoc+ - name: Install pinned Rust toolchain+ uses: dtolnay/[email protected]+ with:+ components: clippy, rustfmt+ targets: wasm32-unknown-unknown+ # This job saves the family; the shards only restore it.+ - uses: ./.github/actions/rust-cache+ with:+ workspaces: libs/cua+ shared-key: libs-cua-test+ - uses: ./.github/actions/nextest+ - name: Build the test binaries once (cargo-nextest archive)+ run: >-+ cargo nextest archive --locked --archive-file "$RUNNER_TEMP/rust-tests.tar.zst"+ -p cua-sdk -p cua-daemon -p cua-cli -p cua-spacesd-client+ -p cua-cluster -p cua-spaces-cluster+ -p cua-teleport -p cua-teleport-bundle -p cua-chromium-storage -p cua-volume+ -p cua-spaces-ext -p cua-spaces-ffi -p cua-spaces-cli+ -p cua-auth -p cua-bindgen -p cua-connections -p cua-crash -p cua-create-failure -p cua-disk -p cua-home+ -p cua-hotspot -p cua-machine-seal -p cua-mock-llm -p cua-push -p cua-repo-cache -p cua-share+ -p cua-snapshot-sync -p cua-spaces-contract -p cua-spaces-transcript -p cua-agent-threads+ # The cargo test steps of shards 1 and 2 build other test binaries. Compile+ # them here so the shards find every unit in the sccache entry this job saves.+ - name: Compile the other test binaries (warms the cache for shards 1 and 2)+ run: |+ cargo test --locked --no-run -p cua-spaces -p cua-mesh -p cua-agent-setup -p cua-telemetry -p cua-agents+ cargo test --locked --no-run -p cua-media-protocol -p cua-media-transport -p cua-media-client -p cua-viewer -p cua-logging \+ --features cua-media-client/media-decode,cua-media-transport/quic+ cargo test --locked --no-run -p cua-media-codec+ - name: Size of the archive+ run: ls -lh "$RUNNER_TEMP/rust-tests.tar.zst" && df -h / | tail -1+ - uses: actions/upload-artifact@v4+ with:+ name: rust-linux-tests+ path: ${{ runner.temp }}/rust-tests.tar.zst+ if-no-files-found: error+ # The archive is already zstd-compressed.+ compression-level: 0+ retention-days: 1+rust-linux:if: ${{ !cancelled() && (github.event_name != 'pull_request' || !github.event.pull_request.draft) && !startsWith(github.head_ref, 'release-please--branches--') && !((github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'ci:full')) && needs.reuse.outputs.pr_subset == 'true') }}name: rust linux shard ${{ matrix.shard }}/3runs-on: ubuntu-latest- needs: reuse+ needs: [reuse, rust-linux-build]timeout-minutes: 60strategy:fail-fast: false@@ -212,7 +282,7 @@ jobs:shell: bashsteps:- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4- # Its debug test binaries outgrow the default cleanup's 43 GB.+ # The archive is prebuilt, but shards 1 and 2 still compile their cargo test steps.- uses: ./.github/actions/linux-rust-prepwith:free-more: "true"@@ -227,8 +297,14 @@ jobs:with:workspaces: libs/cuashared-key: libs-cua-test- save-if: ${{ matrix.shard != 1 && 'false' || '' }}+ # rust-linux-build saves the family.+ save-if: "false"- uses: ./.github/actions/nextest+ - name: Download the test archive+ uses: actions/download-artifact@v4+ with:+ name: rust-linux-tests+ path: ${{ runner.temp }}- name: Test Spaces discovery and post-create directory failuresif: matrix.shard == 1run: cargo test --locked -p cua-spaces --test relay --test cloud_discovery -- --test-threads=4@@ -245,6 +321,12 @@ jobs:# half-open: each scenario asserts wall-clock budgets, so they run# one at a time.run: cargo test --locked -p cua-spaces --test scale -- --test-threads=1 --nocapture+ # The archive is unpacked into libs/cua/target, the path the build job compiled in:+ # integration tests bake env!("CARGO_BIN_EXE_cua") and env!("CARGO_TARGET_TMPDIR") in at+ # compile time, so a run from a temporary directory fails with "No such file" (spawn cua,+ # tempdir_in). cargo creates target/tmp during a build; the archive does not carry it.+ - name: Prepare the archive's target directory+ run: mkdir -p target/tmp# One retry; a test that passes only on the retry is reported and gets a# `flaky` issue (.github/flaky/). The packages below are split by test# name hash, so a shard runs about a third of every crate's tests.@@ -252,15 +334,10 @@ jobs:run: >-"$GITHUB_WORKSPACE/.github/scripts/flaky/flaky.sh" run --suite "sdk-rust-ubuntu-shard-${{ matrix.shard }}" --format junit--results target/nextest/ci/junit.xml --- cargo nextest run --locked --no-fail-fast --config-file "$GITHUB_WORKSPACE/.github/flaky/nextest.toml" --profile ci- --partition hash:${{ matrix.shard }}/3- -p cua-sdk -p cua-daemon -p cua-cli -p cua-spacesd-client- -p cua-cluster -p cua-spaces-cluster- -p cua-teleport -p cua-teleport-bundle -p cua-chromium-storage -p cua-volume- -p cua-spaces-ext -p cua-spaces-ffi -p cua-spaces-cli- -p cua-auth -p cua-bindgen -p cua-connections -p cua-crash -p cua-create-failure -p cua-disk -p cua-home- -p cua-hotspot -p cua-machine-seal -p cua-mock-llm -p cua-push -p cua-repo-cache -p cua-share- -p cua-snapshot-sync -p cua-spaces-contract -p cua-spaces-transcript -p cua-agent-threads --test-threads 4+ cargo nextest run --no-fail-fast --config-file "$GITHUB_WORKSPACE/.github/flaky/nextest.toml" --profile ci+ --archive-file "$RUNNER_TEMP/rust-tests.tar.zst" --workspace-remap . --extract-to . --extract-overwrite+ --partition hash:${{ matrix.shard }}/3 --test-threads 4+ ${{ (github.event_name == 'pull_request' && !contains(github.event.pull_request.labels.*.name, 'ci:full')) && '-E "not (binary(pool_model) & (test(=pool_protocol_is_safe) | test(=mutation_no_epoch_check_breaks_single_writer) | test(=finding_stale_holder_cancels_current_attempt)))"' || '' }}env:# Teleport send acts only on fake hosts in tests; anything that# reaches the real host fails closed.@@ -321,6 +398,29 @@ jobs:echo "shards: $RESULT"test "$RESULT" = success+ # The pool_model checks no longer run on pull requests, so a night or main run+ # that fails on them must be heard: one issue, updated on every failure.+ rust-linux-nightly-report:+ if: ${{ failure() && github.event_name == 'schedule' }}+ name: open or update the nightly issue+ needs: rust-linux+ runs-on: ubuntu-latest+ timeout-minutes: 5+ permissions:+ issues: write+ steps:+ - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4+ with:+ persist-credentials: false+ sparse-checkout: .github/actions/nightly-issue+ - uses: ./.github/actions/nightly-issue+ with:+ title: "Nightly: cua SDK Rust tests are failing"+ detail: >-+ The nightly run of CI: cua SDK failed in the Linux Rust shards. Pull requests skip the+ pool_model model checks (about 7 minutes), so a protocol change that breaks them first+ shows up here or on main.+# macOS and Windows legs (full runs only). The Linux leg is sharded: see# rust-linux and the `rust (ubuntu-latest)` job below.rust: