Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

ci(rust): build Linux tests once, run in 3 shards (CUA-3560, audit cut 5) (#6595)

Francesco Bonacci2393194bd5bb
diff --git a/.github/actions/nightly-issue/action.yml b/.github/actions/nightly-issue/action.yml
new file mode 100644
index 0000000000..34e67d5626
--- /dev/null
+++ b/.github/actions/nightly-issue/action.yml
@@ -0,0 +1,43 @@
+name: "Open or update a nightly failure issue"
+description: >-
+ A job that moved off pull requests to the nightly schedule (or to a path
+ filter on main) must still be heard when it fails. Call this from a final
+ job with `if: failure()` and `permissions: issues: write`: it opens ONE issue
+ per title, or comments on the open one, with the run link. The issue carries
+ the `flaky` and `ci-nightly` labels so the weekly "Flaky tests this week"
+ digest and the people who watch CI issues see it. Needs actions/checkout of
+ this directory first (sparse-checkout: .github/actions/nightly-issue).
+
+inputs:
+ title:
+ description: "Issue title; one open issue per title."
+ required: true
+ detail:
+ description: "One or two plain sentences on what failed and where to look."
+ required: true
+
+runs:
+ using: composite
+ steps:
+ - name: Open or update the issue
+ shell: bash
+ env:
+ GH_TOKEN: ${{ github.token }}
+ TITLE: ${{ inputs.title }}
+ DETAIL: ${{ inputs.detail }}
+ RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+ SHA: ${{ github.sha }}
+ run: |
+ set -euo pipefail
+ gh label create ci-nightly --repo "$GITHUB_REPOSITORY" --color FBCA04 \
+ --description "A nightly or main-only CI job failed" 2>/dev/null || true
+ body="$DETAIL"$'\n\n'"Run: $RUN_URL"$'\n'"Commit: $SHA"
+ issue=$(gh issue list --repo "$GITHUB_REPOSITORY" --state open --label ci-nightly \
+ --search "\"$TITLE\" in:title" --json number,title \
+ --jq "map(select(.title == \"$TITLE\")) | .[0].number // empty")
+ if [[ -n "$issue" ]]; then
+ gh issue comment "$issue" --repo "$GITHUB_REPOSITORY" --body "$body"
+ else
+ gh issue create --repo "$GITHUB_REPOSITORY" --title "$TITLE" --body "$body" \
+ --label ci-nightly --label flaky
+ fi
diff --git a/.github/scripts/tests/test_ci_cua_workspace_tests.py b/.github/scripts/tests/test_ci_cua_workspace_tests.py
index 326c25c3fa..2a89f06f85 100644
--- a/.github/scripts/tests/test_ci_cua_workspace_tests.py
+++ b/.github/scripts/tests/test_ci_cua_workspace_tests.py
@@ -1,6 +1,6 @@
"""Every libs/cua crate that has tests runs them in GitHub CI, or says why not.
-The Linux Rust lane (ci-cua-sdk.yml, `rust-linux`) lists its crates by hand
+The Linux Rust lane (ci-cua-sdk.yml, `rust-linux-build`) lists its crates by hand
with `-p`. A crate missing from the list is never tested there, and a path
filter cannot see that: cua-auth's tests were skipped this way, so three
broken flows tests merged green (CUA-3363). This test makes the list and the
@@ -65,18 +65,19 @@ def tested_packages():
for step in job.get("steps") or []:
run = (step.get("run") or "").replace("\\\n", " ")
for cmd in re.split(r"\n|&&|;", run):
- if re.search(r"cargo (nextest run|test)\b", cmd):
+ if re.search(r"cargo (nextest (run|archive)|test)\b", cmd):
found |= set(re.findall(r"(?:^|\s)(?:-p|--package)[ =](\S+)", cmd))
return frozenset(found)
def nextest_packages():
jobs = yaml.safe_load(WORKFLOW.read_text())["jobs"]
- for step in jobs["rust-linux"]["steps"]:
- run = step.get("run", "")
- if "cargo nextest run" in run:
+ # The test binaries are built once, by `cargo nextest archive`, and the shards run the archive.
+ for step in jobs["rust-linux-build"]["steps"]:
+ run = step.get("run", "").replace("\\\n", " ")
+ if "cargo nextest archive" in run:
return set(re.findall(r"(?:^|\s)-p\s+(\S+)", run))
- raise AssertionError("rust-linux has no cargo nextest step")
+ raise AssertionError("rust-linux-build has no cargo nextest archive step")
TESTED = sorted(package_name(m) for m in members() if has_tests(m))
@@ -86,7 +87,7 @@ TESTED = sorted(package_name(m) for m in members() if has_tests(m))
def test_a_crate_with_tests_runs_them_in_the_linux_lane_or_says_why_not(name):
assert name in tested_packages() or name in ELSEWHERE, (
f"{name} has tests but is not named with `-p` on any `cargo test` or `cargo nextest` step in "
- f".github/workflows: its tests never run on GitHub. Add it to the rust-linux nextest list in ci-cua-sdk.yml, or to ELSEWHERE with the reason."
+ f".github/workflows: its tests never run on GitHub. Add it to the rust-linux-build nextest archive list in ci-cua-sdk.yml, or to ELSEWHERE with the reason."
)
diff --git a/.github/scripts/tests/test_ci_reuse_pr_result.py b/.github/scripts/tests/test_ci_reuse_pr_result.py
index 23d861bcf2..cffa6b1492 100644
--- a/.github/scripts/tests/test_ci_reuse_pr_result.py
+++ b/.github/scripts/tests/test_ci_reuse_pr_result.py
@@ -436,6 +436,37 @@ def test_cua_sdk_rust_linux_leg_is_sharded_behind_the_reused_check_name():
assert "ci:full" in job["if"]
+def test_cua_sdk_rust_tests_are_built_once_and_run_from_the_archive():
+ wf = load("ci-cua-sdk.yml")
+ build, shards = wf["jobs"]["rust-linux-build"], wf["jobs"]["rust-linux"]
+ assert build["needs"] == "reuse" and shards["needs"] == ["reuse", "rust-linux-build"]
+ # Same conditions: the build runs exactly when the shards do.
+ assert build["if"] == shards["if"]
+ build_text, shard_text = str(build["steps"]), str(shards["steps"])
+ assert "cargo nextest archive" in build_text and "rust-linux-tests" in build_text
+ assert "cargo nextest run" in shard_text and "--archive-file" in shard_text and "--workspace-remap" in shard_text
+ # Integration tests bake CARGO_BIN_EXE_* and CARGO_TARGET_TMPDIR in at compile time: the archive must be
+ # unpacked into the build's target directory and target/tmp must exist, or every cua-cli test fails to spawn.
+ assert "--extract-to ." in shard_text and "--extract-overwrite" in shard_text and "mkdir -p target/tmp" in shard_text
+ # The shards never compile the nextest packages again: they only restore the cache.
+ assert "cargo nextest run --locked" not in shard_text
+ assert any(s.get("with", {}).get("save-if") == "false" for s in shards["steps"] if "rust-cache" in str(s.get("uses", "")))
+ assert not any("save-if" in s.get("with", {}) for s in build["steps"] if "rust-cache" in str(s.get("uses", "")))
+
+
+def test_cua_sdk_pool_model_checks_leave_pull_requests_but_run_on_main_and_nightly():
+ text = (ROOT / ".github/workflows/ci-cua-sdk.yml").read_text()
+ wf = load("ci-cua-sdk.yml")
+ assert "schedule" in (wf.get("on") or wf[True])
+ run = next(s["run"] for s in wf["jobs"]["rust-linux"]["steps"] if "cargo nextest run" in s.get("run", ""))
+ # Excluded only on pull requests without `ci:full`; main, the nightly schedule and `ci:full` run them.
+ assert "github.event_name == 'pull_request' && !contains(github.event.pull_request.labels.*.name, 'ci:full')" in run
+ for name in ("pool_protocol_is_safe", "mutation_no_epoch_check_breaks_single_writer", "finding_stale_holder_cancels_current_attempt"):
+ assert f"test(={name})" in run
+ assert f"fn {name}()" in (ROOT / "libs/cua/crates/cua-spaces-cluster/tests/pool_model.rs").read_text()
+ assert "pool_model" in text
+
+
def test_cua_sdk_jobs_in_the_subset_skip_only_when_it_is_reused():
wf = load("ci-cua-sdk.yml")
skip = "needs.reuse.outputs.pr_subset != 'true'"
diff --git a/.github/workflows/README.md b/.github/workflows/README.md
index 4d18b41b71..ea289814f3 100644
--- a/.github/workflows/README.md
+++ b/.github/workflows/README.md
@@ -89,6 +89,16 @@ workflow.
* sccache hashes the rustc version into every object key, so a toolchain bump
simply misses until main re-seeds, and the cap ages the old objects out.
+## Rust tests: build once, run in shards
+
+`CI: cua SDK` compiles the Linux test binaries once (`rust linux build`, a
+`cargo nextest archive` uploaded as the `rust-linux-tests` artifact) and the three
+`rust linux shard N/3` jobs run `cargo nextest run --archive-file ... --partition
+hash:N/3`. The three `pool_model` model checks (about 254, 112 and 62 s) are left out
+of pull request runs and run on every push to main, in the nightly schedule and on a
+pull request with the `ci:full` label. Linux links with rust-lld (rustc's default
+since 1.90); CI test builds already have debug info off, so the archive stays small.
+
## Known flaky tests and quarantine
See [`FLAKY.md`](FLAKY.md). A test goes on the list with an owner and a fix
diff --git a/.github/workflows/ci-cua-sdk.yml b/.github/workflows/ci-cua-sdk.yml
index 29abb90e7a..f47ff07ce6 100644
--- a/.github/workflows/ci-cua-sdk.yml
+++ b/.github/workflows/ci-cua-sdk.yml
@@ -192,15 +192,85 @@ jobs:
exit 1
fi
- # The Linux leg of `rust`, in three shards. Each shard builds (one cache
- # family, saved by shard 1 only) and runs a third of the tests with
- # cargo-nextest partitions. Doctests (not run by nextest) and the budget
- # tests that need to run alone stay in cargo test on shards 1 and 2.
+ # The Linux leg of `rust`: build the test binaries once, run them in three
+ # shards. Compiling and linking the workspace's 176 test binaries took 11 to
+ # 19 minutes, and every shard did it again for 2 to 5 minutes of tests.
+ # `rust-linux-build` compiles them once into a cargo-nextest archive (and
+ # warms the sccache entries the shards' own `cargo test` steps use); each
+ # shard downloads the archive and runs a third of the tests from it with
+ # nextest partitions. Doctests (not run by nextest) and the budget tests that
+ # need to run alone stay in cargo test on shards 1 and 2.
+ #
+ # Linker: x86_64 Linux builds link with rust-lld, the default of every rustc
+ # since 1.90 (the toolchain is pinned to 1.97.1), so there is no mold or
+ # apt lld to install. Debug info is already off for CI tests
+ # (CARGO_PROFILE_DEV_DEBUG=0 above, lighter than `line-tables-only`, which
+ # libs/cua/Cargo.toml keeps for local builds): that keeps the archive small.
+ # macOS and Windows keep their platform linkers; swapping them is not safe
+ # without a run on each OS.
+ rust-linux-build:
+ if: ${{ !cancelled() && (github.event_name != 'pull_request' || !github.event.pull_request.draft) && !startsWith(github.head_ref, 'release-please--branches--') && !((github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'ci:full')) && needs.reuse.outputs.pr_subset == 'true') }}
+ name: rust linux build
+ runs-on: ubuntu-latest
+ needs: reuse
+ timeout-minutes: 60
+ defaults:
+ run:
+ working-directory: libs/cua
+ shell: bash
+ steps:
+ - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
+ # Its debug test binaries outgrow the default cleanup's 43 GB.
+ - uses: ./.github/actions/linux-rust-prep
+ with:
+ free-more: "true"
+ - name: Install protoc
+ uses: ./.github/actions/setup-protoc
+ - name: Install pinned Rust toolchain
+ uses: dtolnay/[email protected]
+ with:
+ components: clippy, rustfmt
+ targets: wasm32-unknown-unknown
+ # This job saves the family; the shards only restore it.
+ - uses: ./.github/actions/rust-cache
+ with:
+ workspaces: libs/cua
+ shared-key: libs-cua-test
+ - uses: ./.github/actions/nextest
+ - name: Build the test binaries once (cargo-nextest archive)
+ run: >-
+ cargo nextest archive --locked --archive-file "$RUNNER_TEMP/rust-tests.tar.zst"
+ -p cua-sdk -p cua-daemon -p cua-cli -p cua-spacesd-client
+ -p cua-cluster -p cua-spaces-cluster
+ -p cua-teleport -p cua-teleport-bundle -p cua-chromium-storage -p cua-volume
+ -p cua-spaces-ext -p cua-spaces-ffi -p cua-spaces-cli
+ -p cua-auth -p cua-bindgen -p cua-connections -p cua-crash -p cua-create-failure -p cua-disk -p cua-home
+ -p cua-hotspot -p cua-machine-seal -p cua-mock-llm -p cua-push -p cua-repo-cache -p cua-share
+ -p cua-snapshot-sync -p cua-spaces-contract -p cua-spaces-transcript -p cua-agent-threads
+ # The cargo test steps of shards 1 and 2 build other test binaries. Compile
+ # them here so the shards find every unit in the sccache entry this job saves.
+ - name: Compile the other test binaries (warms the cache for shards 1 and 2)
+ run: |
+ cargo test --locked --no-run -p cua-spaces -p cua-mesh -p cua-agent-setup -p cua-telemetry -p cua-agents
+ cargo test --locked --no-run -p cua-media-protocol -p cua-media-transport -p cua-media-client -p cua-viewer -p cua-logging \
+ --features cua-media-client/media-decode,cua-media-transport/quic
+ cargo test --locked --no-run -p cua-media-codec
+ - name: Size of the archive
+ run: ls -lh "$RUNNER_TEMP/rust-tests.tar.zst" && df -h / | tail -1
+ - uses: actions/upload-artifact@v4
+ with:
+ name: rust-linux-tests
+ path: ${{ runner.temp }}/rust-tests.tar.zst
+ if-no-files-found: error
+ # The archive is already zstd-compressed.
+ compression-level: 0
+ retention-days: 1
+
rust-linux:
if: ${{ !cancelled() && (github.event_name != 'pull_request' || !github.event.pull_request.draft) && !startsWith(github.head_ref, 'release-please--branches--') && !((github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'ci:full')) && needs.reuse.outputs.pr_subset == 'true') }}
name: rust linux shard ${{ matrix.shard }}/3
runs-on: ubuntu-latest
- needs: reuse
+ needs: [reuse, rust-linux-build]
timeout-minutes: 60
strategy:
fail-fast: false
@@ -212,7 +282,7 @@ jobs:
shell: bash
steps:
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
- # Its debug test binaries outgrow the default cleanup's 43 GB.
+ # The archive is prebuilt, but shards 1 and 2 still compile their cargo test steps.
- uses: ./.github/actions/linux-rust-prep
with:
free-more: "true"
@@ -227,8 +297,14 @@ jobs:
with:
workspaces: libs/cua
shared-key: libs-cua-test
- save-if: ${{ matrix.shard != 1 && 'false' || '' }}
+ # rust-linux-build saves the family.
+ save-if: "false"
- uses: ./.github/actions/nextest
+ - name: Download the test archive
+ uses: actions/download-artifact@v4
+ with:
+ name: rust-linux-tests
+ path: ${{ runner.temp }}
- name: Test Spaces discovery and post-create directory failures
if: matrix.shard == 1
run: cargo test --locked -p cua-spaces --test relay --test cloud_discovery -- --test-threads=4
@@ -245,6 +321,12 @@ jobs:
# half-open: each scenario asserts wall-clock budgets, so they run
# one at a time.
run: cargo test --locked -p cua-spaces --test scale -- --test-threads=1 --nocapture
+ # The archive is unpacked into libs/cua/target, the path the build job compiled in:
+ # integration tests bake env!("CARGO_BIN_EXE_cua") and env!("CARGO_TARGET_TMPDIR") in at
+ # compile time, so a run from a temporary directory fails with "No such file" (spawn cua,
+ # tempdir_in). cargo creates target/tmp during a build; the archive does not carry it.
+ - name: Prepare the archive's target directory
+ run: mkdir -p target/tmp
# One retry; a test that passes only on the retry is reported and gets a
# `flaky` issue (.github/flaky/). The packages below are split by test
# name hash, so a shard runs about a third of every crate's tests.
@@ -252,15 +334,10 @@ jobs:
run: >-
"$GITHUB_WORKSPACE/.github/scripts/flaky/flaky.sh" run --suite "sdk-rust-ubuntu-shard-${{ matrix.shard }}" --format junit
--results target/nextest/ci/junit.xml --
- cargo nextest run --locked --no-fail-fast --config-file "$GITHUB_WORKSPACE/.github/flaky/nextest.toml" --profile ci
- --partition hash:${{ matrix.shard }}/3
- -p cua-sdk -p cua-daemon -p cua-cli -p cua-spacesd-client
- -p cua-cluster -p cua-spaces-cluster
- -p cua-teleport -p cua-teleport-bundle -p cua-chromium-storage -p cua-volume
- -p cua-spaces-ext -p cua-spaces-ffi -p cua-spaces-cli
- -p cua-auth -p cua-bindgen -p cua-connections -p cua-crash -p cua-create-failure -p cua-disk -p cua-home
- -p cua-hotspot -p cua-machine-seal -p cua-mock-llm -p cua-push -p cua-repo-cache -p cua-share
- -p cua-snapshot-sync -p cua-spaces-contract -p cua-spaces-transcript -p cua-agent-threads --test-threads 4
+ cargo nextest run --no-fail-fast --config-file "$GITHUB_WORKSPACE/.github/flaky/nextest.toml" --profile ci
+ --archive-file "$RUNNER_TEMP/rust-tests.tar.zst" --workspace-remap . --extract-to . --extract-overwrite
+ --partition hash:${{ matrix.shard }}/3 --test-threads 4
+ ${{ (github.event_name == 'pull_request' && !contains(github.event.pull_request.labels.*.name, 'ci:full')) && '-E "not (binary(pool_model) & (test(=pool_protocol_is_safe) | test(=mutation_no_epoch_check_breaks_single_writer) | test(=finding_stale_holder_cancels_current_attempt)))"' || '' }}
env:
# Teleport send acts only on fake hosts in tests; anything that
# reaches the real host fails closed.
@@ -321,6 +398,29 @@ jobs:
echo "shards: $RESULT"
test "$RESULT" = success
+ # The pool_model checks no longer run on pull requests, so a night or main run
+ # that fails on them must be heard: one issue, updated on every failure.
+ rust-linux-nightly-report:
+ if: ${{ failure() && github.event_name == 'schedule' }}
+ name: open or update the nightly issue
+ needs: rust-linux
+ runs-on: ubuntu-latest
+ timeout-minutes: 5
+ permissions:
+ issues: write
+ steps:
+ - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
+ with:
+ persist-credentials: false
+ sparse-checkout: .github/actions/nightly-issue
+ - uses: ./.github/actions/nightly-issue
+ with:
+ title: "Nightly: cua SDK Rust tests are failing"
+ detail: >-
+ The nightly run of CI: cua SDK failed in the Linux Rust shards. Pull requests skip the
+ pool_model model checks (about 7 minutes), so a protocol change that breaks them first
+ shows up here or on main.
+
# macOS and Windows legs (full runs only). The Linux leg is sharded: see
# rust-linux and the `rust (ubuntu-latest)` job below.
rust: