Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

ci: E2E: cua SDK builds in parallel and tests on 4 time-balanced shards (about 69 to 25 min) (#5826)

Francesco Bonaccicc2aab774c60
diff --git a/.github/scripts/tests/test_e2e_cua_sdk_scope.py b/.github/scripts/tests/test_e2e_cua_sdk_scope.py
index 90ec5288da..93d52faf5f 100644
--- a/.github/scripts/tests/test_e2e_cua_sdk_scope.py
+++ b/.github/scripts/tests/test_e2e_cua_sdk_scope.py
@@ -353,10 +353,17 @@ def test_every_path_matches_itself_and_nothing_wider():
def test_workflow_wiring():
jobs = WORKFLOW["jobs"]
- changes, local, live = jobs["changes"], jobs["local"], jobs["live"]
- assert local["needs"] == "changes"
- # Only an explicit `false` skips the suite.
- assert "needs.changes.outputs.run != 'false'" in local["if"]
+ changes, live = jobs["changes"], jobs["live"]
+ # The suite is five build jobs, the test shards that need all of them, and
+ # the merge job that needs the shards: skipping the builds skips the rest.
+ builds = ["build-sdk", "build-cli", "build-fixtures", "build-rust-tests", "build-images"]
+ for name in builds:
+ assert jobs[name]["needs"] == "changes", name
+ # Only an explicit `false` skips the suite.
+ assert "needs.changes.outputs.run != 'false'" in jobs[name]["if"], name
+ assert sorted(jobs["shard"]["needs"]) == sorted(builds)
+ assert jobs["matrix"]["needs"] == "shard"
+ assert "needs" not in jobs["shard"].get("if", "")
assert changes["outputs"] == {"run": "${{ steps.scope.outputs.run }}"}
checkout = changes["steps"][0]
assert checkout["with"]["fetch-depth"] == 2
diff --git a/.github/workflows/e2e-cua-sdk.yml b/.github/workflows/e2e-cua-sdk.yml
index ba2d583c4e..96815843f4 100644
--- a/.github/workflows/e2e-cua-sdk.yml
+++ b/.github/workflows/e2e-cua-sdk.yml
@@ -4,9 +4,11 @@ name: "E2E: cua SDK"
# tests/e2e/cua-sdk (see its scenarios.json for the guide x lane registry).
#
# pull_request hermetic (fixtures, daemon) + container (docker + gVisor on
-# the locally built reference images) + docs blocks. A docs
-# edit that leaves every fenced code block as it was skips the
-# suite (the `changes` job, .github/scripts/e2e_cua_sdk_scope.py).
+# the locally built reference images) + docs blocks: five
+# build jobs in parallel, then 4 test shards (see `jobs:`). A
+# docs edit that leaves every fenced code block as it was
+# skips the suite (the `changes` job,
+# .github/scripts/e2e_cua_sdk_scope.py).
# nightly cua-sandbox and local QEMU (KVM) on the reference disk.
# Cua Cloud has ended, so there is no live Fleet lane.
# dispatch any lane set.
@@ -65,6 +67,15 @@ on:
langs:
description: "Languages (py,ts,rust)"
default: "py,ts,rust"
+ # For measuring concurrency: run the same commit at 2 or 3 nearby values,
+ # at least 3 times each, and compare the shards' `measured` seconds in
+ # the run summary.
+ rust_threads:
+ description: "Rust test threads per binary (the default, 2, is what the suite has always used)"
+ default: "2"
+ ts_concurrency:
+ description: "node --test-concurrency (the default, 1, runs files one at a time)"
+ default: "1"
permissions:
contents: read
@@ -83,6 +94,8 @@ env:
CARGO_TERM_COLOR: always
CARGO_BUILD_JOBS: "4"
CUA_E2E_RUN: gh${{ github.run_id }}-${{ github.run_attempt }}
+ # Test shards of a pull request run (the shard job's matrix lists 1..N).
+ E2E_SHARDS: "4"
jobs:
# Pull requests: the paths filter above starts this workflow for any docs
@@ -112,21 +125,112 @@ jobs:
HEAD_REF: ${{ github.head_ref }}
run: python3 .github/scripts/e2e_cua_sdk_scope.py | tee -a "$GITHUB_STEP_SUMMARY"
- local:
- name: hermetic + container (${{ github.event_name == 'workflow_dispatch' && inputs.lanes || 'hermetic,container' }})
+ # Pull requests: build once, test on N shards.
+ #
+ # build-sdk, build-cli, build-fixtures, build-rust-tests and build-images
+ # run side by side on their own runners (the release SDK; the debug CLI
+ # with the TypeScript and wasm builds; the fixtures and the static
+ # cua-spacesd; the Rust test binaries; the reference images) and upload
+ # what they built (ci-artifacts.sh).
+ # shard 1..E2E_SHARDS download all of it and each runs its slice of the
+ # suite. The slices are cut from measured test times (sharding.py,
+ # timings.json) so they finish within a few percent of each other.
+ # matrix merges the shards' results and applies the whole-suite checks
+ # (every cell present, docs coverage) that one run used to apply.
+ #
+ # Before this, one job built everything in series (about 46 minutes of
+ # builds) and then ran the three language halves in series (about 15).
+ build-sdk:
+ name: build / release SDK
needs: changes
if: ${{ !startsWith(github.head_ref, 'release-please--branches--') && (github.event_name != 'schedule') && needs.changes.outputs.run != 'false' }}
runs-on: ubuntu-24.04
- # Run 37562962482 exhausted 180 minutes during cold browser compilation,
- # before image preparation or the suite. Allow the hosted runner's full
- # six-hour budget for the unchanged build and strict test matrix.
- timeout-minutes: 360
+ timeout-minutes: 240
+ env:
+ CARGO_PROFILE_DEV_DEBUG: line-tables-only
+ CARGO_INCREMENTAL: "0"
+ steps:
+ - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
+ - uses: ./.github/actions/linux-rust-prep
+ - name: Install protoc
+ uses: ./.github/actions/setup-protoc
+ - name: Install pinned Rust toolchain
+ uses: dtolnay/[email protected]
+ - uses: ./.github/actions/rust-cache
+ with:
+ workspaces: |
+ libs/cua
+ tests/e2e/cua-sdk/rust
+ libs/cua-spacesd
+ key: cua-e2e
+ - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4
+ with:
+ node-version: "22"
+ - name: Build the release SDK and stage it for Python and Node
+ run: tests/e2e/cua-sdk/ci-setup.sh --only sdk,stage
+ - name: Pack
+ run: tests/e2e/cua-sdk/ci-artifacts.sh pack sdk "$RUNNER_TEMP/build"
+ - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
+ with:
+ name: cua-e2e-build-sdk
+ path: ${{ runner.temp }}/build/cua-e2e-build-sdk.tar.zst
+ compression-level: 0
+ retention-days: 2
+ overwrite: true
+ if-no-files-found: error
+
+ build-cli:
+ name: build / debug CLI, TypeScript, wasm
+ needs: changes
+ if: ${{ !startsWith(github.head_ref, 'release-please--branches--') && (github.event_name != 'schedule') && needs.changes.outputs.run != 'false' }}
+ runs-on: ubuntu-24.04
+ timeout-minutes: 240
+ env:
+ CARGO_PROFILE_DEV_DEBUG: line-tables-only
+ CARGO_INCREMENTAL: "0"
+ steps:
+ - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
+ - name: Unit-test the shard planner and the runner helpers
+ run: python3 -m unittest tests/e2e/cua-sdk/test_sharding.py tests/e2e/cua-sdk/test_run_janitor.py
+ - uses: ./.github/actions/linux-rust-prep
+ - name: Install protoc
+ uses: ./.github/actions/setup-protoc
+ - name: Install pinned Rust toolchain
+ uses: dtolnay/[email protected]
+ with:
+ targets: wasm32-unknown-unknown
+ - uses: ./.github/actions/rust-cache
+ with:
+ workspaces: |
+ libs/cua
+ tests/e2e/cua-sdk/rust
+ libs/cua-spacesd
+ key: cua-e2e
+ - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4
+ with:
+ node-version: "22"
+ # The wasm build generates its bindings from the debug cua-sdk library,
+ # so it follows the debug build on this runner.
+ - name: Build the debug CLI, the TypeScript package and the wasm browser build
+ run: tests/e2e/cua-sdk/ci-setup.sh --only cli,web
+ - name: Pack
+ run: tests/e2e/cua-sdk/ci-artifacts.sh pack cli "$RUNNER_TEMP/build"
+ - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
+ with:
+ name: cua-e2e-build-cli
+ path: ${{ runner.temp }}/build/cua-e2e-build-cli.tar.zst
+ compression-level: 0
+ retention-days: 2
+ overwrite: true
+ if-no-files-found: error
+
+ build-fixtures:
+ name: build / fixtures, static cua-spacesd
+ needs: changes
+ if: ${{ !startsWith(github.head_ref, 'release-please--branches--') && (github.event_name != 'schedule') && needs.changes.outputs.run != 'false' }}
+ runs-on: ubuntu-24.04
+ timeout-minutes: 240
env:
- # Optional: an authenticated Docker Hub keeps anonymous manifest-read
- # rate limits from failing the docs blocks' runtime inspection.
- DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
- # The debug builds, images and the suite's own image builds share the
- # runner's disk: line tables only and no incremental cache.
CARGO_PROFILE_DEV_DEBUG: line-tables-only
CARGO_INCREMENTAL: "0"
steps:
@@ -137,7 +241,7 @@ jobs:
- name: Install pinned Rust toolchain
uses: dtolnay/[email protected]
with:
- targets: wasm32-unknown-unknown,x86_64-unknown-linux-musl
+ targets: x86_64-unknown-linux-musl
- uses: ./.github/actions/rust-cache
with:
workspaces: |
@@ -147,6 +251,7 @@ jobs:
key: cua-e2e
# An image without cua-spacesd gets the static musl build injected, so
# the suite tests this tree's daemon (same build as ci-cua-oci-spacesd).
+ # The shards point CUA_SPACESD_STATIC at this file in their checkout.
- name: Build the static cua-spacesd
working-directory: libs/cua-spacesd
run: |
@@ -154,40 +259,193 @@ jobs:
sudo apt-get update -qq && sudo apt-get install -y -qq musl-tools
cargo build --locked --profile static --target x86_64-unknown-linux-musl -p cua-spacesd --bin cua-spacesd
bash packaging/release/check-static-binary.sh target/x86_64-unknown-linux-musl/static/cua-spacesd
- mkdir -p "$RUNNER_TEMP/static"
- cp target/x86_64-unknown-linux-musl/static/cua-spacesd "$RUNNER_TEMP/static/cua-spacesd-linux-x86_64"
- chmod +x "$RUNNER_TEMP/static/cua-spacesd-linux-x86_64"
- echo "CUA_SPACESD_STATIC=$RUNNER_TEMP/static/cua-spacesd-linux-x86_64" >>"$GITHUB_ENV"
+ chmod +x target/x86_64-unknown-linux-musl/static/cua-spacesd
+ - name: Build the test fixtures
+ run: tests/e2e/cua-sdk/ci-setup.sh --only fixtures
+ - name: Pack
+ run: tests/e2e/cua-sdk/ci-artifacts.sh pack fixtures "$RUNNER_TEMP/build"
+ - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
+ with:
+ name: cua-e2e-build-fixtures
+ path: ${{ runner.temp }}/build/cua-e2e-build-fixtures.tar.zst
+ compression-level: 0
+ retention-days: 2
+ overwrite: true
+ if-no-files-found: error
+
+ build-rust-tests:
+ name: build / Rust test binaries
+ needs: changes
+ if: ${{ !startsWith(github.head_ref, 'release-please--branches--') && (github.event_name != 'schedule') && needs.changes.outputs.run != 'false' }}
+ runs-on: ubuntu-24.04
+ timeout-minutes: 240
+ env:
+ CARGO_PROFILE_DEV_DEBUG: line-tables-only
+ CARGO_INCREMENTAL: "0"
+ steps:
+ - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
+ - uses: ./.github/actions/linux-rust-prep
+ - name: Install protoc
+ uses: ./.github/actions/setup-protoc
+ - name: Install pinned Rust toolchain
+ uses: dtolnay/[email protected]
+ - uses: ./.github/actions/rust-cache
+ with:
+ workspaces: |
+ libs/cua
+ tests/e2e/cua-sdk/rust
+ libs/cua-spacesd
+ key: cua-e2e
+ # The Rust half's own build (its own target directory), done once here
+ # instead of inside `cargo test` on every shard.
+ - name: Build the Rust half's test binaries
+ run: tests/e2e/cua-sdk/ci-setup.sh --only rust-tests
+ - name: Pack
+ run: tests/e2e/cua-sdk/ci-artifacts.sh pack rust-tests "$RUNNER_TEMP/build"
+ - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
+ with:
+ name: cua-e2e-build-rust-tests
+ path: ${{ runner.temp }}/build/cua-e2e-build-rust-tests.tar.zst
+ compression-level: 0
+ retention-days: 2
+ overwrite: true
+ if-no-files-found: error
+
+ build-images:
+ name: build / reference images
+ needs: changes
+ if: ${{ !startsWith(github.head_ref, 'release-please--branches--') && (github.event_name != 'schedule') && needs.changes.outputs.run != 'false' }}
+ runs-on: ubuntu-24.04
+ timeout-minutes: 120
+ env:
+ # Optional: an authenticated Docker Hub keeps anonymous pull rate limits
+ # from failing the base image pulls.
+ DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
+ steps:
+ - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
+ - name: Log in to Docker Hub
+ if: env.DOCKERHUB_USERNAME != ''
+ uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3
+ with:
+ username: ${{ secrets.DOCKERHUB_USERNAME }}
+ password: ${{ secrets.DOCKERHUB_TOKEN }}
+ # cua-spacesd builds in a rust container and the three images on top,
+ # all in the runner's docker.
+ - name: Build the reference images
+ run: tests/e2e/cua-sdk/ci-setup.sh --images --only images
+ - name: Pack
+ run: tests/e2e/cua-sdk/ci-artifacts.sh pack images "$RUNNER_TEMP/build"
+ - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
+ with:
+ name: cua-e2e-build-images
+ path: ${{ runner.temp }}/build/cua-e2e-build-images.tar.zst
+ compression-level: 0
+ retention-days: 2
+ overwrite: true
+ if-no-files-found: error
+
+ shard:
+ name: tests (shard ${{ matrix.shard }})
+ needs: [build-sdk, build-cli, build-fixtures, build-rust-tests, build-images]
+ runs-on: ubuntu-24.04
+ timeout-minutes: 90
+ strategy:
+ fail-fast: false
+ matrix:
+ # The count is E2E_SHARDS: a test pins the two together.
+ shard: [1, 2, 3, 4]
+ env:
+ DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
+ steps:
+ - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
+ # The shared libraries the built binaries link against (the build jobs
+ # installed them with the headers); the disk is not tight here.
+ - uses: ./.github/actions/linux-rust-prep
+ with:
+ free-disk: "false"
- uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4
with:
node-version: "22"
- uses: astral-sh/setup-uv@e92bafb6253dcd438e0484186d7669ea7a8ca1cc # v6
+ - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
+ with:
+ pattern: cua-e2e-build-*
+ path: ${{ runner.temp }}/build
+ merge-multiple: true
- name: Log in to Docker Hub (docs blocks read image manifests)
if: env.DOCKERHUB_USERNAME != ''
uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
- - name: Build the SDK, fixtures, bindings, images and runtimes
- run: tests/e2e/cua-sdk/ci-setup.sh --images
- # The suite builds container images and cua keeps 5 GiB free: give the
- # build cache back (the built images stay).
- - name: Reclaim build space
- run: |
- df -h / | tail -1
- docker builder prune --all --force >/dev/null
- docker image prune --force >/dev/null
- df -h / | tail -1
- - name: Run the suite
+ - name: Unpack the builds, install gVisor, load the images
+ run: tests/e2e/cua-sdk/ci-artifacts.sh prepare "$RUNNER_TEMP/build"
+ - name: Run this shard of the suite
env:
LANES: ${{ github.event_name == 'workflow_dispatch' && inputs.lanes || 'hermetic,container' }}
LANGS: ${{ github.event_name == 'workflow_dispatch' && inputs.langs || 'py,ts,rust' }}
CUA_E2E_PLAYWRIGHT_DIR: ${{ runner.temp }}/pw
- # cua keeps 5 GiB free by default. On this disposable runner, after
- # the setup and the Rust half's own debug build, that refused the
- # suite's 4 GiB image builds with 7.5 GiB still free.
+ CUA_E2E_RUST_THREADS: ${{ github.event_name == 'workflow_dispatch' && inputs.rust_threads || '2' }}
+ CUA_E2E_TS_CONCURRENCY: ${{ github.event_name == 'workflow_dispatch' && inputs.ts_concurrency || '1' }}
+ # The static cua-spacesd the build-fixtures job made, in this checkout.
+ CUA_SPACESD_STATIC: ${{ github.workspace }}/libs/cua-spacesd/target/x86_64-unknown-linux-musl/static/cua-spacesd
+ # cua keeps 5 GiB free by default. On this disposable runner that
+ # refused the suite's 4 GiB image builds with 7.5 GiB still free.
CUA_DISK_MIN_FREE: 2GiB
- run: python3 tests/e2e/cua-sdk/run.py --lanes "$LANES" --langs "$LANGS" --strict --out "$RUNNER_TEMP/cua-e2e"
+ run: >
+ python3 tests/e2e/cua-sdk/run.py --lanes "$LANES" --langs "$LANGS" --strict
+ --shard "${{ matrix.shard }}/$E2E_SHARDS"
+ --rust-prebuilt tests/e2e/cua-sdk/rust/prebuilt
+ --out "$RUNNER_TEMP/cua-e2e"
+ - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
+ if: always()
+ with:
+ name: cua-e2e-shard-${{ matrix.shard }}
+ path: ${{ runner.temp }}/cua-e2e
+ retention-days: 7
+ overwrite: true
+
+ # The check that used to be the one big job: every shard's results merged,
+ # then the whole-suite checks on the merge (every expected cell present, no
+ # unexpected skip, docs code-block coverage) and the matrix in the summary.
+ matrix:
+ name: matrix (${{ github.event_name == 'workflow_dispatch' && inputs.lanes || 'hermetic,container' }})
+ needs: shard
+ if: ${{ !cancelled() && needs.shard.result != 'skipped' }}
+ runs-on: ubuntu-24.04
+ timeout-minutes: 15
+ steps:
+ - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
+ - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
+ with:
+ pattern: cua-e2e-shard-*
+ path: ${{ runner.temp }}/shards
+ - name: Merge the shards' results
+ run: |
+ set -euo pipefail
+ mkdir -p "$RUNNER_TEMP/cua-e2e"
+ for dir in "$RUNNER_TEMP"/shards/cua-e2e-shard-*; do
+ n="${dir##*-}"
+ for f in "$dir"/*.jsonl; do
+ [ -e "$f" ] && cat "$f" >>"$RUNNER_TEMP/cua-e2e/$(basename "$f")"
+ done
+ cp "$dir/shard.json" "$RUNNER_TEMP/cua-e2e/shard-$n.json" || true
+ done
+ ls -l "$RUNNER_TEMP/cua-e2e"
+ - name: Check the merged matrix
+ env:
+ LANES: ${{ github.event_name == 'workflow_dispatch' && inputs.lanes || 'hermetic,container' }}
+ LANGS: ${{ github.event_name == 'workflow_dispatch' && inputs.langs || 'py,ts,rust' }}
+ run: python3 tests/e2e/cua-sdk/run.py --matrix-only --lanes "$LANES" --langs "$LANGS" --strict --out "$RUNNER_TEMP/cua-e2e"
+ - name: Shards covered the whole plan
+ if: ${{ !cancelled() }}
+ run: |
+ set -euo pipefail
+ {
+ printf '## Shards (predicted vs measured seconds)\n\n```\n'
+ python3 tests/e2e/cua-sdk/sharding.py check "$RUNNER_TEMP"/shards/cua-e2e-shard-* --shards "$E2E_SHARDS" | tee /dev/stderr
+ printf '```\n'
+ } >>"$GITHUB_STEP_SUMMARY"
- name: Matrix summary
if: always()
run: |
@@ -201,6 +459,7 @@ jobs:
with:
name: cua-e2e-local
path: ${{ runner.temp }}/cua-e2e
+ overwrite: true
live:
name: nightly (cua-sandbox, qemu)
diff --git a/tests/e2e/cua-sdk/.gitignore b/tests/e2e/cua-sdk/.gitignore
index 39d637e2c5..33fad17e40 100644
--- a/tests/e2e/cua-sdk/.gitignore
+++ b/tests/e2e/cua-sdk/.gitignore
@@ -1,3 +1,4 @@
go/.target/
rust/target/
.playwright/
+rust/prebuilt/
diff --git a/tests/e2e/cua-sdk/ci-artifacts.sh b/tests/e2e/cua-sdk/ci-artifacts.sh
new file mode 100755
index 0000000000..b3680d8d87
--- /dev/null
+++ b/tests/e2e/cua-sdk/ci-artifacts.sh
@@ -0,0 +1,88 @@
+#!/usr/bin/env bash
+# Hands the build jobs' outputs to the test shards of
+# .github/workflows/e2e-cua-sdk.yml. Every archive is a tar.zst of paths
+# relative to the repository root, so the shard sees the tree the single-job
+# workflow used to build (same checkout path, same file modes).
+#
+# ci-artifacts.sh pack sdk|cli|fixtures|rust-tests|images DIR
+# DIR/cua-e2e-build-<name>.tar.zst
+# sdk the release library staged for Python and Node (+ node_modules)
+# cli the debug cua CLI and the TypeScript + wasm browser builds
+# fixtures cua-test-fixtures and the static cua-spacesd the suite
+# injects into plain images
+# rust-tests the prebuilt Rust test binaries
+# images the reference images, as one `docker save`
+# ci-artifacts.sh prepare DIR
+# a shard's setup: gVisor, the archives of DIR unpacked, the images
+# loaded, and the canonical desktop image pulled while they load
+set -euo pipefail
+HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+REPO="$(cd "$HERE/../../.." && pwd)"
+STATIC_SPACESD=libs/cua-spacesd/target/x86_64-unknown-linux-musl/static/cua-spacesd
+
+pack_paths() {
+ local name="$1" out="$2"; shift 2
+ local path
+ for path in "$@"; do
+ [ -e "$REPO/$path" ] || { echo "pack $name: $path was not built" >&2; exit 1; }
+ done
+ tar --zstd -cf "$out/cua-e2e-build-$name.tar.zst" -C "$REPO" "$@"
+ ls -lh "$out/cua-e2e-build-$name.tar.zst"
+}
+
+pack() {
+ local name="$1" out="$2"
+ mkdir -p "$out"
+ case "$name" in
+ sdk)
+ pack_paths sdk "$out" libs/cua/python/src/cua/libcua_sdk.so libs/cua/typescript/node_modules ;;
+ cli)
+ pack_paths cli "$out" libs/cua/target/debug/cua libs/cua/typescript/dist libs/cua/typescript/browser ;;
+ fixtures)
+ pack_paths fixtures "$out" libs/cua/target/debug/cua-test-fixtures "$STATIC_SPACESD" ;;
+ rust-tests)
+ pack_paths rust-tests "$out" tests/e2e/cua-sdk/rust/prebuilt ;;
+ images)
+ local refs
+ refs="$(docker image ls --format '{{.Repository}}:{{.Tag}}' | grep '^cua-e2e-local/' || true)"
+ [ "$(wc -w <<<"$refs")" -ge 3 ] || { echo "pack images: expected 3 images, found: $refs" >&2; exit 1; }
+ # shellcheck disable=SC2086
+ docker save $refs | zstd -T0 -3 -q -o "$out/cua-e2e-build-images.tar.zst"
+ echo "$refs"
+ ls -lh "$out/cua-e2e-build-images.tar.zst" ;;
+ *) echo "unknown archive $name" >&2; exit 2 ;;
+ esac
+}
+
+prepare() {
+ local dir="$1"
+ # gVisor registers a docker runtime and restarts the daemon, so it goes
+ # first; the archives unpack while it installs.
+ "$HERE/ci-setup.sh" --only system &
+ local system=$!
+ local name
+ for name in sdk cli fixtures rust-tests; do
+ tar --zstd -xf "$dir/cua-e2e-build-$name.tar.zst" -C "$REPO"
+ done
+ wait "$system"
+ # The images load while the canonical desktop image (the one the docs
+ # blocks start through Image.linux()) is pulled with the same resolver the
+ # SDK uses, so the first docs block does not wait for the pull.
+ local cua="$REPO/libs/cua/target/debug/cua"
+ (zstd -dc "$dir/cua-e2e-build-images.tar.zst" | docker load) &
+ local load=$!
+ (timeout 900 "$cua" --embedded image pull container:ghcr.io/trycua/linux:24.04 ||
+ echo "::warning::could not prefetch the canonical linux image; the first docs block pulls it") &
+ local pull=$!
+ local rc=0
+ wait "$load" || rc=1
+ wait "$pull" || true
+ docker image ls --format '{{.Repository}}:{{.Tag}} {{.Size}}'
+ return "$rc"
+}
+
+case "${1:-}" in
+ pack) pack "${2:?name}" "${3:?dir}" ;;
+ prepare) prepare "${2:?dir}" ;;
+ *) sed -n '2,/^set -euo/p' "$0" | sed '$d' | sed 's/^# \{0,1\}//' >&2; exit 2 ;;
+esac
diff --git a/tests/e2e/cua-sdk/ci-setup.sh b/tests/e2e/cua-sdk/ci-setup.sh
index 62cbbe37c1..d71ac16e70 100755
--- a/tests/e2e/cua-sdk/ci-setup.sh
+++ b/tests/e2e/cua-sdk/ci-setup.sh
@@ -9,18 +9,71 @@
# two plain images, rootfs form, host arch
# --disks the VM disks of linux and plain/ubuntu-server into
# ~/.cache/cua-images-e2e, plus QEMU and firmware
+#
+# With no --only, every phase runs, in the order below (the nightly job, the
+# compat workflows and a laptop). The PR workflow runs the phases on separate
+# runners and ships their outputs to the test shards:
+#
+# --only PHASE[,PHASE...]
+# system apt packages, gVisor, docker runtimes
+# cli debug cua CLI + cua-sdk
+# fixtures cua-test-fixtures (debug)
+# sdk release cua-sdk
+# stage npm ci, stage the release library for Python and Node
+# web TypeScript build, wasm tools, wasm browser build
+# playwright Playwright's headless Chromium
+# images the reference images (needs --images)
+# disks the VM disks (needs --disks)
+# rust-tests build the Rust half's test binaries into
+# tests/e2e/cua-sdk/rust/prebuilt (+ manifest.json), for
+# `run.py --rust-prebuilt`
set -euo pipefail
HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
REPO="$(cd "$HERE/../../.." && pwd)"
-IMAGES=0; DISKS=0
-for a in "$@"; do
- case "$a" in --images) IMAGES=1 ;; --disks) DISKS=1 ;; *) echo "unknown $a" >&2; exit 2 ;; esac
+IMAGES=0; DISKS=0; ONLY=""
+while [ $# -gt 0 ]; do
+ case "$1" in
+ --images) IMAGES=1 ;;
+ --disks) DISKS=1 ;;
+ --only) ONLY="${2:?--only needs a phase list}"; shift ;;
+ *) echo "unknown $1" >&2; exit 2 ;;
+ esac
+ shift
done
arch="$(uname -m)"; case "$arch" in arm64|aarch64) harch=arm64 ;; *) harch=amd64 ;; esac
step() { echo "::group::$*"; }
end() { echo "::endgroup::"; }
+# run PHASE: true when this invocation includes the phase.
+run() {
+ [ -z "$ONLY" ] && return 0
+ case ",$ONLY," in *",$1,"*) return 0 ;; *) return 1 ;; esac
+}
+if [ -n "$ONLY" ]; then
+ for p in ${ONLY//,/ }; do
+ case "$p" in
+ system|cli|fixtures|sdk|stage|web|playwright|images|disks|rust-tests) ;;
+ *) echo "unknown phase $p" >&2; exit 2 ;;
+ esac
+ done
+ # `images` and `disks` also need their flag: --only images without --images is a mistake.
+ run images && [ "$IMAGES" != 1 ] && { echo "--only images needs --images" >&2; exit 2; }
+ run disks && [ "$DISKS" != 1 ] && { echo "--only disks needs --disks" >&2; exit 2; }
+fi
+# Default (no --only): the phases the flags ask for, as before.
+phase() {
+ if [ -z "$ONLY" ]; then
+ case "$1" in
+ images) [ "$IMAGES" = 1 ] ;;
+ disks) [ "$DISKS" = 1 ] ;;
+ rust-tests) return 1 ;;
+ *) return 0 ;;
+ esac
+ else
+ run "$1"
+ fi
+}
-if [ "$(uname -s)" = Linux ]; then
+if phase system && [ "$(uname -s)" = Linux ]; then
step "system packages, gVisor"
sudo apt-get update -qq
pkgs="genisoimage"
@@ -33,34 +86,84 @@ if [ "$(uname -s)" = Linux ]; then
end
fi
-step "cua SDK, CLI, fixtures"
-cd "$REPO/libs/cua"
-cargo build --locked -p cua-cli -p cua-sdk
-scripts/build-test-fixtures.sh
-cargo build --locked --release -p cua-sdk
-end
+if phase cli || phase fixtures || phase sdk; then
+ step "cua SDK, CLI, fixtures"
+ cd "$REPO/libs/cua"
+ if phase cli; then cargo build --locked -p cua-cli -p cua-sdk; fi
+ if phase fixtures; then scripts/build-test-fixtures.sh; fi
+ if phase sdk; then cargo build --locked --release -p cua-sdk; fi
+ end
+fi
+
+# npm ci once per checkout: `stage` and `web` each need node_modules.
+npm_ci() {
+ cd "$REPO/libs/cua/typescript"
+ [ -e node_modules/.package-lock.json ] || npm ci
+}
+
+if phase stage; then
+ step "npm ci, stage the release library"
+ npm_ci
+ node ../scripts/stage-uniffi-library.mjs --only=python,node
+ end
+fi
+
+if phase web; then
+ step "TypeScript package + browser (wasm) build"
+ npm_ci
+ npm run build
+ if ! command -v wasm-pack >/dev/null; then
+ curl -fsSL https://rustwasm.github.io/wasm-pack/installer/init.sh | sh -s -- -f
+ fi
+ command -v wasm-bindgen >/dev/null || cargo install --locked wasm-bindgen-cli --version 0.2.126
+ npm run build:browser
+ end
+fi
-step "TypeScript package + browser (wasm) build"
-cd "$REPO/libs/cua/typescript"
-npm ci
-node ../scripts/stage-uniffi-library.mjs --only=python,node
-npm run build
-if ! command -v wasm-pack >/dev/null; then
- curl -fsSL https://rustwasm.github.io/wasm-pack/installer/init.sh | sh -s -- -f
+if phase playwright; then
+ step "Playwright Chromium"
+ pw="${CUA_E2E_PLAYWRIGHT_DIR:-$REPO/tests/e2e/cua-sdk/.playwright}"
+ mkdir -p "$pw" && cd "$pw"
+ [ -f package.json ] || echo '{"type":"module"}' >package.json
+ npm i -s playwright@1
+ if [ "$(uname -s)" = Linux ]; then npx playwright install --with-deps chromium-headless-shell; else npx playwright install chromium-headless-shell; fi
+ end
fi
-command -v wasm-bindgen >/dev/null || cargo install --locked wasm-bindgen-cli --version 0.2.126
-npm run build:browser
-end
-step "Playwright Chromium"
-pw="${CUA_E2E_PLAYWRIGHT_DIR:-$REPO/tests/e2e/cua-sdk/.playwright}"
-mkdir -p "$pw" && cd "$pw"
-[ -f package.json ] || echo '{"type":"module"}' >package.json
-npm i -s playwright@1
-if [ "$(uname -s)" = Linux ]; then npx playwright install --with-deps chromium-headless-shell; else npx playwright install chromium-headless-shell; fi
-end
+if phase rust-tests; then
+ step "Rust test binaries"
+ cd "$REPO/tests/e2e/cua-sdk/rust"
+ out="$REPO/tests/e2e/cua-sdk/rust/prebuilt"
+ rm -rf "$out" && mkdir -p "$out"
+ # The same build `cargo test` does, minus running it: every test target of
+ # the crate, copied next to a manifest that run.py --rust-prebuilt reads.
+ cargo test --no-run --message-format=json >"$out/build.json"
+ python3 - "$out" <<'PY'
+import json, shutil, sys
+from pathlib import Path
+
+out = Path(sys.argv[1])
+manifest = []
+for line in (out / "build.json").read_text().splitlines():
+ try:
+ msg = json.loads(line)
+ except ValueError:
+ continue
+ exe = msg.get("executable")
+ if msg.get("reason") == "compiler-artifact" and exe and msg.get("profile", {}).get("test"):
+ name = msg["target"]["name"]
+ shutil.copy2(exe, out / name)
+ manifest.append({"name": name, "file": name})
+if not manifest:
+ raise SystemExit("cargo test --no-run produced no test binaries")
+(out / "manifest.json").write_text(json.dumps(sorted(manifest, key=lambda m: m["name"]), indent=1) + "\n")
+(out / "build.json").unlink()
+print("prebuilt:", ", ".join(m["name"] for m in manifest))
+PY
+ end
+fi
-if [ "$IMAGES" = 1 ]; then
+if phase images; then
step "reference images (rootfs, $harch)"
cd "$REPO/libs/images"
linux/build-spacesd-linux.sh "$harch"
@@ -70,7 +173,7 @@ if [ "$IMAGES" = 1 ]; then
end
fi
-if [ "$DISKS" = 1 ]; then
+if phase disks; then
step "reference disks ($harch) -> ~/.cache/cua-images-e2e"
cd "$REPO/libs/images"
[ -x "linux/dist/$harch/cua-spacesd" ] || linux/build-spacesd-linux.sh "$harch"
diff --git a/tests/e2e/cua-sdk/python/conftest.py b/tests/e2e/cua-sdk/python/conftest.py
index 6509849fe5..4341b9607b 100644
--- a/tests/e2e/cua-sdk/python/conftest.py
+++ b/tests/e2e/cua-sdk/python/conftest.py
@@ -24,6 +24,24 @@ def pytest_configure(config):
)
+def pytest_collection_modifyitems(config, items):
+ """`CUA_E2E_SELECT=<file>` (one node id per line) keeps only those tests:
+ run.py --shard gives each CI shard its slice. An id that no longer exists
+ fails the run, so a plan can never silently drop a test."""
+ chosen = os.environ.get("CUA_E2E_SELECT")
+ if not chosen:
+ return
+ keep = {line.strip() for line in Path(chosen).read_text().splitlines() if line.strip()}
+ missing = keep - {item.nodeid for item in items}
+ if missing:
+ raise pytest.UsageError(
+ f"CUA_E2E_SELECT names tests that do not exist: {sorted(missing)[:5]}"
+ )
+ dropped = [item for item in items if item.nodeid not in keep]
+ items[:] = [item for item in items if item.nodeid in keep]
+ config.hook.pytest_deselected(items=dropped)
+
+
def _mark(item):
m = item.get_closest_marker("e2e")
if m is None:
diff --git a/tests/e2e/cua-sdk/run.py b/tests/e2e/cua-sdk/run.py
index 7767da0159..fa0abb6b19 100755
--- a/tests/e2e/cua-sdk/run.py
+++ b/tests/e2e/cua-sdk/run.py
@@ -17,6 +17,11 @@ must record at least one result. Known bugs are `xfail`, never skips.
After each language, a janitor removes docker containers named
cua-e2e-<run>-* that a crashed test may have left behind.
+
+Sharded CI (`--shard I/N`, see sharding.py): a shard runs only its units and
+checks only what it ran (failures, unexpected skips); the per-shard results
+are concatenated and `--matrix-only --strict` then applies the full-suite
+checks (every expected cell present, docs coverage) to the merged set.
"""
from __future__ import annotations
@@ -35,6 +40,9 @@ from pathlib import Path
SUITE = Path(__file__).resolve().parent
REPO = SUITE.parents[2]
+sys.path.insert(0, str(SUITE))
+import sharding # noqa: E402
+
GATES = {
"container": {"CUA_E2E_CONTAINER": "1"},
"qemu": {"CUA_E2E_QEMU": "1"},
@@ -108,39 +116,123 @@ def janitor(run: str) -> None:
subprocess.run(["docker", "rm", "-f", *ids], capture_output=True)
-def run_langs(langs: list[str], env: dict, timeout: int) -> dict[str, int]:
- codes = {}
+def py_command() -> list[str]:
uv = shutil.which("uv")
+ if not uv:
+ return [sys.executable, "-m", "pytest", "-q", "-rs"]
+ return [
+ uv,
+ "run",
+ "--no-project",
+ "--python",
+ "3.12",
+ "--with",
+ "pytest",
+ # The browse-the-web scenario is an MCP client.
+ "--with",
+ "mcp",
+ "pytest",
+ "-q",
+ "-rs",
+ ]
+
+
+def collect_py(env: dict, timeout: int) -> list[str]:
+ """The node ids pytest would run, in collection order."""
+ out = subprocess.run(
+ [*py_command(), "--collect-only"],
+ env=env,
+ cwd=SUITE / "python",
+ capture_output=True,
+ text=True,
+ timeout=timeout,
+ )
+ ids = []
+ for line in out.stdout.splitlines():
+ if not line.strip():
+ break # the node ids end at the first blank line
+ if re.match(r"[\w./-]+\.py::", line):
+ ids.append(line.strip())
+ if out.returncode != 0 or not ids:
+ sys.stderr.write(out.stdout[-4000:] + out.stderr[-4000:])
+ raise SystemExit(f"cannot collect the Python tests (exit {out.returncode})")
+ return ids
+
+
+def ts_files() -> list[str]:
+ return sorted(p.name for p in (SUITE / "typescript").glob("*.test.mjs"))
+
+
+def rust_binaries(prebuilt: Path) -> dict[str, str]:
+ """Prebuilt integration-test binaries: name -> file, from the manifest
+ ci-setup.sh --only rust-tests writes next to them."""
+ manifest = prebuilt / "manifest.json"
+ if not manifest.exists():
+ raise SystemExit(f"no {manifest}: build the Rust tests with ci-setup.sh --only rust-tests")
+ return {m["name"]: m["file"] for m in json.loads(manifest.read_text())}
+
+
+def run_langs(
+ langs: list[str],
+ env: dict,
+ timeout: int,
+ *,
+ select: dict[str, list[str]] | None = None,
+ ts_concurrency: int = 1,
+ rust_threads: int = 2,
+ rust_prebuilt: Path | None = None,
+ unit_secs: dict[str, float] | None = None,
+) -> dict[str, int]:
+ """Runs each language half. `select` (a shard) limits a half to its units;
+ without it every test runs, as on a developer machine and in the nightly."""
+ codes = {}
+ unit_secs = {} if unit_secs is None else unit_secs
+ # The binaries run with the Rust half's directory as cwd: a relative
+ # --rust-prebuilt (the workflow passes one) must not be taken from there.
+ rust_prebuilt = rust_prebuilt.resolve() if rust_prebuilt is not None else None
for lang in langs:
+ if select is not None and not select.get(lang):
+ continue # nothing of this half in this shard
if lang == "py":
- cmd = (
- [
- uv,
- "run",
- "--no-project",
- "--python",
- "3.12",
- "--with",
- "pytest",
- # The browse-the-web scenario is an MCP client.
- "--with",
- "mcp",
- "pytest",
- "-q",
- "-rs",
- ]
- if uv
- else [sys.executable, "-m", "pytest", "-q", "-rs"]
- )
- codes[lang] = sh(cmd, env, SUITE / "python", timeout)
+ run_env = env
+ if select is not None:
+ chosen = Path(env["CUA_E2E_RESULTS"]) / "py-select.txt"
+ chosen.write_text("\n".join(select["py"]) + "\n")
+ run_env = dict(env, CUA_E2E_SELECT=str(chosen))
+ codes[lang] = sh(py_command(), run_env, SUITE / "python", timeout)
elif lang == "ts":
- tests = sorted(str(p) for p in (SUITE / "typescript").glob("*.test.mjs"))
- codes[lang] = sh(
- ["node", "--test", "--test-concurrency=1", *tests],
- env,
- SUITE / "typescript",
- timeout,
- )
+ names = select["ts"] if select is not None else ts_files()
+ code = 0
+ for name in names:
+ t0 = time.monotonic()
+ rc = sh(
+ [
+ "node",
+ "--test",
+ f"--test-concurrency={ts_concurrency}",
+ str(SUITE / "typescript" / name),
+ ],
+ env,
+ SUITE / "typescript",
+ timeout,
+ )
+ unit_secs[f"ts:{name}"] = round(time.monotonic() - t0, 1)
+ code = code or rc
+ codes[lang] = code
+ elif lang == "rust" and rust_prebuilt is not None:
+ binaries = rust_binaries(rust_prebuilt)
+ code = 0
+ for name in select["rust"] if select is not None else sorted(binaries):
+ t0 = time.monotonic()
+ rc = sh(
+ [str(rust_prebuilt / binaries[name]), f"--test-threads={rust_threads}"],
+ env,
+ SUITE / "rust",
+ timeout,
+ )
+ unit_secs[f"rust:{name}"] = round(time.monotonic() - t0, 1)
+ code = code or rc
+ codes[lang] = code
elif lang == "rust":
codes[lang] = sh(
[
@@ -149,7 +241,7 @@ def run_langs(langs: list[str], env: dict, timeout: int) -> dict[str, int]:
"--manifest-path",
str(SUITE / "rust" / "Cargo.toml"),
"--",
- "--test-threads=2",
+ f"--test-threads={rust_threads}",
],
env,
SUITE / "rust",
@@ -161,6 +253,26 @@ def run_langs(langs: list[str], env: dict, timeout: int) -> dict[str, int]:
return codes
+def plan_shard(
+ langs: list[str], env: dict, timeout: int, spec: str, timings: dict, rust_prebuilt: Path | None
+) -> tuple[dict[str, list[str]], int, float]:
+ """This shard's units per language, the total unit count and the shard's
+ predicted seconds. Every shard computes the same plan."""
+ index, count = sharding.parse_shard(spec)
+ units: list[sharding.Unit] = []
+ if "py" in langs:
+ units += [("py", n) for n in collect_py(env, timeout)]
+ if "ts" in langs:
+ units += [("ts", n) for n in ts_files()]
+ if "rust" in langs:
+ if rust_prebuilt is None:
+ raise SystemExit("--shard needs --rust-prebuilt for the Rust half")
+ units += [("rust", n) for n in rust_binaries(rust_prebuilt)]
+ mine = sharding.assign(units, count, timings)[index - 1]
+ chosen = {lang: [u for lg, u in mine if lg == lang] for lang in ("py", "ts", "rust")}
+ return chosen, len(set(units)), sharding.predicted(mine, timings)
+
+
def expected(sc: dict, lane: str, lang: str) -> bool:
return lang in sc["langs"] and lang in sc.get("only", {}).get(lane, sc["langs"])
@@ -240,19 +352,20 @@ def render(out: Path, lanes: list[str] | None) -> tuple[str, list[str]]:
return md, problems
+def skip_problems(rows: list[dict], lanes: list[str]) -> list[str]:
+ return [
+ f"unexpected skip in requested lane: {r['scenario']}/{r['lane']}/{r['lang']}: {r['reason']}"
+ for r in rows
+ if r["lane"] in lanes
+ and r["status"] == "skip"
+ and not any(a in r.get("reason", "") for a in ALLOWED_SKIPS)
+ ]
+
+
def strict_problems(out: Path, lanes: list[str], langs: list[str]) -> list[str]:
reg = json.loads((SUITE / "scenarios.json").read_text())
rows = load(out)
- probs = []
- for r in rows:
- if (
- r["lane"] in lanes
- and r["status"] == "skip"
- and not any(a in r.get("reason", "") for a in ALLOWED_SKIPS)
- ):
- probs.append(
- f"unexpected skip in requested lane: {r['scenario']}/{r['lane']}/{r['lang']}: {r['reason']}"
- )
+ probs = skip_problems(rows, lanes)
seen = {(r["scenario"], r["lane"], r["lang"]) for r in rows}
for sc in reg["scenarios"]:
for lane in sc["lanes"]:
@@ -295,6 +408,65 @@ def docs_coverage(out: Path, lanes: list[str], langs: list[str]) -> list[str]:
return problems
+def run_shard(
+ a: argparse.Namespace, lanes: list[str], langs: list[str], env: dict, out: Path
+) -> int:
+ """One shard of the suite. Only this shard's results exist here, so the
+ checks that need the whole matrix (missing cells, docs coverage) wait for
+ the merge; failures and unexpected skips are checked now."""
+ timings = sharding.load_timings(a.timings)
+ chosen, total, expect = plan_shard(langs, env, a.timeout, a.shard, timings, a.rust_prebuilt)
+ index, count = sharding.parse_shard(a.shard)
+ print(
+ f"\nshard {index}/{count}: "
+ + ", ".join(f"{len(chosen[lang])} {lang}" for lang in ("py", "ts", "rust"))
+ + f" of {total} units, predicted {expect:.0f}s",
+ flush=True,
+ )
+ unit_secs: dict[str, float] = {}
+ t0 = time.monotonic()
+ codes = run_langs(
+ langs,
+ env,
+ a.timeout,
+ select=chosen,
+ ts_concurrency=a.ts_concurrency,
+ rust_threads=a.rust_threads,
+ rust_prebuilt=a.rust_prebuilt,
+ unit_secs=unit_secs,
+ )
+ wall = time.monotonic() - t0
+ rows = load(out)
+ problems = [
+ f"{r['status']}: {r['scenario']}/{r['lane']}/{r['lang']}: {r['test']}"
+ for r in rows
+ if r["status"] in ("fail", "xpass")
+ ]
+ if a.strict:
+ problems += skip_problems(rows, lanes)
+ (out / "shard.json").write_text(
+ json.dumps(
+ {
+ "shard": index,
+ "of": count,
+ "total_units": total,
+ "units": chosen,
+ "predicted_secs": round(expect, 1),
+ "wall_secs": round(wall, 1),
+ "unit_secs": unit_secs,
+ "settings": {"ts_concurrency": a.ts_concurrency, "rust_threads": a.rust_threads},
+ },
+ indent=1,
+ )
+ + "\n"
+ )
+ print(f"\nshard {index}/{count} exit codes: {codes} in {wall:.0f}s (predicted {expect:.0f}s)")
+ if problems:
+ print("\n".join(["", "PROBLEMS:"] + problems))
+ return 1
+ return 1 if any(c != 0 for c in codes.values()) else 0
+
+
def main() -> int:
ap = argparse.ArgumentParser(description=__doc__.split("\n")[0])
ap.add_argument(
@@ -310,7 +482,39 @@ def main() -> int:
ap.add_argument("--timeout", type=int, default=5400, help="per-language timeout (s)")
ap.add_argument("--strict", action="store_true")
ap.add_argument("--matrix-only", action="store_true")
+ ap.add_argument(
+ "--shard",
+ metavar="I/N",
+ help="run only shard I of N (sharding.py); merge the shards' results, then run "
+ "`--matrix-only --strict` on the merge for the full-suite checks",
+ )
+ ap.add_argument(
+ "--timings", type=Path, default=sharding.TIMINGS, help="unit weights for --shard"
+ )
+ ap.add_argument(
+ "--rust-prebuilt",
+ type=Path,
+ help="directory of prebuilt Rust test binaries + manifest.json (ci-setup.sh --only "
+ "rust-tests); run instead of `cargo test`",
+ )
+ ap.add_argument(
+ "--ts-concurrency",
+ type=int,
+ default=int(os.environ.get("CUA_E2E_TS_CONCURRENCY", "1")),
+ help="node --test-concurrency (files run one at a time by default)",
+ )
+ ap.add_argument(
+ "--rust-threads",
+ type=int,
+ default=int(os.environ.get("CUA_E2E_RUST_THREADS", "2")),
+ help="--test-threads of the Rust test binaries",
+ )
a = ap.parse_args()
+ # Paths are used from other working directories (the halves run in their
+ # own), so fix them against the invocation directory now.
+ a.rust_prebuilt = a.rust_prebuilt.resolve() if a.rust_prebuilt else None
+ a.timings = a.timings.resolve()
+ a.out = a.out.resolve() if a.out else None
lanes = [x for x in a.lanes.split(",") if x]
langs = [x for x in a.langs.split(",") if x]
out = (
@@ -323,8 +527,17 @@ def main() -> int:
env = dict(os.environ, CUA_E2E_RUN=a.run, CUA_E2E_RESULTS=str(out))
for lane in lanes:
env.update(GATES.get(lane, {}))
+ if a.shard:
+ return run_shard(a, lanes, langs, env, out)
t0 = time.monotonic()
- codes = run_langs(langs, env, a.timeout)
+ codes = run_langs(
+ langs,
+ env,
+ a.timeout,
+ ts_concurrency=a.ts_concurrency,
+ rust_threads=a.rust_threads,
+ rust_prebuilt=a.rust_prebuilt,
+ )
print(f"\nexit codes: {codes} in {time.monotonic() - t0:.0f}s")
md, problems = render(out, lanes)
print(f"\n## cua SDK e2e matrix (run {a.run})\n\n{md}")
diff --git a/tests/e2e/cua-sdk/sharding.py b/tests/e2e/cua-sdk/sharding.py
new file mode 100644
index 0000000000..c7d43c5b6d
--- /dev/null
+++ b/tests/e2e/cua-sdk/sharding.py
@@ -0,0 +1,182 @@
+#!/usr/bin/env python3
+"""Time-balanced shards for the cua SDK e2e suite (stdlib only).
+
+The PR workflow runs the suite on N runners. Every shard collects the same
+list of units, weighs each one from timings.json and runs the units the plan
+gives it, so the shards are disjoint and together cover every unit without any
+coordination:
+
+ py one pytest test (node id; weighed by test name)
+ ts one tests/e2e/cua-sdk/typescript/*.test.mjs file
+ rust one prebuilt integration-test binary of tests/e2e/cua-sdk/rust
+
+A unit missing from timings.json (a new test) gets the language's default
+weight, so it still runs on exactly one shard.
+
+The plan is greedy longest-processing-time: heaviest unit first, onto the
+shard with the least work so far, ties broken by name. With ~120 units and the
+heaviest at about a third of a shard that lands within 1% of even.
+
+Refresh timings.json from the `cua-e2e-shard-*` artifacts of a green run:
+
+ for i in 1 2 3 4; do gh run download RUN -n cua-e2e-shard-$i -D art/$i; done
+ python3 tests/e2e/cua-sdk/sharding.py timings art/* >tests/e2e/cua-sdk/timings.json
+ python3 tests/e2e/cua-sdk/sharding.py check art/* --shards 4
+
+`check` prints each shard's predicted and measured seconds and the spread, and
+fails when the shards together do not cover the plan (a shard missing, a unit
+dropped or planned twice).
+"""
+
+from __future__ import annotations
+
+import json
+import statistics
+import sys
+from pathlib import Path
+
+TIMINGS = Path(__file__).with_name("timings.json")
+LANGS = ("py", "ts", "rust")
+Unit = tuple[str, str] # (lang, unit)
+
+
+def load_timings(path: Path = TIMINGS) -> dict:
+ return json.loads(Path(path).read_text()) if Path(path).exists() else {}
+
+
+def timing_key(lang: str, unit: str) -> str:
+ """py units are pytest node ids, weighed by the test's own name (what the
+ results record): the part after `file.py::`."""
+ return unit.split("::", 1)[1] if lang == "py" and "::" in unit else unit
+
+
+def weight(timings: dict, lang: str, unit: str) -> float:
+ w = timings.get(lang, {}).get(timing_key(lang, unit))
+ if w is None:
+ w = timings.get("default", {}).get(lang, 1.0)
+ return float(w)
+
+
+def parse_shard(spec: str) -> tuple[int, int]:
+ """`2/4` -> (2, 4), 1-based."""
+ try:
+ i, n = (int(x) for x in spec.split("/"))
+ except ValueError:
+ raise SystemExit(f"--shard {spec!r}: want I/N, for example 2/4") from None
+ if not 1 <= i <= n:
+ raise SystemExit(f"--shard {spec!r}: I must be 1..N")
+ return i, n
+
+
+def assign(units: list[Unit], n: int, timings: dict) -> list[list[Unit]]:
+ """Split units into n shards by weight (longest processing time first)."""
+ if n < 1:
+ raise ValueError("need at least one shard")
+ load = [0.0] * n
+ shards: list[list[Unit]] = [[] for _ in range(n)]
+ for lang, unit in sorted(set(units), key=lambda u: (-weight(timings, *u), u)):
+ i = min(range(n), key=lambda k: (load[k], k))
+ shards[i].append((lang, unit))
+ load[i] += weight(timings, lang, unit)
+ return shards
+
+
+def predicted(shard: list[Unit], timings: dict) -> float:
+ return sum(weight(timings, *u) for u in shard)
+
+
+def spread(values: list[float]) -> float:
+ """(max - min) / max, as a percentage."""
+ top = max(values, default=0.0)
+ return 0.0 if top <= 0 else 100.0 * (top - min(values)) / top
+
+
+def _jsonl(path: Path) -> list[dict]:
+ if not path.exists():
+ return []
+ return [json.loads(line) for line in path.read_text().splitlines() if line.strip()]
+
+
+def measured(dirs: list[Path]) -> dict:
+ """Per-unit seconds from shard result dirs: py from the per-test results,
+ ts and rust from the wall time each file or binary took (shard.json)."""
+ py: dict[str, list[float]] = {}
+ other: dict[str, dict[str, list[float]]] = {"ts": {}, "rust": {}}
+ for d in dirs:
+ for r in _jsonl(d / "py.jsonl"):
+ py.setdefault(r["test"], []).append(float(r.get("secs", 0)))
+ meta = d / "shard.json"
+ if meta.exists():
+ for key, secs in json.loads(meta.read_text()).get("unit_secs", {}).items():
+ lang, unit = key.split(":", 1)
+ other.setdefault(lang, {}).setdefault(unit, []).append(float(secs))
+ out: dict = {"py": {k: round(statistics.median(v), 1) for k, v in sorted(py.items())}}
+ for lang in ("ts", "rust"):
+ out[lang] = {k: round(statistics.median(v), 1) for k, v in sorted(other[lang].items())}
+ return out
+
+
+def build_timings(dirs: list[Path]) -> dict:
+ t = measured(dirs)
+ t["default"] = {
+ lang: round(statistics.median(t[lang].values()), 1) if t[lang] else 1.0 for lang in LANGS
+ }
+ return t
+
+
+def main(argv: list[str]) -> int:
+ if len(argv) < 2 or argv[0] not in ("timings", "check"):
+ print(__doc__)
+ return 2
+ cmd, rest = argv[0], argv[1:]
+ n = 0
+ if "--shards" in rest:
+ k = rest.index("--shards")
+ n = int(rest[k + 1])
+ del rest[k : k + 2]
+ dirs = [Path(a) for a in rest]
+ if cmd == "timings":
+ print(json.dumps(build_timings(dirs), indent=1, sort_keys=True))
+ return 0
+ metas = sorted(
+ (json.loads((d / "shard.json").read_text()) for d in dirs), key=lambda m: m["shard"]
+ )
+ for m in metas:
+ settings = ", ".join(f"{k}={v}" for k, v in m.get("settings", {}).items())
+ print(
+ f"shard {m['shard']}: predicted {m['predicted_secs']:6.1f}s, "
+ f"measured {m['wall_secs']:6.1f}s ({settings})"
+ )
+ print(
+ f"spread: predicted {spread([m['predicted_secs'] for m in metas]):.1f}%, "
+ f"measured {spread([m['wall_secs'] for m in metas]):.1f}%"
+ )
+ problems = coverage_problems(metas, n)
+ if problems:
+ print("\n".join(["", "PROBLEMS:", *problems]))
+ return 1
+ return 0
+
+
+def coverage_problems(metas: list[dict], n: int = 0) -> list[str]:
+ """The shards together ran the plan: one shard of each index, one plan,
+ every unit on exactly one shard."""
+ problems = []
+ count = n or (metas[0]["of"] if metas else 0)
+ if sorted(m["shard"] for m in metas) != list(range(1, count + 1)):
+ problems.append(f"want shards 1..{count}, found {sorted(m['shard'] for m in metas)}")
+ if {m["of"] for m in metas} - {count}:
+ problems.append(f"shards disagree on the shard count: {sorted({m['of'] for m in metas})}")
+ totals = {m["total_units"] for m in metas}
+ if len(totals) != 1:
+ problems.append(f"shards planned different suites: {sorted(totals)} units")
+ ran = [(lang, u) for m in metas for lang, units in m["units"].items() for u in units]
+ if len(set(ran)) != len(ran):
+ problems.append("a unit is planned on more than one shard")
+ if totals and len(ran) != next(iter(totals)):
+ problems.append(f"{len(ran)} units planned across shards, the suite has {sorted(totals)}")
+ return problems
+
+
+if __name__ == "__main__":
+ sys.exit(main(sys.argv[1:]))
diff --git a/tests/e2e/cua-sdk/test_sharding.py b/tests/e2e/cua-sdk/test_sharding.py
new file mode 100644
index 0000000000..d50c93b031
--- /dev/null
+++ b/tests/e2e/cua-sdk/test_sharding.py
@@ -0,0 +1,225 @@
+"""Shard planning for the cua SDK e2e suite (stdlib only).
+
+python3 -m unittest tests/e2e/cua-sdk/test_sharding.py
+"""
+
+from __future__ import annotations
+
+import importlib.util
+import json
+import tempfile
+import unittest
+from pathlib import Path
+
+HERE = Path(__file__).resolve().parent
+
+
+def _load(name: str):
+ spec = importlib.util.spec_from_file_location(f"e2e_{name}", HERE / f"{name}.py")
+ module = importlib.util.module_from_spec(spec)
+ import sys
+
+ sys.path.insert(0, str(HERE))
+ spec.loader.exec_module(module)
+ return module
+
+
+sharding = _load("sharding")
+run = _load("run")
+SHARDS = 4 # the E2E_SHARDS of .github/workflows/e2e-cua-sdk.yml
+
+
+def known_units(timings: dict) -> list[tuple[str, str]]:
+ units = [("py", f"test_x.py::{name}") for name in timings["py"]]
+ units += [("ts", name) for name in timings["ts"]]
+ units += [("rust", name) for name in timings["rust"]]
+ return units
+
+
+class Assign(unittest.TestCase):
+ timings = sharding.load_timings()
+
+ def test_every_unit_runs_on_exactly_one_shard(self) -> None:
+ units = known_units(self.timings) + [("py", "test_new.py::test_added_later")]
+ shards = sharding.assign(units, SHARDS, self.timings)
+ flat = [u for shard in shards for u in shard]
+ self.assertEqual(sorted(flat), sorted(set(units)))
+ self.assertEqual(len(flat), len(set(flat)))
+
+ def test_the_plan_is_deterministic_whatever_the_collection_order(self) -> None:
+ units = known_units(self.timings)
+ self.assertEqual(
+ sharding.assign(units, SHARDS, self.timings),
+ sharding.assign(list(reversed(units)), SHARDS, self.timings),
+ )
+
+ def test_shards_finish_within_ten_percent(self) -> None:
+ shards = sharding.assign(known_units(self.timings), SHARDS, self.timings)
+ loads = [sharding.predicted(s, self.timings) for s in shards]
+ self.assertLess(sharding.spread(loads), 10.0, loads)
+
+ def test_a_unit_without_a_timing_gets_the_default_weight(self) -> None:
+ self.assertEqual(
+ sharding.weight(self.timings, "py", "test_new.py::test_added_later"),
+ self.timings["default"]["py"],
+ )
+ self.assertEqual(
+ sharding.weight(self.timings, "ts", "new.test.mjs"), self.timings["default"]["ts"]
+ )
+
+ def test_py_units_are_weighed_by_test_name(self) -> None:
+ name = next(k for k, v in self.timings["py"].items() if v > 10)
+ self.assertEqual(
+ sharding.weight(self.timings, "py", f"python/test_a.py::{name}"),
+ self.timings["py"][name],
+ )
+
+ def test_more_shards_than_units_leaves_some_empty(self) -> None:
+ shards = sharding.assign([("ts", "a.test.mjs")], 3, {})
+ self.assertEqual(sum(1 for s in shards if s), 1)
+
+ def test_shard_spec(self) -> None:
+ self.assertEqual(sharding.parse_shard("2/4"), (2, 4))
+ for bad in ("0/4", "5/4", "x", "1/0", "2"):
+ with self.assertRaises(SystemExit):
+ sharding.parse_shard(bad)
+
+
+class Measured(unittest.TestCase):
+ def test_timings_come_from_results_and_shard_files(self) -> None:
+ with tempfile.TemporaryDirectory() as tmp:
+ a, b = Path(tmp, "a"), Path(tmp, "b")
+ for d, secs in ((a, 4.0), (b, 6.0)):
+ d.mkdir()
+ (d / "py.jsonl").write_text(json.dumps({"test": "test_one", "secs": secs}) + "\n")
+ (d / "shard.json").write_text(
+ json.dumps({"unit_secs": {"ts:core.test.mjs": secs * 2}})
+ )
+ t = sharding.build_timings([a, b])
+ self.assertEqual(t["py"], {"test_one": 5.0})
+ self.assertEqual(t["ts"], {"core.test.mjs": 10.0})
+ self.assertEqual(t["rust"], {})
+ self.assertEqual(t["default"]["py"], 5.0)
+
+
+class Workflow(unittest.TestCase):
+ def test_the_shard_matrix_matches_the_shard_count(self) -> None:
+ import re
+
+ text = (HERE.parents[2] / ".github" / "workflows" / "e2e-cua-sdk.yml").read_text()
+ count = int(re.search(r'^ E2E_SHARDS: "(\d+)"', text, re.M).group(1))
+ matrix = re.search(r"^\s+shard: \[([\d, ]+)\]", text, re.M).group(1)
+ self.assertEqual([int(x) for x in matrix.split(",")], list(range(1, count + 1)))
+ self.assertEqual(count, SHARDS)
+
+
+class Coverage(unittest.TestCase):
+ @staticmethod
+ def meta(shard: int, units: dict, total: int = 4, of: int = 2) -> dict:
+ return {"shard": shard, "of": of, "total_units": total, "units": units}
+
+ def test_complete_shards_pass(self) -> None:
+ metas = [
+ self.meta(1, {"py": ["a", "b"], "ts": [], "rust": []}),
+ self.meta(2, {"py": ["c"], "ts": ["x"], "rust": []}),
+ ]
+ self.assertEqual(sharding.coverage_problems(metas, 2), [])
+
+ def test_a_missing_shard_is_reported(self) -> None:
+ metas = [self.meta(1, {"py": ["a", "b", "c", "d"], "ts": [], "rust": []})]
+ self.assertTrue(sharding.coverage_problems(metas, 2))
+
+ def test_a_dropped_or_doubled_unit_is_reported(self) -> None:
+ dropped = [
+ self.meta(1, {"py": ["a", "b"], "ts": [], "rust": []}),
+ self.meta(2, {"py": ["c"], "ts": [], "rust": []}),
+ ]
+ doubled = [
+ self.meta(1, {"py": ["a", "b"], "ts": [], "rust": []}),
+ self.meta(2, {"py": ["b", "c"], "ts": [], "rust": []}),
+ ]
+ self.assertTrue(sharding.coverage_problems(dropped, 2))
+ self.assertTrue(sharding.coverage_problems(doubled, 2))
+
+
+class RunnerChecks(unittest.TestCase):
+ def test_a_shard_flags_unexpected_skips_but_not_allowed_ones(self) -> None:
+ rows = [
+ {
+ "scenario": "s",
+ "lane": "container",
+ "lang": "py",
+ "status": "skip",
+ "reason": "docker is not on PATH",
+ },
+ {
+ "scenario": "s",
+ "lane": "hermetic",
+ "lang": "ts",
+ "status": "skip",
+ "reason": "playwright is not installed",
+ },
+ {
+ "scenario": "s",
+ "lane": "qemu",
+ "lang": "py",
+ "status": "skip",
+ "reason": "not requested",
+ },
+ ]
+ problems = run.skip_problems(rows, ["hermetic", "container"])
+ self.assertEqual(len(problems), 1)
+ self.assertIn("docker is not on PATH", problems[0])
+
+ def test_a_relative_rust_prebuilt_runs_from_the_invocation_directory(self) -> None:
+ """The workflow passes `--rust-prebuilt tests/e2e/cua-sdk/rust/prebuilt` from the
+ repository root; the binaries run with the Rust half's directory as cwd."""
+ import os
+ import subprocess
+ import sys
+
+ rel = "tests/e2e/cua-sdk/rust/prebuilt"
+ with tempfile.TemporaryDirectory() as tmp:
+ prebuilt = Path(tmp, rel)
+ prebuilt.mkdir(parents=True)
+ fake = prebuilt / "guides"
+ fake.write_text('#!/bin/sh\necho "ran $0 $@" >"$CUA_E2E_RESULTS/ran.txt"\n')
+ fake.chmod(0o755)
+ (prebuilt / "manifest.json").write_text(
+ json.dumps([{"name": "guides", "file": "guides"}])
+ )
+ proc = subprocess.run(
+ [
+ sys.executable,
+ str(HERE / "run.py"),
+ "--lanes",
+ "hermetic",
+ "--langs",
+ "rust",
+ "--shard",
+ "1/1",
+ "--rust-prebuilt",
+ rel,
+ "--out",
+ "out",
+ ],
+ cwd=tmp,
+ env=dict(os.environ, CUA_E2E_RUN="gh123456-1"),
+ capture_output=True,
+ text=True,
+ )
+ self.assertEqual(proc.returncode, 0, proc.stdout + proc.stderr)
+ ran = Path(tmp, "out", "ran.txt").read_text()
+ self.assertTrue(ran.strip().endswith(f"{rel}/guides --test-threads=2"), ran)
+ self.assertNotIn("rust/tests/e2e", ran)
+
+ def test_prebuilt_rust_binaries_are_listed_by_the_manifest(self) -> None:
+ with tempfile.TemporaryDirectory() as tmp:
+ Path(tmp, "manifest.json").write_text(json.dumps([{"name": "core", "file": "core"}]))
+ self.assertEqual(run.rust_binaries(Path(tmp)), {"core": "core"})
+ with self.assertRaises(SystemExit):
+ run.rust_binaries(Path(tmp, "missing"))
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/tests/e2e/cua-sdk/timings.json b/tests/e2e/cua-sdk/timings.json
new file mode 100644
index 0000000000..c42b62aa8e
--- /dev/null
+++ b/tests/e2e/cua-sdk/timings.json
@@ -0,0 +1,132 @@
+{
+ "default": {
+ "py": 3.0,
+ "rust": 10.0,
+ "ts": 20.0
+ },
+ "py": {
+ "test_a_pool_name_is_cloud_ended": 0.0,
+ "test_agnostic_cloud_ended_and_direct": 2.5,
+ "test_agnostic_container_ssh_only": 4.6,
+ "test_agnostic_container_vnc_only": 5.6,
+ "test_agnostic_qemu_ssh_only": 0.0,
+ "test_browse_local_sandbox": 12.3,
+ "test_browse_mcp_surface": 0.8,
+ "test_cli_build_push_boot_vm": 0.0,
+ "test_cli_build_push_run_container": 16.3,
+ "test_container_block[cua-sdk/guides/desktop#desktop-python]": 52.1,
+ "test_container_block[cua-sdk/guides/desktop#desktop-viewer-url]": 2.8,
+ "test_container_block[cua-sdk/guides/desktop#terminal]": 3.0,
+ "test_container_block[cua-sdk/guides/desktop#typed-driver]": 3.0,
+ "test_container_block[cua-sdk/guides/images#start-dev]": 48.2,
+ "test_container_block[cua-sdk/guides/lifecycle#ephemeral-or-persistent]": 4.6,
+ "test_container_block[cua-sdk/guides/lifecycle#inspect]": 4.4,
+ "test_container_block[cua-sdk/guides/lifecycle#reconnect-list-delete]": 4.5,
+ "test_container_block[cua-sdk/guides/own-harness#harness-python]": 5.8,
+ "test_container_block[cua-sdk/guides/services#call]": 2.2,
+ "test_container_block[cua-sdk/guides/services#forward-python]": 2.0,
+ "test_container_block[cua-sdk/guides/services#mcp-call]": 8.9,
+ "test_container_block[cua-sdk/guides/services#mcp-local]": 7.8,
+ "test_container_block[cua-sdk/guides/services#mcp-server]": 0.0,
+ "test_container_block[cua-sdk/guides/services#public-url]": 1.9,
+ "test_container_block[cua-sdk/guides/services#sidecar-by-name]": 4.0,
+ "test_container_block[cua-sdk/guides/services#sidecar-local]": 3.3,
+ "test_container_block[cua-sdk/guides/services#start-local]": 5.0,
+ "test_container_block[cua-sdk/guides/test-in-ci#ci-overlay-python]": 1.8,
+ "test_container_block[cua-sdk/index#local-ephemeral]": 2.1,
+ "test_container_block[cua-sdk/quickstart#local-server]": 2.2,
+ "test_container_block[cua-sdk/reference/python/sandbox#sandbox-create-py]": 2.1,
+ "test_container_block[cua-sdk/reference/sandbox/index#sandboxes-create-py]": 2.4,
+ "test_container_block[spaces/guides/create-a-space-with-the-sdk#first-space]": 4.5,
+ "test_container_ts_block[cua-sdk/guides/desktop#desktop-typescript]": 6.7,
+ "test_container_ts_block[cua-sdk/guides/own-harness#harness-typescript]": 7.2,
+ "test_container_ts_block[cua-sdk/guides/services#mcp-typescript]": 9.4,
+ "test_container_ts_block[cua-sdk/guides/services#typescript]": 3.5,
+ "test_container_ts_block[cua-sdk/index#local-ephemeral-ts]": 3.6,
+ "test_container_ts_block[cua-sdk/quickstart#typescript]": 3.3,
+ "test_container_ts_block[cua-sdk/reference/sandbox/index#sandboxes-create-ts]": 2.9,
+ "test_container_ts_block[spaces/guides/create-a-space-with-the-sdk#first-space-ts]": 6.0,
+ "test_contrib_block[cua-sdk/guides/local-runtimes#contrib-python]": 0.0,
+ "test_daemon_vs_embedded_local_container": 8.0,
+ "test_daemon_vs_embedded_mock": 2.5,
+ "test_direct_connect_docker": 2.0,
+ "test_direct_connect_mock": 0.7,
+ "test_docs_block[cua-sdk/guides/images#canonical]": 1.8,
+ "test_docs_block[cua-sdk/guides/images#chain]": 0.2,
+ "test_docs_block[cua-sdk/guides/images#fork]": 0.2,
+ "test_docs_block[cua-sdk/guides/images#inspect]": 0.2,
+ "test_docs_block[cua-sdk/guides/images#install-packages]": 0.2,
+ "test_docs_block[cua-sdk/guides/images#registry]": 0.2,
+ "test_docs_block[cua-sdk/guides/own-harness#harness-connect-python]": 2.0,
+ "test_docs_block[cua-sdk/guides/run-a-coding-agent#harness-ids]": 0.1,
+ "test_docs_block[cua-sdk/guides/services#mcp-server]": 0.0,
+ "test_docs_block[cua-sdk/guides/services#run-with-key-file]": 0.2,
+ "test_docs_block[cua-sdk/guides/services#run-with-secrets]": 0.2,
+ "test_docs_block[cua-sdk/index#direct-connect]": 2.0,
+ "test_docs_block[cua-sdk/reference/cua#cua-embedded-py]": 0.2,
+ "test_docs_block[cua-sdk/reference/cua#local-doctor-py]": 0.2,
+ "test_docs_block[cua-sdk/reference/errors#cua-error-py]": 1.9,
+ "test_docs_block[cua-sdk/reference/image#canonical-image-py]": 0.1,
+ "test_docs_block[cua-sdk/reference/image#image-alias-py]": 0.1,
+ "test_docs_block[cua-sdk/reference/image#normalize-image-py]": 0.1,
+ "test_docs_block[cua-sdk/reference/python/index#image-py]": 0.2,
+ "test_docs_block[cua-sdk/reference/python/interfaces#files-write-text-py]": 2.1,
+ "test_docs_block[cua-sdk/reference/python/interfaces#mouse-click-py]": 2.0,
+ "test_docs_block[cua-sdk/reference/python/sandbox#image-from-dict-py]": 0.2,
+ "test_docs_block[cua-sdk/reference/python/sandbox#image-py]": 0.2,
+ "test_docs_block[cua-sdk/reference/python/sandbox#registry-secret-py]": 0.2,
+ "test_docs_block[cua-sdk/reference/python/sandbox#sandbox-connect-py]": 2.1,
+ "test_docs_block[cua-sdk/reference/sandbox/index#sandboxes-connect-url-py]": 2.0,
+ "test_docs_block[cua-sdk/reference/sandbox/lifecycle#parse-sandbox-ref-py]": 0.1,
+ "test_docs_block[cua-sdk/reference/sandbox/lifecycle#qualify-sandbox-ref-py]": 0.1,
+ "test_docs_block[cua-sdk/reference/spaces/index#spaces-add-py]": 1.8,
+ "test_docs_block[cua-sdk/reference/spacesd/computer#spacesd-client-screenshot-py]": 1.8,
+ "test_docs_block[cua-sdk/reference/spacesd/files#spacesd-client-upload-py]": 1.9,
+ "test_docs_block[cua-sdk/reference/spacesd/shell#spacesd-client-run-py]": 1.9,
+ "test_docs_block[cua-sdk/reference/spacesd/shell#spacesd-client-spawn-py]": 1.9,
+ "test_docs_block[spaces/guides/add-a-space#add-direct]": 1.8,
+ "test_docs_ts_block[cua-sdk/guides/images#typescript]": 7.0,
+ "test_docs_ts_block[cua-sdk/guides/own-harness#harness-connect-typescript]": 3.5,
+ "test_docs_ts_block[cua-sdk/guides/services#sidecar-typescript]": 1.7,
+ "test_docs_ts_block[cua-sdk/index#direct-connect-ts]": 3.4,
+ "test_docs_ts_block[cua-sdk/reference/cua#cua-embedded-ts]": 1.8,
+ "test_docs_ts_block[cua-sdk/reference/errors#cua-error-ts]": 3.4,
+ "test_docs_ts_block[cua-sdk/reference/sandbox/index#sandboxes-connect-url-ts]": 3.6,
+ "test_docs_ts_block[cua-sdk/reference/spacesd/shell#spacesd-client-run-ts]": 3.6,
+ "test_docs_ts_block[spaces/guides/add-a-space#add-direct-ts]": 3.5,
+ "test_fork_many_from_a_memory_checkpoint": 0.0,
+ "test_guides_are_tagged": 0.0,
+ "test_local_container_desktop": 2.5,
+ "test_local_false_is_cloud_ended": 0.0,
+ "test_local_qemu_env_token_reaches_guest": 0.0,
+ "test_local_qemu_fork_fan_out_of_a_pinned_checkpoint": 0.0,
+ "test_local_qemu_lifecycle_probe_forward": 0.0,
+ "test_on_cloud_is_cloud_ended": 0.0,
+ "test_parity_cloud_ended_and_local_vm_refuses_layers": 0.1,
+ "test_parity_sidecars_by_name_through_the_daemon": 20.3,
+ "test_run_omarchy_local": 2.5,
+ "test_sdk_local_layers_apply": 7.1,
+ "test_share_a_link": 0.9,
+ "test_tcp_probe_implies_listening": 1.0,
+ "test_typed_click_reaches_gtk": 1.3,
+ "test_unified_api_cloud_ended_and_daemon_public_url": 15.6,
+ "test_unified_api_plain_image_gvisor": 17.3,
+ "test_viewer_local": 0.0
+ },
+ "rust": {
+ "core": 34.0,
+ "cua-sdk-e2e": 0.1,
+ "fork_batch": 0.1,
+ "guides": 27.0,
+ "local_vm": 0.1,
+ "share": 1.0,
+ "unified": 3.0
+ },
+ "ts": {
+ "browser.test.mjs": 3.0,
+ "core.test.mjs": 35.0,
+ "guides.test.mjs": 31.0,
+ "share.test.mjs": 2.0,
+ "unified.test.mjs": 31.0
+ }
+}
\ No newline at end of file