Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
ci: E2E: cua SDK builds in parallel and tests on 4 time-balanced shards (about 69 to 25 min) (#5826)
Francesco Bonaccicc2aab774c60
diff --git a/.github/scripts/tests/test_e2e_cua_sdk_scope.py b/.github/scripts/tests/test_e2e_cua_sdk_scope.pyindex 90ec5288da..93d52faf5f 100644--- a/.github/scripts/tests/test_e2e_cua_sdk_scope.py+++ b/.github/scripts/tests/test_e2e_cua_sdk_scope.py@@ -353,10 +353,17 @@ def test_every_path_matches_itself_and_nothing_wider():def test_workflow_wiring():jobs = WORKFLOW["jobs"]- changes, local, live = jobs["changes"], jobs["local"], jobs["live"]- assert local["needs"] == "changes"- # Only an explicit `false` skips the suite.- assert "needs.changes.outputs.run != 'false'" in local["if"]+ changes, live = jobs["changes"], jobs["live"]+ # The suite is five build jobs, the test shards that need all of them, and+ # the merge job that needs the shards: skipping the builds skips the rest.+ builds = ["build-sdk", "build-cli", "build-fixtures", "build-rust-tests", "build-images"]+ for name in builds:+ assert jobs[name]["needs"] == "changes", name+ # Only an explicit `false` skips the suite.+ assert "needs.changes.outputs.run != 'false'" in jobs[name]["if"], name+ assert sorted(jobs["shard"]["needs"]) == sorted(builds)+ assert jobs["matrix"]["needs"] == "shard"+ assert "needs" not in jobs["shard"].get("if", "")assert changes["outputs"] == {"run": "${{ steps.scope.outputs.run }}"}checkout = changes["steps"][0]assert checkout["with"]["fetch-depth"] == 2diff --git a/.github/workflows/e2e-cua-sdk.yml b/.github/workflows/e2e-cua-sdk.ymlindex ba2d583c4e..96815843f4 100644--- a/.github/workflows/e2e-cua-sdk.yml+++ b/.github/workflows/e2e-cua-sdk.yml@@ -4,9 +4,11 @@ name: "E2E: cua SDK"# tests/e2e/cua-sdk (see its scenarios.json for the guide x lane registry).## pull_request hermetic (fixtures, daemon) + container (docker + gVisor on-# the locally built reference images) + docs blocks. A docs-# edit that leaves every fenced code block as it was skips the-# suite (the `changes` job, .github/scripts/e2e_cua_sdk_scope.py).+# the locally built reference images) + docs blocks: five+# build jobs in parallel, then 4 test shards (see `jobs:`). A+# docs edit that leaves every fenced code block as it was+# skips the suite (the `changes` job,+# .github/scripts/e2e_cua_sdk_scope.py).# nightly cua-sandbox and local QEMU (KVM) on the reference disk.# Cua Cloud has ended, so there is no live Fleet lane.# dispatch any lane set.@@ -65,6 +67,15 @@ on:langs:description: "Languages (py,ts,rust)"default: "py,ts,rust"+ # For measuring concurrency: run the same commit at 2 or 3 nearby values,+ # at least 3 times each, and compare the shards' `measured` seconds in+ # the run summary.+ rust_threads:+ description: "Rust test threads per binary (the default, 2, is what the suite has always used)"+ default: "2"+ ts_concurrency:+ description: "node --test-concurrency (the default, 1, runs files one at a time)"+ default: "1"permissions:contents: read@@ -83,6 +94,8 @@ env:CARGO_TERM_COLOR: alwaysCARGO_BUILD_JOBS: "4"CUA_E2E_RUN: gh${{ github.run_id }}-${{ github.run_attempt }}+ # Test shards of a pull request run (the shard job's matrix lists 1..N).+ E2E_SHARDS: "4"jobs:# Pull requests: the paths filter above starts this workflow for any docs@@ -112,21 +125,112 @@ jobs:HEAD_REF: ${{ github.head_ref }}run: python3 .github/scripts/e2e_cua_sdk_scope.py | tee -a "$GITHUB_STEP_SUMMARY"- local:- name: hermetic + container (${{ github.event_name == 'workflow_dispatch' && inputs.lanes || 'hermetic,container' }})+ # Pull requests: build once, test on N shards.+ #+ # build-sdk, build-cli, build-fixtures, build-rust-tests and build-images+ # run side by side on their own runners (the release SDK; the debug CLI+ # with the TypeScript and wasm builds; the fixtures and the static+ # cua-spacesd; the Rust test binaries; the reference images) and upload+ # what they built (ci-artifacts.sh).+ # shard 1..E2E_SHARDS download all of it and each runs its slice of the+ # suite. The slices are cut from measured test times (sharding.py,+ # timings.json) so they finish within a few percent of each other.+ # matrix merges the shards' results and applies the whole-suite checks+ # (every cell present, docs coverage) that one run used to apply.+ #+ # Before this, one job built everything in series (about 46 minutes of+ # builds) and then ran the three language halves in series (about 15).+ build-sdk:+ name: build / release SDKneeds: changesif: ${{ !startsWith(github.head_ref, 'release-please--branches--') && (github.event_name != 'schedule') && needs.changes.outputs.run != 'false' }}runs-on: ubuntu-24.04- # Run 37562962482 exhausted 180 minutes during cold browser compilation,- # before image preparation or the suite. Allow the hosted runner's full- # six-hour budget for the unchanged build and strict test matrix.- timeout-minutes: 360+ timeout-minutes: 240+ env:+ CARGO_PROFILE_DEV_DEBUG: line-tables-only+ CARGO_INCREMENTAL: "0"+ steps:+ - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4+ - uses: ./.github/actions/linux-rust-prep+ - name: Install protoc+ uses: ./.github/actions/setup-protoc+ - name: Install pinned Rust toolchain+ uses: dtolnay/[email protected]+ - uses: ./.github/actions/rust-cache+ with:+ workspaces: |+ libs/cua+ tests/e2e/cua-sdk/rust+ libs/cua-spacesd+ key: cua-e2e+ - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4+ with:+ node-version: "22"+ - name: Build the release SDK and stage it for Python and Node+ run: tests/e2e/cua-sdk/ci-setup.sh --only sdk,stage+ - name: Pack+ run: tests/e2e/cua-sdk/ci-artifacts.sh pack sdk "$RUNNER_TEMP/build"+ - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4+ with:+ name: cua-e2e-build-sdk+ path: ${{ runner.temp }}/build/cua-e2e-build-sdk.tar.zst+ compression-level: 0+ retention-days: 2+ overwrite: true+ if-no-files-found: error++ build-cli:+ name: build / debug CLI, TypeScript, wasm+ needs: changes+ if: ${{ !startsWith(github.head_ref, 'release-please--branches--') && (github.event_name != 'schedule') && needs.changes.outputs.run != 'false' }}+ runs-on: ubuntu-24.04+ timeout-minutes: 240+ env:+ CARGO_PROFILE_DEV_DEBUG: line-tables-only+ CARGO_INCREMENTAL: "0"+ steps:+ - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4+ - name: Unit-test the shard planner and the runner helpers+ run: python3 -m unittest tests/e2e/cua-sdk/test_sharding.py tests/e2e/cua-sdk/test_run_janitor.py+ - uses: ./.github/actions/linux-rust-prep+ - name: Install protoc+ uses: ./.github/actions/setup-protoc+ - name: Install pinned Rust toolchain+ uses: dtolnay/[email protected]+ with:+ targets: wasm32-unknown-unknown+ - uses: ./.github/actions/rust-cache+ with:+ workspaces: |+ libs/cua+ tests/e2e/cua-sdk/rust+ libs/cua-spacesd+ key: cua-e2e+ - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4+ with:+ node-version: "22"+ # The wasm build generates its bindings from the debug cua-sdk library,+ # so it follows the debug build on this runner.+ - name: Build the debug CLI, the TypeScript package and the wasm browser build+ run: tests/e2e/cua-sdk/ci-setup.sh --only cli,web+ - name: Pack+ run: tests/e2e/cua-sdk/ci-artifacts.sh pack cli "$RUNNER_TEMP/build"+ - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4+ with:+ name: cua-e2e-build-cli+ path: ${{ runner.temp }}/build/cua-e2e-build-cli.tar.zst+ compression-level: 0+ retention-days: 2+ overwrite: true+ if-no-files-found: error++ build-fixtures:+ name: build / fixtures, static cua-spacesd+ needs: changes+ if: ${{ !startsWith(github.head_ref, 'release-please--branches--') && (github.event_name != 'schedule') && needs.changes.outputs.run != 'false' }}+ runs-on: ubuntu-24.04+ timeout-minutes: 240env:- # Optional: an authenticated Docker Hub keeps anonymous manifest-read- # rate limits from failing the docs blocks' runtime inspection.- DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}- # The debug builds, images and the suite's own image builds share the- # runner's disk: line tables only and no incremental cache.CARGO_PROFILE_DEV_DEBUG: line-tables-onlyCARGO_INCREMENTAL: "0"steps:@@ -137,7 +241,7 @@ jobs:- name: Install pinned Rust toolchainuses: dtolnay/[email protected]with:- targets: wasm32-unknown-unknown,x86_64-unknown-linux-musl+ targets: x86_64-unknown-linux-musl- uses: ./.github/actions/rust-cachewith:workspaces: |@@ -147,6 +251,7 @@ jobs:key: cua-e2e# An image without cua-spacesd gets the static musl build injected, so# the suite tests this tree's daemon (same build as ci-cua-oci-spacesd).+ # The shards point CUA_SPACESD_STATIC at this file in their checkout.- name: Build the static cua-spacesdworking-directory: libs/cua-spacesdrun: |@@ -154,40 +259,193 @@ jobs:sudo apt-get update -qq && sudo apt-get install -y -qq musl-toolscargo build --locked --profile static --target x86_64-unknown-linux-musl -p cua-spacesd --bin cua-spacesdbash packaging/release/check-static-binary.sh target/x86_64-unknown-linux-musl/static/cua-spacesd- mkdir -p "$RUNNER_TEMP/static"- cp target/x86_64-unknown-linux-musl/static/cua-spacesd "$RUNNER_TEMP/static/cua-spacesd-linux-x86_64"- chmod +x "$RUNNER_TEMP/static/cua-spacesd-linux-x86_64"- echo "CUA_SPACESD_STATIC=$RUNNER_TEMP/static/cua-spacesd-linux-x86_64" >>"$GITHUB_ENV"+ chmod +x target/x86_64-unknown-linux-musl/static/cua-spacesd+ - name: Build the test fixtures+ run: tests/e2e/cua-sdk/ci-setup.sh --only fixtures+ - name: Pack+ run: tests/e2e/cua-sdk/ci-artifacts.sh pack fixtures "$RUNNER_TEMP/build"+ - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4+ with:+ name: cua-e2e-build-fixtures+ path: ${{ runner.temp }}/build/cua-e2e-build-fixtures.tar.zst+ compression-level: 0+ retention-days: 2+ overwrite: true+ if-no-files-found: error++ build-rust-tests:+ name: build / Rust test binaries+ needs: changes+ if: ${{ !startsWith(github.head_ref, 'release-please--branches--') && (github.event_name != 'schedule') && needs.changes.outputs.run != 'false' }}+ runs-on: ubuntu-24.04+ timeout-minutes: 240+ env:+ CARGO_PROFILE_DEV_DEBUG: line-tables-only+ CARGO_INCREMENTAL: "0"+ steps:+ - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4+ - uses: ./.github/actions/linux-rust-prep+ - name: Install protoc+ uses: ./.github/actions/setup-protoc+ - name: Install pinned Rust toolchain+ uses: dtolnay/[email protected]+ - uses: ./.github/actions/rust-cache+ with:+ workspaces: |+ libs/cua+ tests/e2e/cua-sdk/rust+ libs/cua-spacesd+ key: cua-e2e+ # The Rust half's own build (its own target directory), done once here+ # instead of inside `cargo test` on every shard.+ - name: Build the Rust half's test binaries+ run: tests/e2e/cua-sdk/ci-setup.sh --only rust-tests+ - name: Pack+ run: tests/e2e/cua-sdk/ci-artifacts.sh pack rust-tests "$RUNNER_TEMP/build"+ - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4+ with:+ name: cua-e2e-build-rust-tests+ path: ${{ runner.temp }}/build/cua-e2e-build-rust-tests.tar.zst+ compression-level: 0+ retention-days: 2+ overwrite: true+ if-no-files-found: error++ build-images:+ name: build / reference images+ needs: changes+ if: ${{ !startsWith(github.head_ref, 'release-please--branches--') && (github.event_name != 'schedule') && needs.changes.outputs.run != 'false' }}+ runs-on: ubuntu-24.04+ timeout-minutes: 120+ env:+ # Optional: an authenticated Docker Hub keeps anonymous pull rate limits+ # from failing the base image pulls.+ DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}+ steps:+ - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4+ - name: Log in to Docker Hub+ if: env.DOCKERHUB_USERNAME != ''+ uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3+ with:+ username: ${{ secrets.DOCKERHUB_USERNAME }}+ password: ${{ secrets.DOCKERHUB_TOKEN }}+ # cua-spacesd builds in a rust container and the three images on top,+ # all in the runner's docker.+ - name: Build the reference images+ run: tests/e2e/cua-sdk/ci-setup.sh --images --only images+ - name: Pack+ run: tests/e2e/cua-sdk/ci-artifacts.sh pack images "$RUNNER_TEMP/build"+ - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4+ with:+ name: cua-e2e-build-images+ path: ${{ runner.temp }}/build/cua-e2e-build-images.tar.zst+ compression-level: 0+ retention-days: 2+ overwrite: true+ if-no-files-found: error++ shard:+ name: tests (shard ${{ matrix.shard }})+ needs: [build-sdk, build-cli, build-fixtures, build-rust-tests, build-images]+ runs-on: ubuntu-24.04+ timeout-minutes: 90+ strategy:+ fail-fast: false+ matrix:+ # The count is E2E_SHARDS: a test pins the two together.+ shard: [1, 2, 3, 4]+ env:+ DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}+ steps:+ - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4+ # The shared libraries the built binaries link against (the build jobs+ # installed them with the headers); the disk is not tight here.+ - uses: ./.github/actions/linux-rust-prep+ with:+ free-disk: "false"- uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4with:node-version: "22"- uses: astral-sh/setup-uv@e92bafb6253dcd438e0484186d7669ea7a8ca1cc # v6+ - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4+ with:+ pattern: cua-e2e-build-*+ path: ${{ runner.temp }}/build+ merge-multiple: true- name: Log in to Docker Hub (docs blocks read image manifests)if: env.DOCKERHUB_USERNAME != ''uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3with:username: ${{ secrets.DOCKERHUB_USERNAME }}password: ${{ secrets.DOCKERHUB_TOKEN }}- - name: Build the SDK, fixtures, bindings, images and runtimes- run: tests/e2e/cua-sdk/ci-setup.sh --images- # The suite builds container images and cua keeps 5 GiB free: give the- # build cache back (the built images stay).- - name: Reclaim build space- run: |- df -h / | tail -1- docker builder prune --all --force >/dev/null- docker image prune --force >/dev/null- df -h / | tail -1- - name: Run the suite+ - name: Unpack the builds, install gVisor, load the images+ run: tests/e2e/cua-sdk/ci-artifacts.sh prepare "$RUNNER_TEMP/build"+ - name: Run this shard of the suiteenv:LANES: ${{ github.event_name == 'workflow_dispatch' && inputs.lanes || 'hermetic,container' }}LANGS: ${{ github.event_name == 'workflow_dispatch' && inputs.langs || 'py,ts,rust' }}CUA_E2E_PLAYWRIGHT_DIR: ${{ runner.temp }}/pw- # cua keeps 5 GiB free by default. On this disposable runner, after- # the setup and the Rust half's own debug build, that refused the- # suite's 4 GiB image builds with 7.5 GiB still free.+ CUA_E2E_RUST_THREADS: ${{ github.event_name == 'workflow_dispatch' && inputs.rust_threads || '2' }}+ CUA_E2E_TS_CONCURRENCY: ${{ github.event_name == 'workflow_dispatch' && inputs.ts_concurrency || '1' }}+ # The static cua-spacesd the build-fixtures job made, in this checkout.+ CUA_SPACESD_STATIC: ${{ github.workspace }}/libs/cua-spacesd/target/x86_64-unknown-linux-musl/static/cua-spacesd+ # cua keeps 5 GiB free by default. On this disposable runner that+ # refused the suite's 4 GiB image builds with 7.5 GiB still free.CUA_DISK_MIN_FREE: 2GiB- run: python3 tests/e2e/cua-sdk/run.py --lanes "$LANES" --langs "$LANGS" --strict --out "$RUNNER_TEMP/cua-e2e"+ run: >+ python3 tests/e2e/cua-sdk/run.py --lanes "$LANES" --langs "$LANGS" --strict+ --shard "${{ matrix.shard }}/$E2E_SHARDS"+ --rust-prebuilt tests/e2e/cua-sdk/rust/prebuilt+ --out "$RUNNER_TEMP/cua-e2e"+ - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4+ if: always()+ with:+ name: cua-e2e-shard-${{ matrix.shard }}+ path: ${{ runner.temp }}/cua-e2e+ retention-days: 7+ overwrite: true++ # The check that used to be the one big job: every shard's results merged,+ # then the whole-suite checks on the merge (every expected cell present, no+ # unexpected skip, docs code-block coverage) and the matrix in the summary.+ matrix:+ name: matrix (${{ github.event_name == 'workflow_dispatch' && inputs.lanes || 'hermetic,container' }})+ needs: shard+ if: ${{ !cancelled() && needs.shard.result != 'skipped' }}+ runs-on: ubuntu-24.04+ timeout-minutes: 15+ steps:+ - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4+ - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4+ with:+ pattern: cua-e2e-shard-*+ path: ${{ runner.temp }}/shards+ - name: Merge the shards' results+ run: |+ set -euo pipefail+ mkdir -p "$RUNNER_TEMP/cua-e2e"+ for dir in "$RUNNER_TEMP"/shards/cua-e2e-shard-*; do+ n="${dir##*-}"+ for f in "$dir"/*.jsonl; do+ [ -e "$f" ] && cat "$f" >>"$RUNNER_TEMP/cua-e2e/$(basename "$f")"+ done+ cp "$dir/shard.json" "$RUNNER_TEMP/cua-e2e/shard-$n.json" || true+ done+ ls -l "$RUNNER_TEMP/cua-e2e"+ - name: Check the merged matrix+ env:+ LANES: ${{ github.event_name == 'workflow_dispatch' && inputs.lanes || 'hermetic,container' }}+ LANGS: ${{ github.event_name == 'workflow_dispatch' && inputs.langs || 'py,ts,rust' }}+ run: python3 tests/e2e/cua-sdk/run.py --matrix-only --lanes "$LANES" --langs "$LANGS" --strict --out "$RUNNER_TEMP/cua-e2e"+ - name: Shards covered the whole plan+ if: ${{ !cancelled() }}+ run: |+ set -euo pipefail+ {+ printf '## Shards (predicted vs measured seconds)\n\n```\n'+ python3 tests/e2e/cua-sdk/sharding.py check "$RUNNER_TEMP"/shards/cua-e2e-shard-* --shards "$E2E_SHARDS" | tee /dev/stderr+ printf '```\n'+ } >>"$GITHUB_STEP_SUMMARY"- name: Matrix summaryif: always()run: |@@ -201,6 +459,7 @@ jobs:with:name: cua-e2e-localpath: ${{ runner.temp }}/cua-e2e+ overwrite: truelive:name: nightly (cua-sandbox, qemu)diff --git a/tests/e2e/cua-sdk/.gitignore b/tests/e2e/cua-sdk/.gitignoreindex 39d637e2c5..33fad17e40 100644--- a/tests/e2e/cua-sdk/.gitignore+++ b/tests/e2e/cua-sdk/.gitignore@@ -1,3 +1,4 @@go/.target/rust/target/.playwright/+rust/prebuilt/diff --git a/tests/e2e/cua-sdk/ci-artifacts.sh b/tests/e2e/cua-sdk/ci-artifacts.shnew file mode 100755index 0000000000..b3680d8d87--- /dev/null+++ b/tests/e2e/cua-sdk/ci-artifacts.sh@@ -0,0 +1,88 @@+#!/usr/bin/env bash+# Hands the build jobs' outputs to the test shards of+# .github/workflows/e2e-cua-sdk.yml. Every archive is a tar.zst of paths+# relative to the repository root, so the shard sees the tree the single-job+# workflow used to build (same checkout path, same file modes).+#+# ci-artifacts.sh pack sdk|cli|fixtures|rust-tests|images DIR+# DIR/cua-e2e-build-<name>.tar.zst+# sdk the release library staged for Python and Node (+ node_modules)+# cli the debug cua CLI and the TypeScript + wasm browser builds+# fixtures cua-test-fixtures and the static cua-spacesd the suite+# injects into plain images+# rust-tests the prebuilt Rust test binaries+# images the reference images, as one `docker save`+# ci-artifacts.sh prepare DIR+# a shard's setup: gVisor, the archives of DIR unpacked, the images+# loaded, and the canonical desktop image pulled while they load+set -euo pipefail+HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"+REPO="$(cd "$HERE/../../.." && pwd)"+STATIC_SPACESD=libs/cua-spacesd/target/x86_64-unknown-linux-musl/static/cua-spacesd++pack_paths() {+ local name="$1" out="$2"; shift 2+ local path+ for path in "$@"; do+ [ -e "$REPO/$path" ] || { echo "pack $name: $path was not built" >&2; exit 1; }+ done+ tar --zstd -cf "$out/cua-e2e-build-$name.tar.zst" -C "$REPO" "$@"+ ls -lh "$out/cua-e2e-build-$name.tar.zst"+}++pack() {+ local name="$1" out="$2"+ mkdir -p "$out"+ case "$name" in+ sdk)+ pack_paths sdk "$out" libs/cua/python/src/cua/libcua_sdk.so libs/cua/typescript/node_modules ;;+ cli)+ pack_paths cli "$out" libs/cua/target/debug/cua libs/cua/typescript/dist libs/cua/typescript/browser ;;+ fixtures)+ pack_paths fixtures "$out" libs/cua/target/debug/cua-test-fixtures "$STATIC_SPACESD" ;;+ rust-tests)+ pack_paths rust-tests "$out" tests/e2e/cua-sdk/rust/prebuilt ;;+ images)+ local refs+ refs="$(docker image ls --format '{{.Repository}}:{{.Tag}}' | grep '^cua-e2e-local/' || true)"+ [ "$(wc -w <<<"$refs")" -ge 3 ] || { echo "pack images: expected 3 images, found: $refs" >&2; exit 1; }+ # shellcheck disable=SC2086+ docker save $refs | zstd -T0 -3 -q -o "$out/cua-e2e-build-images.tar.zst"+ echo "$refs"+ ls -lh "$out/cua-e2e-build-images.tar.zst" ;;+ *) echo "unknown archive $name" >&2; exit 2 ;;+ esac+}++prepare() {+ local dir="$1"+ # gVisor registers a docker runtime and restarts the daemon, so it goes+ # first; the archives unpack while it installs.+ "$HERE/ci-setup.sh" --only system &+ local system=$!+ local name+ for name in sdk cli fixtures rust-tests; do+ tar --zstd -xf "$dir/cua-e2e-build-$name.tar.zst" -C "$REPO"+ done+ wait "$system"+ # The images load while the canonical desktop image (the one the docs+ # blocks start through Image.linux()) is pulled with the same resolver the+ # SDK uses, so the first docs block does not wait for the pull.+ local cua="$REPO/libs/cua/target/debug/cua"+ (zstd -dc "$dir/cua-e2e-build-images.tar.zst" | docker load) &+ local load=$!+ (timeout 900 "$cua" --embedded image pull container:ghcr.io/trycua/linux:24.04 ||+ echo "::warning::could not prefetch the canonical linux image; the first docs block pulls it") &+ local pull=$!+ local rc=0+ wait "$load" || rc=1+ wait "$pull" || true+ docker image ls --format '{{.Repository}}:{{.Tag}} {{.Size}}'+ return "$rc"+}++case "${1:-}" in+ pack) pack "${2:?name}" "${3:?dir}" ;;+ prepare) prepare "${2:?dir}" ;;+ *) sed -n '2,/^set -euo/p' "$0" | sed '$d' | sed 's/^# \{0,1\}//' >&2; exit 2 ;;+esacdiff --git a/tests/e2e/cua-sdk/ci-setup.sh b/tests/e2e/cua-sdk/ci-setup.shindex 62cbbe37c1..d71ac16e70 100755--- a/tests/e2e/cua-sdk/ci-setup.sh+++ b/tests/e2e/cua-sdk/ci-setup.sh@@ -9,18 +9,71 @@# two plain images, rootfs form, host arch# --disks the VM disks of linux and plain/ubuntu-server into# ~/.cache/cua-images-e2e, plus QEMU and firmware+#+# With no --only, every phase runs, in the order below (the nightly job, the+# compat workflows and a laptop). The PR workflow runs the phases on separate+# runners and ships their outputs to the test shards:+#+# --only PHASE[,PHASE...]+# system apt packages, gVisor, docker runtimes+# cli debug cua CLI + cua-sdk+# fixtures cua-test-fixtures (debug)+# sdk release cua-sdk+# stage npm ci, stage the release library for Python and Node+# web TypeScript build, wasm tools, wasm browser build+# playwright Playwright's headless Chromium+# images the reference images (needs --images)+# disks the VM disks (needs --disks)+# rust-tests build the Rust half's test binaries into+# tests/e2e/cua-sdk/rust/prebuilt (+ manifest.json), for+# `run.py --rust-prebuilt`set -euo pipefailHERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"REPO="$(cd "$HERE/../../.." && pwd)"-IMAGES=0; DISKS=0-for a in "$@"; do- case "$a" in --images) IMAGES=1 ;; --disks) DISKS=1 ;; *) echo "unknown $a" >&2; exit 2 ;; esac+IMAGES=0; DISKS=0; ONLY=""+while [ $# -gt 0 ]; do+ case "$1" in+ --images) IMAGES=1 ;;+ --disks) DISKS=1 ;;+ --only) ONLY="${2:?--only needs a phase list}"; shift ;;+ *) echo "unknown $1" >&2; exit 2 ;;+ esac+ shiftdonearch="$(uname -m)"; case "$arch" in arm64|aarch64) harch=arm64 ;; *) harch=amd64 ;; esacstep() { echo "::group::$*"; }end() { echo "::endgroup::"; }+# run PHASE: true when this invocation includes the phase.+run() {+ [ -z "$ONLY" ] && return 0+ case ",$ONLY," in *",$1,"*) return 0 ;; *) return 1 ;; esac+}+if [ -n "$ONLY" ]; then+ for p in ${ONLY//,/ }; do+ case "$p" in+ system|cli|fixtures|sdk|stage|web|playwright|images|disks|rust-tests) ;;+ *) echo "unknown phase $p" >&2; exit 2 ;;+ esac+ done+ # `images` and `disks` also need their flag: --only images without --images is a mistake.+ run images && [ "$IMAGES" != 1 ] && { echo "--only images needs --images" >&2; exit 2; }+ run disks && [ "$DISKS" != 1 ] && { echo "--only disks needs --disks" >&2; exit 2; }+fi+# Default (no --only): the phases the flags ask for, as before.+phase() {+ if [ -z "$ONLY" ]; then+ case "$1" in+ images) [ "$IMAGES" = 1 ] ;;+ disks) [ "$DISKS" = 1 ] ;;+ rust-tests) return 1 ;;+ *) return 0 ;;+ esac+ else+ run "$1"+ fi+}-if [ "$(uname -s)" = Linux ]; then+if phase system && [ "$(uname -s)" = Linux ]; thenstep "system packages, gVisor"sudo apt-get update -qqpkgs="genisoimage"@@ -33,34 +86,84 @@ if [ "$(uname -s)" = Linux ]; thenendfi-step "cua SDK, CLI, fixtures"-cd "$REPO/libs/cua"-cargo build --locked -p cua-cli -p cua-sdk-scripts/build-test-fixtures.sh-cargo build --locked --release -p cua-sdk-end+if phase cli || phase fixtures || phase sdk; then+ step "cua SDK, CLI, fixtures"+ cd "$REPO/libs/cua"+ if phase cli; then cargo build --locked -p cua-cli -p cua-sdk; fi+ if phase fixtures; then scripts/build-test-fixtures.sh; fi+ if phase sdk; then cargo build --locked --release -p cua-sdk; fi+ end+fi++# npm ci once per checkout: `stage` and `web` each need node_modules.+npm_ci() {+ cd "$REPO/libs/cua/typescript"+ [ -e node_modules/.package-lock.json ] || npm ci+}++if phase stage; then+ step "npm ci, stage the release library"+ npm_ci+ node ../scripts/stage-uniffi-library.mjs --only=python,node+ end+fi++if phase web; then+ step "TypeScript package + browser (wasm) build"+ npm_ci+ npm run build+ if ! command -v wasm-pack >/dev/null; then+ curl -fsSL https://rustwasm.github.io/wasm-pack/installer/init.sh | sh -s -- -f+ fi+ command -v wasm-bindgen >/dev/null || cargo install --locked wasm-bindgen-cli --version 0.2.126+ npm run build:browser+ end+fi-step "TypeScript package + browser (wasm) build"-cd "$REPO/libs/cua/typescript"-npm ci-node ../scripts/stage-uniffi-library.mjs --only=python,node-npm run build-if ! command -v wasm-pack >/dev/null; then- curl -fsSL https://rustwasm.github.io/wasm-pack/installer/init.sh | sh -s -- -f+if phase playwright; then+ step "Playwright Chromium"+ pw="${CUA_E2E_PLAYWRIGHT_DIR:-$REPO/tests/e2e/cua-sdk/.playwright}"+ mkdir -p "$pw" && cd "$pw"+ [ -f package.json ] || echo '{"type":"module"}' >package.json+ npm i -s playwright@1+ if [ "$(uname -s)" = Linux ]; then npx playwright install --with-deps chromium-headless-shell; else npx playwright install chromium-headless-shell; fi+ endfi-command -v wasm-bindgen >/dev/null || cargo install --locked wasm-bindgen-cli --version 0.2.126-npm run build:browser-end-step "Playwright Chromium"-pw="${CUA_E2E_PLAYWRIGHT_DIR:-$REPO/tests/e2e/cua-sdk/.playwright}"-mkdir -p "$pw" && cd "$pw"-[ -f package.json ] || echo '{"type":"module"}' >package.json-npm i -s playwright@1-if [ "$(uname -s)" = Linux ]; then npx playwright install --with-deps chromium-headless-shell; else npx playwright install chromium-headless-shell; fi-end+if phase rust-tests; then+ step "Rust test binaries"+ cd "$REPO/tests/e2e/cua-sdk/rust"+ out="$REPO/tests/e2e/cua-sdk/rust/prebuilt"+ rm -rf "$out" && mkdir -p "$out"+ # The same build `cargo test` does, minus running it: every test target of+ # the crate, copied next to a manifest that run.py --rust-prebuilt reads.+ cargo test --no-run --message-format=json >"$out/build.json"+ python3 - "$out" <<'PY'+import json, shutil, sys+from pathlib import Path++out = Path(sys.argv[1])+manifest = []+for line in (out / "build.json").read_text().splitlines():+ try:+ msg = json.loads(line)+ except ValueError:+ continue+ exe = msg.get("executable")+ if msg.get("reason") == "compiler-artifact" and exe and msg.get("profile", {}).get("test"):+ name = msg["target"]["name"]+ shutil.copy2(exe, out / name)+ manifest.append({"name": name, "file": name})+if not manifest:+ raise SystemExit("cargo test --no-run produced no test binaries")+(out / "manifest.json").write_text(json.dumps(sorted(manifest, key=lambda m: m["name"]), indent=1) + "\n")+(out / "build.json").unlink()+print("prebuilt:", ", ".join(m["name"] for m in manifest))+PY+ end+fi-if [ "$IMAGES" = 1 ]; then+if phase images; thenstep "reference images (rootfs, $harch)"cd "$REPO/libs/images"linux/build-spacesd-linux.sh "$harch"@@ -70,7 +173,7 @@ if [ "$IMAGES" = 1 ]; thenendfi-if [ "$DISKS" = 1 ]; then+if phase disks; thenstep "reference disks ($harch) -> ~/.cache/cua-images-e2e"cd "$REPO/libs/images"[ -x "linux/dist/$harch/cua-spacesd" ] || linux/build-spacesd-linux.sh "$harch"diff --git a/tests/e2e/cua-sdk/python/conftest.py b/tests/e2e/cua-sdk/python/conftest.pyindex 6509849fe5..4341b9607b 100644--- a/tests/e2e/cua-sdk/python/conftest.py+++ b/tests/e2e/cua-sdk/python/conftest.py@@ -24,6 +24,24 @@ def pytest_configure(config):)+def pytest_collection_modifyitems(config, items):+ """`CUA_E2E_SELECT=<file>` (one node id per line) keeps only those tests:+ run.py --shard gives each CI shard its slice. An id that no longer exists+ fails the run, so a plan can never silently drop a test."""+ chosen = os.environ.get("CUA_E2E_SELECT")+ if not chosen:+ return+ keep = {line.strip() for line in Path(chosen).read_text().splitlines() if line.strip()}+ missing = keep - {item.nodeid for item in items}+ if missing:+ raise pytest.UsageError(+ f"CUA_E2E_SELECT names tests that do not exist: {sorted(missing)[:5]}"+ )+ dropped = [item for item in items if item.nodeid not in keep]+ items[:] = [item for item in items if item.nodeid in keep]+ config.hook.pytest_deselected(items=dropped)++def _mark(item):m = item.get_closest_marker("e2e")if m is None:diff --git a/tests/e2e/cua-sdk/run.py b/tests/e2e/cua-sdk/run.pyindex 7767da0159..fa0abb6b19 100755--- a/tests/e2e/cua-sdk/run.py+++ b/tests/e2e/cua-sdk/run.py@@ -17,6 +17,11 @@ must record at least one result. Known bugs are `xfail`, never skips.After each language, a janitor removes docker containers namedcua-e2e-<run>-* that a crashed test may have left behind.++Sharded CI (`--shard I/N`, see sharding.py): a shard runs only its units and+checks only what it ran (failures, unexpected skips); the per-shard results+are concatenated and `--matrix-only --strict` then applies the full-suite+checks (every expected cell present, docs coverage) to the merged set."""from __future__ import annotations@@ -35,6 +40,9 @@ from pathlib import PathSUITE = Path(__file__).resolve().parentREPO = SUITE.parents[2]+sys.path.insert(0, str(SUITE))+import sharding # noqa: E402+GATES = {"container": {"CUA_E2E_CONTAINER": "1"},"qemu": {"CUA_E2E_QEMU": "1"},@@ -108,39 +116,123 @@ def janitor(run: str) -> None:subprocess.run(["docker", "rm", "-f", *ids], capture_output=True)-def run_langs(langs: list[str], env: dict, timeout: int) -> dict[str, int]:- codes = {}+def py_command() -> list[str]:uv = shutil.which("uv")+ if not uv:+ return [sys.executable, "-m", "pytest", "-q", "-rs"]+ return [+ uv,+ "run",+ "--no-project",+ "--python",+ "3.12",+ "--with",+ "pytest",+ # The browse-the-web scenario is an MCP client.+ "--with",+ "mcp",+ "pytest",+ "-q",+ "-rs",+ ]+++def collect_py(env: dict, timeout: int) -> list[str]:+ """The node ids pytest would run, in collection order."""+ out = subprocess.run(+ [*py_command(), "--collect-only"],+ env=env,+ cwd=SUITE / "python",+ capture_output=True,+ text=True,+ timeout=timeout,+ )+ ids = []+ for line in out.stdout.splitlines():+ if not line.strip():+ break # the node ids end at the first blank line+ if re.match(r"[\w./-]+\.py::", line):+ ids.append(line.strip())+ if out.returncode != 0 or not ids:+ sys.stderr.write(out.stdout[-4000:] + out.stderr[-4000:])+ raise SystemExit(f"cannot collect the Python tests (exit {out.returncode})")+ return ids+++def ts_files() -> list[str]:+ return sorted(p.name for p in (SUITE / "typescript").glob("*.test.mjs"))+++def rust_binaries(prebuilt: Path) -> dict[str, str]:+ """Prebuilt integration-test binaries: name -> file, from the manifest+ ci-setup.sh --only rust-tests writes next to them."""+ manifest = prebuilt / "manifest.json"+ if not manifest.exists():+ raise SystemExit(f"no {manifest}: build the Rust tests with ci-setup.sh --only rust-tests")+ return {m["name"]: m["file"] for m in json.loads(manifest.read_text())}+++def run_langs(+ langs: list[str],+ env: dict,+ timeout: int,+ *,+ select: dict[str, list[str]] | None = None,+ ts_concurrency: int = 1,+ rust_threads: int = 2,+ rust_prebuilt: Path | None = None,+ unit_secs: dict[str, float] | None = None,+) -> dict[str, int]:+ """Runs each language half. `select` (a shard) limits a half to its units;+ without it every test runs, as on a developer machine and in the nightly."""+ codes = {}+ unit_secs = {} if unit_secs is None else unit_secs+ # The binaries run with the Rust half's directory as cwd: a relative+ # --rust-prebuilt (the workflow passes one) must not be taken from there.+ rust_prebuilt = rust_prebuilt.resolve() if rust_prebuilt is not None else Nonefor lang in langs:+ if select is not None and not select.get(lang):+ continue # nothing of this half in this shardif lang == "py":- cmd = (- [- uv,- "run",- "--no-project",- "--python",- "3.12",- "--with",- "pytest",- # The browse-the-web scenario is an MCP client.- "--with",- "mcp",- "pytest",- "-q",- "-rs",- ]- if uv- else [sys.executable, "-m", "pytest", "-q", "-rs"]- )- codes[lang] = sh(cmd, env, SUITE / "python", timeout)+ run_env = env+ if select is not None:+ chosen = Path(env["CUA_E2E_RESULTS"]) / "py-select.txt"+ chosen.write_text("\n".join(select["py"]) + "\n")+ run_env = dict(env, CUA_E2E_SELECT=str(chosen))+ codes[lang] = sh(py_command(), run_env, SUITE / "python", timeout)elif lang == "ts":- tests = sorted(str(p) for p in (SUITE / "typescript").glob("*.test.mjs"))- codes[lang] = sh(- ["node", "--test", "--test-concurrency=1", *tests],- env,- SUITE / "typescript",- timeout,- )+ names = select["ts"] if select is not None else ts_files()+ code = 0+ for name in names:+ t0 = time.monotonic()+ rc = sh(+ [+ "node",+ "--test",+ f"--test-concurrency={ts_concurrency}",+ str(SUITE / "typescript" / name),+ ],+ env,+ SUITE / "typescript",+ timeout,+ )+ unit_secs[f"ts:{name}"] = round(time.monotonic() - t0, 1)+ code = code or rc+ codes[lang] = code+ elif lang == "rust" and rust_prebuilt is not None:+ binaries = rust_binaries(rust_prebuilt)+ code = 0+ for name in select["rust"] if select is not None else sorted(binaries):+ t0 = time.monotonic()+ rc = sh(+ [str(rust_prebuilt / binaries[name]), f"--test-threads={rust_threads}"],+ env,+ SUITE / "rust",+ timeout,+ )+ unit_secs[f"rust:{name}"] = round(time.monotonic() - t0, 1)+ code = code or rc+ codes[lang] = codeelif lang == "rust":codes[lang] = sh([@@ -149,7 +241,7 @@ def run_langs(langs: list[str], env: dict, timeout: int) -> dict[str, int]:"--manifest-path",str(SUITE / "rust" / "Cargo.toml"),"--",- "--test-threads=2",+ f"--test-threads={rust_threads}",],env,SUITE / "rust",@@ -161,6 +253,26 @@ def run_langs(langs: list[str], env: dict, timeout: int) -> dict[str, int]:return codes+def plan_shard(+ langs: list[str], env: dict, timeout: int, spec: str, timings: dict, rust_prebuilt: Path | None+) -> tuple[dict[str, list[str]], int, float]:+ """This shard's units per language, the total unit count and the shard's+ predicted seconds. Every shard computes the same plan."""+ index, count = sharding.parse_shard(spec)+ units: list[sharding.Unit] = []+ if "py" in langs:+ units += [("py", n) for n in collect_py(env, timeout)]+ if "ts" in langs:+ units += [("ts", n) for n in ts_files()]+ if "rust" in langs:+ if rust_prebuilt is None:+ raise SystemExit("--shard needs --rust-prebuilt for the Rust half")+ units += [("rust", n) for n in rust_binaries(rust_prebuilt)]+ mine = sharding.assign(units, count, timings)[index - 1]+ chosen = {lang: [u for lg, u in mine if lg == lang] for lang in ("py", "ts", "rust")}+ return chosen, len(set(units)), sharding.predicted(mine, timings)++def expected(sc: dict, lane: str, lang: str) -> bool:return lang in sc["langs"] and lang in sc.get("only", {}).get(lane, sc["langs"])@@ -240,19 +352,20 @@ def render(out: Path, lanes: list[str] | None) -> tuple[str, list[str]]:return md, problems+def skip_problems(rows: list[dict], lanes: list[str]) -> list[str]:+ return [+ f"unexpected skip in requested lane: {r['scenario']}/{r['lane']}/{r['lang']}: {r['reason']}"+ for r in rows+ if r["lane"] in lanes+ and r["status"] == "skip"+ and not any(a in r.get("reason", "") for a in ALLOWED_SKIPS)+ ]++def strict_problems(out: Path, lanes: list[str], langs: list[str]) -> list[str]:reg = json.loads((SUITE / "scenarios.json").read_text())rows = load(out)- probs = []- for r in rows:- if (- r["lane"] in lanes- and r["status"] == "skip"- and not any(a in r.get("reason", "") for a in ALLOWED_SKIPS)- ):- probs.append(- f"unexpected skip in requested lane: {r['scenario']}/{r['lane']}/{r['lang']}: {r['reason']}"- )+ probs = skip_problems(rows, lanes)seen = {(r["scenario"], r["lane"], r["lang"]) for r in rows}for sc in reg["scenarios"]:for lane in sc["lanes"]:@@ -295,6 +408,65 @@ def docs_coverage(out: Path, lanes: list[str], langs: list[str]) -> list[str]:return problems+def run_shard(+ a: argparse.Namespace, lanes: list[str], langs: list[str], env: dict, out: Path+) -> int:+ """One shard of the suite. Only this shard's results exist here, so the+ checks that need the whole matrix (missing cells, docs coverage) wait for+ the merge; failures and unexpected skips are checked now."""+ timings = sharding.load_timings(a.timings)+ chosen, total, expect = plan_shard(langs, env, a.timeout, a.shard, timings, a.rust_prebuilt)+ index, count = sharding.parse_shard(a.shard)+ print(+ f"\nshard {index}/{count}: "+ + ", ".join(f"{len(chosen[lang])} {lang}" for lang in ("py", "ts", "rust"))+ + f" of {total} units, predicted {expect:.0f}s",+ flush=True,+ )+ unit_secs: dict[str, float] = {}+ t0 = time.monotonic()+ codes = run_langs(+ langs,+ env,+ a.timeout,+ select=chosen,+ ts_concurrency=a.ts_concurrency,+ rust_threads=a.rust_threads,+ rust_prebuilt=a.rust_prebuilt,+ unit_secs=unit_secs,+ )+ wall = time.monotonic() - t0+ rows = load(out)+ problems = [+ f"{r['status']}: {r['scenario']}/{r['lane']}/{r['lang']}: {r['test']}"+ for r in rows+ if r["status"] in ("fail", "xpass")+ ]+ if a.strict:+ problems += skip_problems(rows, lanes)+ (out / "shard.json").write_text(+ json.dumps(+ {+ "shard": index,+ "of": count,+ "total_units": total,+ "units": chosen,+ "predicted_secs": round(expect, 1),+ "wall_secs": round(wall, 1),+ "unit_secs": unit_secs,+ "settings": {"ts_concurrency": a.ts_concurrency, "rust_threads": a.rust_threads},+ },+ indent=1,+ )+ + "\n"+ )+ print(f"\nshard {index}/{count} exit codes: {codes} in {wall:.0f}s (predicted {expect:.0f}s)")+ if problems:+ print("\n".join(["", "PROBLEMS:"] + problems))+ return 1+ return 1 if any(c != 0 for c in codes.values()) else 0++def main() -> int:ap = argparse.ArgumentParser(description=__doc__.split("\n")[0])ap.add_argument(@@ -310,7 +482,39 @@ def main() -> int:ap.add_argument("--timeout", type=int, default=5400, help="per-language timeout (s)")ap.add_argument("--strict", action="store_true")ap.add_argument("--matrix-only", action="store_true")+ ap.add_argument(+ "--shard",+ metavar="I/N",+ help="run only shard I of N (sharding.py); merge the shards' results, then run "+ "`--matrix-only --strict` on the merge for the full-suite checks",+ )+ ap.add_argument(+ "--timings", type=Path, default=sharding.TIMINGS, help="unit weights for --shard"+ )+ ap.add_argument(+ "--rust-prebuilt",+ type=Path,+ help="directory of prebuilt Rust test binaries + manifest.json (ci-setup.sh --only "+ "rust-tests); run instead of `cargo test`",+ )+ ap.add_argument(+ "--ts-concurrency",+ type=int,+ default=int(os.environ.get("CUA_E2E_TS_CONCURRENCY", "1")),+ help="node --test-concurrency (files run one at a time by default)",+ )+ ap.add_argument(+ "--rust-threads",+ type=int,+ default=int(os.environ.get("CUA_E2E_RUST_THREADS", "2")),+ help="--test-threads of the Rust test binaries",+ )a = ap.parse_args()+ # Paths are used from other working directories (the halves run in their+ # own), so fix them against the invocation directory now.+ a.rust_prebuilt = a.rust_prebuilt.resolve() if a.rust_prebuilt else None+ a.timings = a.timings.resolve()+ a.out = a.out.resolve() if a.out else Nonelanes = [x for x in a.lanes.split(",") if x]langs = [x for x in a.langs.split(",") if x]out = (@@ -323,8 +527,17 @@ def main() -> int:env = dict(os.environ, CUA_E2E_RUN=a.run, CUA_E2E_RESULTS=str(out))for lane in lanes:env.update(GATES.get(lane, {}))+ if a.shard:+ return run_shard(a, lanes, langs, env, out)t0 = time.monotonic()- codes = run_langs(langs, env, a.timeout)+ codes = run_langs(+ langs,+ env,+ a.timeout,+ ts_concurrency=a.ts_concurrency,+ rust_threads=a.rust_threads,+ rust_prebuilt=a.rust_prebuilt,+ )print(f"\nexit codes: {codes} in {time.monotonic() - t0:.0f}s")md, problems = render(out, lanes)print(f"\n## cua SDK e2e matrix (run {a.run})\n\n{md}")diff --git a/tests/e2e/cua-sdk/sharding.py b/tests/e2e/cua-sdk/sharding.pynew file mode 100644index 0000000000..c7d43c5b6d--- /dev/null+++ b/tests/e2e/cua-sdk/sharding.py@@ -0,0 +1,182 @@+#!/usr/bin/env python3+"""Time-balanced shards for the cua SDK e2e suite (stdlib only).++The PR workflow runs the suite on N runners. Every shard collects the same+list of units, weighs each one from timings.json and runs the units the plan+gives it, so the shards are disjoint and together cover every unit without any+coordination:++ py one pytest test (node id; weighed by test name)+ ts one tests/e2e/cua-sdk/typescript/*.test.mjs file+ rust one prebuilt integration-test binary of tests/e2e/cua-sdk/rust++A unit missing from timings.json (a new test) gets the language's default+weight, so it still runs on exactly one shard.++The plan is greedy longest-processing-time: heaviest unit first, onto the+shard with the least work so far, ties broken by name. With ~120 units and the+heaviest at about a third of a shard that lands within 1% of even.++Refresh timings.json from the `cua-e2e-shard-*` artifacts of a green run:++ for i in 1 2 3 4; do gh run download RUN -n cua-e2e-shard-$i -D art/$i; done+ python3 tests/e2e/cua-sdk/sharding.py timings art/* >tests/e2e/cua-sdk/timings.json+ python3 tests/e2e/cua-sdk/sharding.py check art/* --shards 4++`check` prints each shard's predicted and measured seconds and the spread, and+fails when the shards together do not cover the plan (a shard missing, a unit+dropped or planned twice).+"""++from __future__ import annotations++import json+import statistics+import sys+from pathlib import Path++TIMINGS = Path(__file__).with_name("timings.json")+LANGS = ("py", "ts", "rust")+Unit = tuple[str, str] # (lang, unit)+++def load_timings(path: Path = TIMINGS) -> dict:+ return json.loads(Path(path).read_text()) if Path(path).exists() else {}+++def timing_key(lang: str, unit: str) -> str:+ """py units are pytest node ids, weighed by the test's own name (what the+ results record): the part after `file.py::`."""+ return unit.split("::", 1)[1] if lang == "py" and "::" in unit else unit+++def weight(timings: dict, lang: str, unit: str) -> float:+ w = timings.get(lang, {}).get(timing_key(lang, unit))+ if w is None:+ w = timings.get("default", {}).get(lang, 1.0)+ return float(w)+++def parse_shard(spec: str) -> tuple[int, int]:+ """`2/4` -> (2, 4), 1-based."""+ try:+ i, n = (int(x) for x in spec.split("/"))+ except ValueError:+ raise SystemExit(f"--shard {spec!r}: want I/N, for example 2/4") from None+ if not 1 <= i <= n:+ raise SystemExit(f"--shard {spec!r}: I must be 1..N")+ return i, n+++def assign(units: list[Unit], n: int, timings: dict) -> list[list[Unit]]:+ """Split units into n shards by weight (longest processing time first)."""+ if n < 1:+ raise ValueError("need at least one shard")+ load = [0.0] * n+ shards: list[list[Unit]] = [[] for _ in range(n)]+ for lang, unit in sorted(set(units), key=lambda u: (-weight(timings, *u), u)):+ i = min(range(n), key=lambda k: (load[k], k))+ shards[i].append((lang, unit))+ load[i] += weight(timings, lang, unit)+ return shards+++def predicted(shard: list[Unit], timings: dict) -> float:+ return sum(weight(timings, *u) for u in shard)+++def spread(values: list[float]) -> float:+ """(max - min) / max, as a percentage."""+ top = max(values, default=0.0)+ return 0.0 if top <= 0 else 100.0 * (top - min(values)) / top+++def _jsonl(path: Path) -> list[dict]:+ if not path.exists():+ return []+ return [json.loads(line) for line in path.read_text().splitlines() if line.strip()]+++def measured(dirs: list[Path]) -> dict:+ """Per-unit seconds from shard result dirs: py from the per-test results,+ ts and rust from the wall time each file or binary took (shard.json)."""+ py: dict[str, list[float]] = {}+ other: dict[str, dict[str, list[float]]] = {"ts": {}, "rust": {}}+ for d in dirs:+ for r in _jsonl(d / "py.jsonl"):+ py.setdefault(r["test"], []).append(float(r.get("secs", 0)))+ meta = d / "shard.json"+ if meta.exists():+ for key, secs in json.loads(meta.read_text()).get("unit_secs", {}).items():+ lang, unit = key.split(":", 1)+ other.setdefault(lang, {}).setdefault(unit, []).append(float(secs))+ out: dict = {"py": {k: round(statistics.median(v), 1) for k, v in sorted(py.items())}}+ for lang in ("ts", "rust"):+ out[lang] = {k: round(statistics.median(v), 1) for k, v in sorted(other[lang].items())}+ return out+++def build_timings(dirs: list[Path]) -> dict:+ t = measured(dirs)+ t["default"] = {+ lang: round(statistics.median(t[lang].values()), 1) if t[lang] else 1.0 for lang in LANGS+ }+ return t+++def main(argv: list[str]) -> int:+ if len(argv) < 2 or argv[0] not in ("timings", "check"):+ print(__doc__)+ return 2+ cmd, rest = argv[0], argv[1:]+ n = 0+ if "--shards" in rest:+ k = rest.index("--shards")+ n = int(rest[k + 1])+ del rest[k : k + 2]+ dirs = [Path(a) for a in rest]+ if cmd == "timings":+ print(json.dumps(build_timings(dirs), indent=1, sort_keys=True))+ return 0+ metas = sorted(+ (json.loads((d / "shard.json").read_text()) for d in dirs), key=lambda m: m["shard"]+ )+ for m in metas:+ settings = ", ".join(f"{k}={v}" for k, v in m.get("settings", {}).items())+ print(+ f"shard {m['shard']}: predicted {m['predicted_secs']:6.1f}s, "+ f"measured {m['wall_secs']:6.1f}s ({settings})"+ )+ print(+ f"spread: predicted {spread([m['predicted_secs'] for m in metas]):.1f}%, "+ f"measured {spread([m['wall_secs'] for m in metas]):.1f}%"+ )+ problems = coverage_problems(metas, n)+ if problems:+ print("\n".join(["", "PROBLEMS:", *problems]))+ return 1+ return 0+++def coverage_problems(metas: list[dict], n: int = 0) -> list[str]:+ """The shards together ran the plan: one shard of each index, one plan,+ every unit on exactly one shard."""+ problems = []+ count = n or (metas[0]["of"] if metas else 0)+ if sorted(m["shard"] for m in metas) != list(range(1, count + 1)):+ problems.append(f"want shards 1..{count}, found {sorted(m['shard'] for m in metas)}")+ if {m["of"] for m in metas} - {count}:+ problems.append(f"shards disagree on the shard count: {sorted({m['of'] for m in metas})}")+ totals = {m["total_units"] for m in metas}+ if len(totals) != 1:+ problems.append(f"shards planned different suites: {sorted(totals)} units")+ ran = [(lang, u) for m in metas for lang, units in m["units"].items() for u in units]+ if len(set(ran)) != len(ran):+ problems.append("a unit is planned on more than one shard")+ if totals and len(ran) != next(iter(totals)):+ problems.append(f"{len(ran)} units planned across shards, the suite has {sorted(totals)}")+ return problems+++if __name__ == "__main__":+ sys.exit(main(sys.argv[1:]))diff --git a/tests/e2e/cua-sdk/test_sharding.py b/tests/e2e/cua-sdk/test_sharding.pynew file mode 100644index 0000000000..d50c93b031--- /dev/null+++ b/tests/e2e/cua-sdk/test_sharding.py@@ -0,0 +1,225 @@+"""Shard planning for the cua SDK e2e suite (stdlib only).++python3 -m unittest tests/e2e/cua-sdk/test_sharding.py+"""++from __future__ import annotations++import importlib.util+import json+import tempfile+import unittest+from pathlib import Path++HERE = Path(__file__).resolve().parent+++def _load(name: str):+ spec = importlib.util.spec_from_file_location(f"e2e_{name}", HERE / f"{name}.py")+ module = importlib.util.module_from_spec(spec)+ import sys++ sys.path.insert(0, str(HERE))+ spec.loader.exec_module(module)+ return module+++sharding = _load("sharding")+run = _load("run")+SHARDS = 4 # the E2E_SHARDS of .github/workflows/e2e-cua-sdk.yml+++def known_units(timings: dict) -> list[tuple[str, str]]:+ units = [("py", f"test_x.py::{name}") for name in timings["py"]]+ units += [("ts", name) for name in timings["ts"]]+ units += [("rust", name) for name in timings["rust"]]+ return units+++class Assign(unittest.TestCase):+ timings = sharding.load_timings()++ def test_every_unit_runs_on_exactly_one_shard(self) -> None:+ units = known_units(self.timings) + [("py", "test_new.py::test_added_later")]+ shards = sharding.assign(units, SHARDS, self.timings)+ flat = [u for shard in shards for u in shard]+ self.assertEqual(sorted(flat), sorted(set(units)))+ self.assertEqual(len(flat), len(set(flat)))++ def test_the_plan_is_deterministic_whatever_the_collection_order(self) -> None:+ units = known_units(self.timings)+ self.assertEqual(+ sharding.assign(units, SHARDS, self.timings),+ sharding.assign(list(reversed(units)), SHARDS, self.timings),+ )++ def test_shards_finish_within_ten_percent(self) -> None:+ shards = sharding.assign(known_units(self.timings), SHARDS, self.timings)+ loads = [sharding.predicted(s, self.timings) for s in shards]+ self.assertLess(sharding.spread(loads), 10.0, loads)++ def test_a_unit_without_a_timing_gets_the_default_weight(self) -> None:+ self.assertEqual(+ sharding.weight(self.timings, "py", "test_new.py::test_added_later"),+ self.timings["default"]["py"],+ )+ self.assertEqual(+ sharding.weight(self.timings, "ts", "new.test.mjs"), self.timings["default"]["ts"]+ )++ def test_py_units_are_weighed_by_test_name(self) -> None:+ name = next(k for k, v in self.timings["py"].items() if v > 10)+ self.assertEqual(+ sharding.weight(self.timings, "py", f"python/test_a.py::{name}"),+ self.timings["py"][name],+ )++ def test_more_shards_than_units_leaves_some_empty(self) -> None:+ shards = sharding.assign([("ts", "a.test.mjs")], 3, {})+ self.assertEqual(sum(1 for s in shards if s), 1)++ def test_shard_spec(self) -> None:+ self.assertEqual(sharding.parse_shard("2/4"), (2, 4))+ for bad in ("0/4", "5/4", "x", "1/0", "2"):+ with self.assertRaises(SystemExit):+ sharding.parse_shard(bad)+++class Measured(unittest.TestCase):+ def test_timings_come_from_results_and_shard_files(self) -> None:+ with tempfile.TemporaryDirectory() as tmp:+ a, b = Path(tmp, "a"), Path(tmp, "b")+ for d, secs in ((a, 4.0), (b, 6.0)):+ d.mkdir()+ (d / "py.jsonl").write_text(json.dumps({"test": "test_one", "secs": secs}) + "\n")+ (d / "shard.json").write_text(+ json.dumps({"unit_secs": {"ts:core.test.mjs": secs * 2}})+ )+ t = sharding.build_timings([a, b])+ self.assertEqual(t["py"], {"test_one": 5.0})+ self.assertEqual(t["ts"], {"core.test.mjs": 10.0})+ self.assertEqual(t["rust"], {})+ self.assertEqual(t["default"]["py"], 5.0)+++class Workflow(unittest.TestCase):+ def test_the_shard_matrix_matches_the_shard_count(self) -> None:+ import re++ text = (HERE.parents[2] / ".github" / "workflows" / "e2e-cua-sdk.yml").read_text()+ count = int(re.search(r'^ E2E_SHARDS: "(\d+)"', text, re.M).group(1))+ matrix = re.search(r"^\s+shard: \[([\d, ]+)\]", text, re.M).group(1)+ self.assertEqual([int(x) for x in matrix.split(",")], list(range(1, count + 1)))+ self.assertEqual(count, SHARDS)+++class Coverage(unittest.TestCase):+ @staticmethod+ def meta(shard: int, units: dict, total: int = 4, of: int = 2) -> dict:+ return {"shard": shard, "of": of, "total_units": total, "units": units}++ def test_complete_shards_pass(self) -> None:+ metas = [+ self.meta(1, {"py": ["a", "b"], "ts": [], "rust": []}),+ self.meta(2, {"py": ["c"], "ts": ["x"], "rust": []}),+ ]+ self.assertEqual(sharding.coverage_problems(metas, 2), [])++ def test_a_missing_shard_is_reported(self) -> None:+ metas = [self.meta(1, {"py": ["a", "b", "c", "d"], "ts": [], "rust": []})]+ self.assertTrue(sharding.coverage_problems(metas, 2))++ def test_a_dropped_or_doubled_unit_is_reported(self) -> None:+ dropped = [+ self.meta(1, {"py": ["a", "b"], "ts": [], "rust": []}),+ self.meta(2, {"py": ["c"], "ts": [], "rust": []}),+ ]+ doubled = [+ self.meta(1, {"py": ["a", "b"], "ts": [], "rust": []}),+ self.meta(2, {"py": ["b", "c"], "ts": [], "rust": []}),+ ]+ self.assertTrue(sharding.coverage_problems(dropped, 2))+ self.assertTrue(sharding.coverage_problems(doubled, 2))+++class RunnerChecks(unittest.TestCase):+ def test_a_shard_flags_unexpected_skips_but_not_allowed_ones(self) -> None:+ rows = [+ {+ "scenario": "s",+ "lane": "container",+ "lang": "py",+ "status": "skip",+ "reason": "docker is not on PATH",+ },+ {+ "scenario": "s",+ "lane": "hermetic",+ "lang": "ts",+ "status": "skip",+ "reason": "playwright is not installed",+ },+ {+ "scenario": "s",+ "lane": "qemu",+ "lang": "py",+ "status": "skip",+ "reason": "not requested",+ },+ ]+ problems = run.skip_problems(rows, ["hermetic", "container"])+ self.assertEqual(len(problems), 1)+ self.assertIn("docker is not on PATH", problems[0])++ def test_a_relative_rust_prebuilt_runs_from_the_invocation_directory(self) -> None:+ """The workflow passes `--rust-prebuilt tests/e2e/cua-sdk/rust/prebuilt` from the+ repository root; the binaries run with the Rust half's directory as cwd."""+ import os+ import subprocess+ import sys++ rel = "tests/e2e/cua-sdk/rust/prebuilt"+ with tempfile.TemporaryDirectory() as tmp:+ prebuilt = Path(tmp, rel)+ prebuilt.mkdir(parents=True)+ fake = prebuilt / "guides"+ fake.write_text('#!/bin/sh\necho "ran $0 $@" >"$CUA_E2E_RESULTS/ran.txt"\n')+ fake.chmod(0o755)+ (prebuilt / "manifest.json").write_text(+ json.dumps([{"name": "guides", "file": "guides"}])+ )+ proc = subprocess.run(+ [+ sys.executable,+ str(HERE / "run.py"),+ "--lanes",+ "hermetic",+ "--langs",+ "rust",+ "--shard",+ "1/1",+ "--rust-prebuilt",+ rel,+ "--out",+ "out",+ ],+ cwd=tmp,+ env=dict(os.environ, CUA_E2E_RUN="gh123456-1"),+ capture_output=True,+ text=True,+ )+ self.assertEqual(proc.returncode, 0, proc.stdout + proc.stderr)+ ran = Path(tmp, "out", "ran.txt").read_text()+ self.assertTrue(ran.strip().endswith(f"{rel}/guides --test-threads=2"), ran)+ self.assertNotIn("rust/tests/e2e", ran)++ def test_prebuilt_rust_binaries_are_listed_by_the_manifest(self) -> None:+ with tempfile.TemporaryDirectory() as tmp:+ Path(tmp, "manifest.json").write_text(json.dumps([{"name": "core", "file": "core"}]))+ self.assertEqual(run.rust_binaries(Path(tmp)), {"core": "core"})+ with self.assertRaises(SystemExit):+ run.rust_binaries(Path(tmp, "missing"))+++if __name__ == "__main__":+ unittest.main()diff --git a/tests/e2e/cua-sdk/timings.json b/tests/e2e/cua-sdk/timings.jsonnew file mode 100644index 0000000000..c42b62aa8e--- /dev/null+++ b/tests/e2e/cua-sdk/timings.json@@ -0,0 +1,132 @@+{+ "default": {+ "py": 3.0,+ "rust": 10.0,+ "ts": 20.0+ },+ "py": {+ "test_a_pool_name_is_cloud_ended": 0.0,+ "test_agnostic_cloud_ended_and_direct": 2.5,+ "test_agnostic_container_ssh_only": 4.6,+ "test_agnostic_container_vnc_only": 5.6,+ "test_agnostic_qemu_ssh_only": 0.0,+ "test_browse_local_sandbox": 12.3,+ "test_browse_mcp_surface": 0.8,+ "test_cli_build_push_boot_vm": 0.0,+ "test_cli_build_push_run_container": 16.3,+ "test_container_block[cua-sdk/guides/desktop#desktop-python]": 52.1,+ "test_container_block[cua-sdk/guides/desktop#desktop-viewer-url]": 2.8,+ "test_container_block[cua-sdk/guides/desktop#terminal]": 3.0,+ "test_container_block[cua-sdk/guides/desktop#typed-driver]": 3.0,+ "test_container_block[cua-sdk/guides/images#start-dev]": 48.2,+ "test_container_block[cua-sdk/guides/lifecycle#ephemeral-or-persistent]": 4.6,+ "test_container_block[cua-sdk/guides/lifecycle#inspect]": 4.4,+ "test_container_block[cua-sdk/guides/lifecycle#reconnect-list-delete]": 4.5,+ "test_container_block[cua-sdk/guides/own-harness#harness-python]": 5.8,+ "test_container_block[cua-sdk/guides/services#call]": 2.2,+ "test_container_block[cua-sdk/guides/services#forward-python]": 2.0,+ "test_container_block[cua-sdk/guides/services#mcp-call]": 8.9,+ "test_container_block[cua-sdk/guides/services#mcp-local]": 7.8,+ "test_container_block[cua-sdk/guides/services#mcp-server]": 0.0,+ "test_container_block[cua-sdk/guides/services#public-url]": 1.9,+ "test_container_block[cua-sdk/guides/services#sidecar-by-name]": 4.0,+ "test_container_block[cua-sdk/guides/services#sidecar-local]": 3.3,+ "test_container_block[cua-sdk/guides/services#start-local]": 5.0,+ "test_container_block[cua-sdk/guides/test-in-ci#ci-overlay-python]": 1.8,+ "test_container_block[cua-sdk/index#local-ephemeral]": 2.1,+ "test_container_block[cua-sdk/quickstart#local-server]": 2.2,+ "test_container_block[cua-sdk/reference/python/sandbox#sandbox-create-py]": 2.1,+ "test_container_block[cua-sdk/reference/sandbox/index#sandboxes-create-py]": 2.4,+ "test_container_block[spaces/guides/create-a-space-with-the-sdk#first-space]": 4.5,+ "test_container_ts_block[cua-sdk/guides/desktop#desktop-typescript]": 6.7,+ "test_container_ts_block[cua-sdk/guides/own-harness#harness-typescript]": 7.2,+ "test_container_ts_block[cua-sdk/guides/services#mcp-typescript]": 9.4,+ "test_container_ts_block[cua-sdk/guides/services#typescript]": 3.5,+ "test_container_ts_block[cua-sdk/index#local-ephemeral-ts]": 3.6,+ "test_container_ts_block[cua-sdk/quickstart#typescript]": 3.3,+ "test_container_ts_block[cua-sdk/reference/sandbox/index#sandboxes-create-ts]": 2.9,+ "test_container_ts_block[spaces/guides/create-a-space-with-the-sdk#first-space-ts]": 6.0,+ "test_contrib_block[cua-sdk/guides/local-runtimes#contrib-python]": 0.0,+ "test_daemon_vs_embedded_local_container": 8.0,+ "test_daemon_vs_embedded_mock": 2.5,+ "test_direct_connect_docker": 2.0,+ "test_direct_connect_mock": 0.7,+ "test_docs_block[cua-sdk/guides/images#canonical]": 1.8,+ "test_docs_block[cua-sdk/guides/images#chain]": 0.2,+ "test_docs_block[cua-sdk/guides/images#fork]": 0.2,+ "test_docs_block[cua-sdk/guides/images#inspect]": 0.2,+ "test_docs_block[cua-sdk/guides/images#install-packages]": 0.2,+ "test_docs_block[cua-sdk/guides/images#registry]": 0.2,+ "test_docs_block[cua-sdk/guides/own-harness#harness-connect-python]": 2.0,+ "test_docs_block[cua-sdk/guides/run-a-coding-agent#harness-ids]": 0.1,+ "test_docs_block[cua-sdk/guides/services#mcp-server]": 0.0,+ "test_docs_block[cua-sdk/guides/services#run-with-key-file]": 0.2,+ "test_docs_block[cua-sdk/guides/services#run-with-secrets]": 0.2,+ "test_docs_block[cua-sdk/index#direct-connect]": 2.0,+ "test_docs_block[cua-sdk/reference/cua#cua-embedded-py]": 0.2,+ "test_docs_block[cua-sdk/reference/cua#local-doctor-py]": 0.2,+ "test_docs_block[cua-sdk/reference/errors#cua-error-py]": 1.9,+ "test_docs_block[cua-sdk/reference/image#canonical-image-py]": 0.1,+ "test_docs_block[cua-sdk/reference/image#image-alias-py]": 0.1,+ "test_docs_block[cua-sdk/reference/image#normalize-image-py]": 0.1,+ "test_docs_block[cua-sdk/reference/python/index#image-py]": 0.2,+ "test_docs_block[cua-sdk/reference/python/interfaces#files-write-text-py]": 2.1,+ "test_docs_block[cua-sdk/reference/python/interfaces#mouse-click-py]": 2.0,+ "test_docs_block[cua-sdk/reference/python/sandbox#image-from-dict-py]": 0.2,+ "test_docs_block[cua-sdk/reference/python/sandbox#image-py]": 0.2,+ "test_docs_block[cua-sdk/reference/python/sandbox#registry-secret-py]": 0.2,+ "test_docs_block[cua-sdk/reference/python/sandbox#sandbox-connect-py]": 2.1,+ "test_docs_block[cua-sdk/reference/sandbox/index#sandboxes-connect-url-py]": 2.0,+ "test_docs_block[cua-sdk/reference/sandbox/lifecycle#parse-sandbox-ref-py]": 0.1,+ "test_docs_block[cua-sdk/reference/sandbox/lifecycle#qualify-sandbox-ref-py]": 0.1,+ "test_docs_block[cua-sdk/reference/spaces/index#spaces-add-py]": 1.8,+ "test_docs_block[cua-sdk/reference/spacesd/computer#spacesd-client-screenshot-py]": 1.8,+ "test_docs_block[cua-sdk/reference/spacesd/files#spacesd-client-upload-py]": 1.9,+ "test_docs_block[cua-sdk/reference/spacesd/shell#spacesd-client-run-py]": 1.9,+ "test_docs_block[cua-sdk/reference/spacesd/shell#spacesd-client-spawn-py]": 1.9,+ "test_docs_block[spaces/guides/add-a-space#add-direct]": 1.8,+ "test_docs_ts_block[cua-sdk/guides/images#typescript]": 7.0,+ "test_docs_ts_block[cua-sdk/guides/own-harness#harness-connect-typescript]": 3.5,+ "test_docs_ts_block[cua-sdk/guides/services#sidecar-typescript]": 1.7,+ "test_docs_ts_block[cua-sdk/index#direct-connect-ts]": 3.4,+ "test_docs_ts_block[cua-sdk/reference/cua#cua-embedded-ts]": 1.8,+ "test_docs_ts_block[cua-sdk/reference/errors#cua-error-ts]": 3.4,+ "test_docs_ts_block[cua-sdk/reference/sandbox/index#sandboxes-connect-url-ts]": 3.6,+ "test_docs_ts_block[cua-sdk/reference/spacesd/shell#spacesd-client-run-ts]": 3.6,+ "test_docs_ts_block[spaces/guides/add-a-space#add-direct-ts]": 3.5,+ "test_fork_many_from_a_memory_checkpoint": 0.0,+ "test_guides_are_tagged": 0.0,+ "test_local_container_desktop": 2.5,+ "test_local_false_is_cloud_ended": 0.0,+ "test_local_qemu_env_token_reaches_guest": 0.0,+ "test_local_qemu_fork_fan_out_of_a_pinned_checkpoint": 0.0,+ "test_local_qemu_lifecycle_probe_forward": 0.0,+ "test_on_cloud_is_cloud_ended": 0.0,+ "test_parity_cloud_ended_and_local_vm_refuses_layers": 0.1,+ "test_parity_sidecars_by_name_through_the_daemon": 20.3,+ "test_run_omarchy_local": 2.5,+ "test_sdk_local_layers_apply": 7.1,+ "test_share_a_link": 0.9,+ "test_tcp_probe_implies_listening": 1.0,+ "test_typed_click_reaches_gtk": 1.3,+ "test_unified_api_cloud_ended_and_daemon_public_url": 15.6,+ "test_unified_api_plain_image_gvisor": 17.3,+ "test_viewer_local": 0.0+ },+ "rust": {+ "core": 34.0,+ "cua-sdk-e2e": 0.1,+ "fork_batch": 0.1,+ "guides": 27.0,+ "local_vm": 0.1,+ "share": 1.0,+ "unified": 3.0+ },+ "ts": {+ "browser.test.mjs": 3.0,+ "core.test.mjs": 35.0,+ "guides.test.mjs": 31.0,+ "share.test.mjs": 2.0,+ "unified.test.mjs": 31.0+ }+}\ No newline at end of file