Skip to content

Dodaios is an open-source agent OS for your desktop: it runs your coding agents, browser and apps by voice or chat, on your own machine. Formerly Personal Jarvis.

ci: shard the Windows suite by its own timings (#568)

Ruben Lütkeb98f0e82fae1
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index aa52b3b149..022c6b72c2 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -225,18 +225,33 @@ jobs:
path: test_durations.json
key: test-durations-never-exact
restore-keys: test-durations-
+ # Windows runs the same files at other speeds: sharded by the Linux
+ # history, one Windows shard took 12 minutes while the others took 7 to
+ # 8, and the gate waits for the slowest. It has a history of its own,
+ # in a cache of its own so the Linux one keeps its key and version.
+ - name: Restore the Windows duration history once
+ if: steps.lanes.outputs.python == 'true'
+ uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
+ with:
+ path: test_durations_windows.json
+ key: test-durations-windows-never-exact
+ restore-keys: test-durations-windows-
- name: Ensure a duration snapshot exists
id: snapshot
if: steps.lanes.outputs.python == 'true'
run: |
test -f test_durations.json || echo '{}' > test_durations.json
+ # Until Windows has a history it shards by the Linux one, as before.
+ test -s test_durations_windows.json || cp test_durations.json test_durations_windows.json
echo "name=test-duration-snapshot-${GITHUB_RUN_ATTEMPT}" >> "$GITHUB_OUTPUT"
- name: Share duration snapshot with every shard
if: steps.lanes.outputs.python == 'true'
uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
with:
name: ${{ steps.snapshot.outputs.name }}
- path: test_durations.json
+ path: |
+ test_durations.json
+ test_durations_windows.json
if-no-files-found: error
retention-days: 7
@@ -524,8 +539,10 @@ jobs:
total=$(python -c "import json,os; print(len(json.loads(os.environ['SHARDS'])))")
python scripts/ci/run_tests_parallel.py tests \
--shard "$SHARD/$total" \
+ --durations test_durations_windows.json \
--report "out/test-report-windows-$SHARD.json" \
- --junit-out "out/junit-windows-$SHARD.xml"
+ --junit-out "out/junit-windows-$SHARD.xml" \
+ --durations-out "out/durations-windows-$SHARD.json"
- name: Ratchet (a failure not in the baseline blocks)
env:
SHARD: ${{ matrix.shard }}
@@ -610,18 +627,51 @@ jobs:
- name: Flaky files and failures (report only)
if: ${{ !cancelled() }}
run: python scripts/ci/flaky_report.py summary reports
- # The nightly run refreshes the cache: a push the merge queue already
- # proved runs no tests, and only a run on main may write a cache every
- # branch can read.
- - name: Merge the duration cache (main only)
- if: github.ref == 'refs/heads/main' && (github.event_name == 'push' || github.event_name == 'schedule')
- run: python scripts/ci/ratchet_tests.py durations --out test_durations.json reports
- - name: Save the duration cache (main only)
- if: github.ref == 'refs/heads/main' && (github.event_name == 'push' || github.event_name == 'schedule')
+
+ # Shard timings for the next runs, one history per OS. Only a run on main
+ # may write a cache every branch can read: the nightly run, a manual run on
+ # main and a push that bypassed the queue (a push the merge queue already
+ # proved runs no tests). Only complete, green shards feed it: a history
+ # with holes would unbalance every run after it.
+ duration-cache:
+ name: duration cache (main only)
+ needs: [detect, tests-linux, tests-windows]
+ if: needs.detect.outputs.python == 'true' && github.ref == 'refs/heads/main'
+ runs-on: ubuntu-latest
+ timeout-minutes: 10
+ steps:
+ - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+ with:
+ persist-credentials: false
+ - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
+ with:
+ python-version: "3.11"
+ - name: Download the Linux shard reports
+ uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
+ with:
+ pattern: tests-linux-*
+ path: reports/linux
+ merge-multiple: true
+ - name: Download the Windows shard reports
+ uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
+ with:
+ pattern: tests-windows-*
+ path: reports/windows
+ merge-multiple: true
+ - name: Merge the shard timings
+ run: |
+ python scripts/ci/ratchet_tests.py durations --out test_durations.json reports/linux
+ python scripts/ci/ratchet_tests.py durations --out test_durations_windows.json reports/windows
+ - name: Save the Linux duration cache
uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
with:
path: test_durations.json
key: test-durations-${{ github.run_id }}
+ - name: Save the Windows duration cache
+ uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
+ with:
+ path: test_durations_windows.json
+ key: test-durations-windows-${{ github.run_id }}
# Three test shards and the build side by side: `npm test` alone took seven
# minutes in one process and was the longest Linux leg of a run.
@@ -1112,6 +1162,7 @@ jobs:
- tests-windows
- tests-macos
- test-report
+ - duration-cache
- frontend
- jarvisctl
- deps
diff --git a/docs/ci-pipeline.md b/docs/ci-pipeline.md
index 7c36b360d4..01fea62ea0 100644
--- a/docs/ci-pipeline.md
+++ b/docs/ci-pipeline.md
@@ -68,7 +68,8 @@ request, merge queue, and again on the push of the same commit).
| `tests linux 1..6` | The whole suite in six shards. Batches of files run in fresh processes with a wall-clock budget; a failed batch is re-run file by file, a failed file once more (a pass there is reported as flaky). | `scripts/ci/run_tests_parallel.py` |
| `tests windows 1..6` | The whole suite in six shards on every tier, pull requests included, so a Windows-only failure blocks the pull request instead of ejecting it from the merge queue. | `scripts/ci/run_tests_parallel.py` |
| `tests macos 1..3` | Nightly, release and manual runs with macOS only. | — |
-| `test report + floor` | Proves the six Linux shards cover every discovered file exactly once, enforces the min-passed floor. The nightly run on main (and a push that bypassed the queue) refreshes the duration cache. Detection freezes one shared duration snapshot for all shards, including partial reruns. | `scripts/ci/ratchet_tests.py` |
+| `test report + floor` | Proves the six Linux shards cover every discovered file exactly once, enforces the min-passed floor. Detection freezes one shared duration snapshot for all shards, including partial reruns. | `scripts/ci/ratchet_tests.py` |
+| `duration cache` | Runs on main only (nightly, a manual run, a push that bypassed the queue) and only after every Linux and Windows shard succeeded: merges the shard timings into one history per OS. Windows shards by its own history; until one exists it uses the Linux one. | `scripts/ci/ratchet_tests.py durations` |
| Lanes | `frontend` (three test shards and the build, side by side), `jarvisctl`, `deps`, `realtime` (+ slim container), `updater` (in-app update, native handover, restart helper), `agent runtimes` (+ slim container + the real Hermes and OpenClaw at their pinned version; see [agent-runtimes.md](agent-runtimes.md#ci-and-release-gates)), `dragdrop`, `browser`, `macOS desktop`, `installer smoke` — each only when its paths change, each as wide as the tier: Linux + Windows on a pull request, + macOS in the merge queue, + Intel and ARM in the wide matrix. | `scripts/ci/plan_run.py` |
| `release qualification` | Tag CI requires a full run with macOS and the browser-auth E2E evidence gate: a plugin labeled verified needs a completed journey, every completed journey ships as verified, and every other plugin ships labeled preview. The step summary counts both. Ordinary branch/PR/nightly CI skips this release-only job. | `scripts/ci/check_plugin_auth_contract.py --require-e2e-pass` |
| `CI gate` | Aggregates every job. **The only required check**, on the pull request and again on its merge-queue group. A missing or skipped selected lane fails. Unselected lanes may skip; nightly is strict except event-specific jobs. | `scripts/ci/required_results.py` |
diff --git a/tests/unit/ci/test_pipeline_audit.py b/tests/unit/ci/test_pipeline_audit.py
index 786c43925a..eaafca8f4c 100644
--- a/tests/unit/ci/test_pipeline_audit.py
+++ b/tests/unit/ci/test_pipeline_audit.py
@@ -454,12 +454,48 @@ def test_a_red_nightly_reaches_someone_and_only_a_nightly_on_main_can_write_issu
assert {"os": "windows-latest", "runtime": "hermes"} in only_nightly and len(only_nightly) == 3
-def test_the_duration_cache_is_refreshed_by_the_nightly_run_on_main():
- steps = workflows()["ci.yml"]["jobs"]["test-report"]["steps"]
- for name in ("Merge the duration cache (main only)", "Save the duration cache (main only)"):
- condition = next(s for s in steps if s.get("name") == name)["if"]
- assert "github.ref == 'refs/heads/main'" in condition
- assert "github.event_name == 'schedule'" in condition
+def test_only_a_complete_green_run_on_main_writes_the_duration_caches():
+ ci = workflows()["ci.yml"]["jobs"]
+ job = ci["duration-cache"]
+ # Any run on main (nightly, manual, a push that bypassed the queue), and
+ # only after every Linux and Windows shard succeeded: no `!cancelled()`.
+ assert job["if"] == "needs.detect.outputs.python == 'true' && github.ref == 'refs/heads/main'"
+ assert set(job["needs"]) == {"detect", "tests-linux", "tests-windows"}
+ saves = {
+ s["with"]["path"]: s["with"]["key"]
+ for s in job["steps"]
+ if "cache/save@" in s.get("uses", "")
+ }
+ assert saves == {
+ "test_durations.json": "test-durations-${{ github.run_id }}",
+ "test_durations_windows.json": "test-durations-windows-${{ github.run_id }}",
+ }
+ assert not any("cache/save@" in s.get("uses", "") for s in ci["test-report"]["steps"])
+ assert "duration-cache" in ci["gate"]["needs"]
+
+
+def test_windows_shards_by_its_own_history_and_falls_back_to_the_linux_one():
+ ci = workflows()["ci.yml"]["jobs"]
+ detect = ci["detect"]["steps"]
+ restores = {
+ s["with"]["path"]: s["with"]["restore-keys"]
+ for s in detect
+ if s.get("uses", "").startswith("actions/cache/restore@")
+ }
+ # Two caches with two keys: the Linux history keeps its key and version.
+ assert restores == {
+ "test_durations.json": "test-durations-",
+ "test_durations_windows.json": "test-durations-windows-",
+ }
+ ensure = next(s for s in detect if s.get("id") == "snapshot")["run"]
+ assert "test -s test_durations_windows.json || cp test_durations.json" in ensure
+ shared = next(s for s in detect if s.get("name") == "Share duration snapshot with every shard")
+ assert shared["with"]["path"].split() == ["test_durations.json", "test_durations_windows.json"]
+ windows = next(s for s in ci["tests-windows"]["steps"] if s.get("name") == "Run shard")["run"]
+ assert "--durations test_durations_windows.json" in windows
+ assert '--durations-out "out/durations-windows-$SHARD.json"' in windows
+ linux = next(s for s in ci["tests-linux"]["steps"] if s.get("name") == "Run shard")["run"]
+ assert "test_durations_windows" not in linux
@pytest.mark.parametrize("payload", [{}, {"failed_ids": [], "counts": {}, "files": 0}])