Dodaios is an open-source agent OS for your desktop: it runs your coding agents, browser and apps by voice or chat, on your own machine. Formerly Personal Jarvis.
Python agent-orchestrationai-agentai-coding-assistantclaude-codecoding-agentcomputer-usedesktop-applocal-firstmcppersonal-jarvisvoice-assistantvoice-coding
ci: spend the five macOS slots once per change and throttle the nightly (#596)
Ruben Lütkee744262c3c22
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.ymlindex b3757762c9..a928336f3a 100644--- a/.github/workflows/ci.yml+++ b/.github/workflows/ci.yml@@ -38,12 +38,12 @@# times - pull request, merge queue, and again on the push that landed the# very same commit.# pr pull request: the whole Linux + Windows suite and every affected-# lane on Linux, Windows and macOS. The macOS legs here are the-# short ones (one to four minutes): a macOS failure shows on the-# pull request, and they end before the Windows shards do.-# queue merge-queue group: everything `pr` runs plus the long macOS legs-# (real runtimes, real browser). Nothing macOS-specific that a lane-# tests lands unproven; the groups share the five macOS slots.+# lane on Linux and Windows. No macOS: five macOS slots serve+# every run, and macOS legs on each push made pull requests wait.+# queue merge-queue group: everything `pr` runs plus every macOS leg of+# the affected lanes (real runtimes and real browser included).+# Nothing macOS-specific that a lane tests lands unproven, and the+# five macOS slots are spent once per change.# full nightly, release tags, manual full dispatch: Intel macOS, the ARM# runners, every interpreter, the macOS test shards, and the# Hermes install on Windows (14-23 minutes). A red nightly opens an@@ -64,7 +64,11 @@ on:push:branches: [main]schedule:- - cron: "17 2 * * *" # nightly strict full run, macOS included+ # Nightly strict full run, macOS included. GitHub starts this+ # repository's schedules about seven hours late (measured over a week:+ # the 02:17 slot ran between 08:30 and 09:30 UTC, in the middle of the+ # working morning). 18:17 aims at the small hours.+ - cron: "17 18 * * *"workflow_dispatch:inputs:full:@@ -139,6 +143,7 @@ jobs:e2e_matrix: ${{ steps.mode.outputs.e2e_matrix }}browser_os: ${{ steps.mode.outputs.browser_os }}desktop_legs: ${{ steps.mode.outputs.desktop_legs }}+ matrix_parallel: ${{ steps.mode.outputs.matrix_parallel }}duration_snapshot: ${{ steps.snapshot.outputs.name }}release: ${{ github.ref_type == 'tag' }}steps:@@ -560,6 +565,7 @@ jobs:if: needs.detect.outputs.macos == 'true' && needs.detect.outputs.python == 'true'strategy:fail-fast: false+ max-parallel: ${{ fromJSON(needs.detect.outputs.matrix_parallel) }}matrix:shard: [1, 2, 3]runs-on: macos-latest@@ -724,6 +730,7 @@ jobs:if: needs.detect.outputs.cli == 'true'strategy:fail-fast: false+ max-parallel: ${{ fromJSON(needs.detect.outputs.matrix_parallel) }}matrix:os: ${{ fromJSON(needs.detect.outputs.os_matrix) }}runs-on: ${{ matrix.os }}@@ -781,8 +788,9 @@ jobs:if: needs.detect.outputs.deps == 'true'strategy:fail-fast: false- # Windows ARM and Intel macOS on every change, both interpreters and- # Apple Silicon in the wide matrix (plan_run.py).+ max-parallel: ${{ fromJSON(needs.detect.outputs.matrix_parallel) }}+ # Windows ARM on a pull request, Intel macOS added in the queue, both+ # interpreters and Apple Silicon in the wide matrix (plan_run.py).matrix:include: ${{ fromJSON(needs.detect.outputs.crypto_matrix) }}runs-on: ${{ matrix.runner }}@@ -819,6 +827,7 @@ jobs:if: needs.detect.outputs.realtime == 'true'strategy:fail-fast: false+ max-parallel: ${{ fromJSON(needs.detect.outputs.matrix_parallel) }}matrix:os: ${{ fromJSON(needs.detect.outputs.lane_os) }}runs-on: ${{ matrix.os }}@@ -856,8 +865,8 @@ jobs:# The in-app updater on every OS it ships to. Each one replaces a running# app differently (Inno handover, .app swap, AppImage swap, git transaction# + restart helper), and the POSIX-only tests run the real relaunch waiter.- # A pull request runs Linux, macOS and the real Windows upgrade; the merge- # queue adds the second Windows interpreter; the wide matrix adds the+ # A pull request runs Linux and the real Windows upgrade; the merge queue+ # adds macOS and the second Windows interpreter; the wide matrix adds the# Intel Mac (scripts/ci/plan_run.py).## Python: 3.11 is the floor, 3.12 is what the native installers freeze.@@ -872,6 +881,7 @@ jobs:if: needs.detect.outputs.updater == 'true'strategy:fail-fast: false+ max-parallel: ${{ fromJSON(needs.detect.outputs.matrix_parallel) }}matrix:include: ${{ fromJSON(needs.detect.outputs.updater_matrix) }}runs-on: ${{ matrix.os }}@@ -991,6 +1001,7 @@ jobs:if: needs.detect.outputs.dragdrop == 'true'strategy:fail-fast: false+ max-parallel: ${{ fromJSON(needs.detect.outputs.matrix_parallel) }}matrix:os: ${{ fromJSON(needs.detect.outputs.os_matrix) }}runs-on: ${{ matrix.os }}@@ -1047,14 +1058,15 @@ jobs:# Hermes / OpenClaw society agents on every OS a person runs them on. The# lane (scripts/ci/agent_runtime_suite.py) covers the runtimes AND the chat- # and society code that drives them, on all three OSes on every pull- # request: a macOS failure shows before the queue. Fakes only: no key, no- # network to a model.+ # and society code that drives them. A pull request proves Linux and+ # Windows; the merge queue adds macOS, so a macOS failure stops the change+ # before it lands. Fakes only: no key, no network to a model.agent-runtimes-unit:needs: detectif: needs.detect.outputs.agent_runtimes == 'true'strategy:fail-fast: false+ max-parallel: ${{ fromJSON(needs.detect.outputs.matrix_parallel) }}matrix:os: ${{ fromJSON(needs.detect.outputs.lane_os) }}runs-on: ${{ matrix.os }}@@ -1117,6 +1129,7 @@ jobs:if: needs.detect.outputs.agent_runtimes == 'true'strategy:fail-fast: false+ max-parallel: ${{ fromJSON(needs.detect.outputs.matrix_parallel) }}matrix:include: ${{ fromJSON(needs.detect.outputs.e2e_matrix) }}runs-on: ${{ matrix.os }}diff --git a/AGENTS.md b/AGENTS.mdindex 8fa6dde6db..75a16e7094 100644--- a/AGENTS.md+++ b/AGENTS.md@@ -255,8 +255,8 @@ queue once green; the queue squash-merges it after `CI gate` passes again onthe PR merged with the newest main. The train enqueues only with its bottoken: `agent_land.py` therefore requests auto-merge, and a PR opened anyother way needs `gh pr merge <branch> --auto --squash` once, or it waits for-a person. A pull request runs Linux, Windows and the short macOS checks;-the long macOS checks (real runtimes, real browser) run in the merge+a person. A pull request runs Linux and Windows; every macOS check of the+affected lanes runs, blocking, in the mergequeue (`docs/ci-pipeline.md`, Tiers). Direct pushes to main are blocked foreveryone but admins.Triage any red job against the exact base; never add to a baseline to hide adiff --git a/docs/agent-runtimes.md b/docs/agent-runtimes.mdindex 50c3c130ea..52bca4f3ce 100644--- a/docs/agent-runtimes.md+++ b/docs/agent-runtimes.md@@ -407,7 +407,7 @@ each of them:| Job | Where | What it proves ||---|---|---|-| `agent-runtimes-unit` | Linux, Windows and macOS on every pull request and merge group; the portable base install | The runtime drivers, gateway, ACP runner, chat service, send queue, controls, society binding, roster, routines and society routes (`python scripts/ci/agent_runtime_suite.py`). Fakes only. |+| `agent-runtimes-unit` | Linux and Windows on a pull request, macOS added in the merge queue; the portable base install | The runtime drivers, gateway, ACP runner, chat service, send queue, controls, society binding, roster, routines and society routes (`python scripts/ci/agent_runtime_suite.py`). Fakes only. || `agent-runtimes-headless` | `python:3.11-slim` | The runtime package imports and its tests pass on a server with no GPU, audio or desktop stack, and the app's Hermes setup without curl stops with "Setup needs curl". || `agent-runtimes-e2e` | Every pull request: Linux × Hermes, OpenClaw and Windows × OpenClaw. Merge queue: macOS × Hermes, OpenClaw added. Nightly and release: Windows × Hermes added | `scripts/ci/agent_runtime_e2e.py`: the app's own setup job (`manager.wait_ready`) installs the release pinned in `jarvis/agent_runtimes/runtime-versions.json` with a desktop app's PATH, never npm or a manual installer. Then two turns per model-gateway protocol against scripted providers (`--replay`), the fake-model driver check with turn-1 budgets (Hermes 30 s, OpenClaw 90 s), `pytest tests/unit/agent_runtimes`, and proof that the Windows user PATH gained no `agent_runtimes` entry and OpenClaw edited no shell profile. No key and no network to a model. |diff --git a/docs/ci-pipeline.md b/docs/ci-pipeline.mdindex eceac20d49..7c44a890ab 100644--- a/docs/ci-pipeline.md+++ b/docs/ci-pipeline.md@@ -24,8 +24,8 @@ to prove (`scripts/ci/plan_run.py` emits the tier and every job matrix):| Tier | When | What runs | Target || --- | --- | --- | --- |-| `pr` | pull request, branch run dispatched by the merge train | Static gates, the whole Linux + Windows suite, every affected lane on Linux, Windows and macOS. The macOS legs here are the short ones, one to four minutes (realtime contract, updater, agent-runtime unit suite, native crypto on Intel, the Apple Silicon desktop bundle): a macOS failure shows on the pull request, and they end before the Windows shards do. | ~10 min |-| `queue` | merge-queue group; a push that reached main without the queue | Everything `pr` runs plus the long macOS legs: both real runtimes and the real browser on macOS (5 to 10 minutes each). The blocking proof of the exact tree main gets; nothing macOS-specific that a lane tests lands unproven. | ~11-13 min, longer when several groups need the five macOS slots at once |+| `pr` | pull request, branch run dispatched by the merge train | Static gates, the whole Linux + Windows suite, every affected lane on Linux and Windows. No macOS. | ~10 min |+| `queue` | merge-queue group; a push that reached main without the queue | Everything `pr` runs plus every macOS leg of the affected lanes: realtime contract, updater, agent-runtime unit suite, native crypto on Intel, the Apple Silicon desktop bundle, both real runtimes and the real browser. The blocking proof of the exact tree main gets; nothing macOS-specific that a lane tests lands unproven. | ~11-17 min || `full` | nightly, release tag, manual full dispatch | The wide matrix: Intel macOS, the ARM runners, every interpreter, the macOS test shards, all three macOS desktop rows, and the Hermes install on Windows. A red nightly opens the issue "Nightly CI is red" (`nightly-alert.yml`). | ~40 min || proven | push to main of a commit the merge queue already passed | `detect` and `CI gate` only. | under 1 min |@@ -35,7 +35,7 @@ one, turns every lane off. Any doubt (a direct push, an API error) fails opento the `queue` tier. A nightly run never accepts an earlier proof.`CI gate` waits for the slowest job of a run, and several merge groups build-at once on five macOS slots (the queue builds two groups in parallel). The+at once on five macOS slots. Thefirst queue runs of this design carried six to eight macOS legs each: threegroups and one manual full run waited more than half an hour on macOS alone.And the Hermes install on Windows (13 minutes of upstream installer before@@ -46,13 +46,21 @@ nightly run, which opens the issue "Nightly CI is red", and at the latest bythe release run, which blocks publication.macOS is different: a macOS failure must stop a change, not surface the-morning after. The short macOS legs run on every pull request, where they-finish inside the time the Windows shards take anyway. The long ones (both-real runtimes, 10 and 7 minutes, and the real browser) run once per change,-in the merge queue, which builds three groups at a time. When several-groups carry them at once they wait for one of the five macOS slots, and-the queue run takes that much longer. The whole Python suite on macOS-(three shards of nine minutes) still runs nightly and on release tags only.+morning after. Every macOS leg of the affected lanes, the real runtimes (10+and 7 minutes) and the real browser included, therefore runs in the merge+queue, before anything lands. It does not run on the pull request: five+macOS slots serve every run, and when the short legs ran on each push as+well (one morning, 2026-10-13), pull requests waited 6 to 25 minutes for a+slot. The queue builds two groups at a time, so at most sixteen macOS legs+compete for the five slots; in the first such group all eight were done 14+minutes in. The whole Python suite on macOS (three shards of nine minutes)+still runs nightly and on release tags only.++The nightly run has no reader waiting for it, so its matrices run one leg at+a time (`matrix_parallel`) instead of queueing 22 macOS jobs ahead of every+merge group. Its cron slot is 18:17 UTC: GitHub starts this repository's+schedules about seven hours late, and the old 02:17 slot ran between 08:30+and 09:30 UTC, in the working morning.A branch gets the whole macOS matrix before the queue with`gh workflow run ci.yml --ref <branch> -f full=false -f include_macos=true`.diff --git a/scripts/ci/plan_run.py b/scripts/ci/plan_run.pyindex 0a862c7fa6..215329cec1 100644--- a/scripts/ci/plan_run.py+++ b/scripts/ci/plan_run.py@@ -8,16 +8,16 @@ run has to prove, not what could be run:``pr`` A pull request (or a branch run the merge train dispatched). Fastfeedback: the whole Linux and Windows suite and every affected lane- on Linux, Windows and macOS. The macOS legs here are the short- ones (one to four minutes): they end inside the nine minutes the- Windows shards take, so a macOS failure shows on the pull request- and costs the run no time. It is the admission ticket to the- merge queue.+ on Linux and Windows. No macOS: five macOS slots serve every run,+ and with macOS legs on each push, pull requests waited 6 to 25+ minutes for a slot on 2026-10-13. It is the admission ticket to+ the merge queue.``queue`` A merge-queue group, or a push that reached main without one. Theblocking proof of the exact tree main gets: everything ``pr`` runs- plus the long macOS legs, the real runtimes and the real browser- (5 to 10 minutes each). Nothing macOS-specific that a lane tests- lands unproven; the price is paid once per change, not per push.+ plus every macOS leg of the affected lanes, short and long (the+ real runtimes and the real browser included). Nothing+ macOS-specific that a lane tests lands unproven, and the five+ slots are spent once per change, not once per push.``full`` Nightly, a release tag, a manual full dispatch. The wide matrix:Intel macOS, the ARM runners, every interpreter, the macOS testshards, and the Hermes install on Windows (14 to 23 minutes). A@@ -60,6 +60,7 @@ TIERS = ("pr", "queue", "full")CI_WORKFLOW_PATH = ".github/workflows/ci.yml"MAIN_REF = "refs/heads/main"+UNTHROTTLED = 20 # above the widest matrix: every leg at onceLINUX_SHARDS = 6# Six Windows shards finish in about the time of the Linux ones plus the# slower checkout; four made Windows the longest leg of every run.@@ -93,9 +94,13 @@ def decide("full_mode": full,"macos": macos,"pr_mode": pr_mode,- # One macOS leg per affected lane on every per-change tier; a full- # run carries them only when the caller asked for macOS.- "mac_lanes": tier != "full" or macos,+ # The macOS legs of the affected lanes: in the queue, and wherever+ # the caller asked for macOS.+ "mac_lanes": tier == "queue" or macos,+ # The nightly run has no reader waiting for it but shares the macOS+ # slots with every merge group: its matrices run one leg at a time+ # instead of queueing 22 macOS jobs ahead of everybody else's.+ "matrix_parallel": 1 if event == "schedule" else UNTHROTTLED,}@@ -103,10 +108,8 @@ def matrices(tier: str, *, macos: bool) -> dict[str, Any]:"""Every job matrix of ci.yml for a tier. ``macos`` adds the wide legs."""if tier not in TIERS:raise ValueError(f"unknown tier {tier!r}")- mac_lanes = tier != "full" or macos+ mac_lanes = tier == "queue" or macosqueue_legs = tier != "pr"- # The long macOS legs: once per change, in the queue, and in the wide matrix.- long_mac = tier == "queue" or macosbase_os = [LINUX, WINDOWS]updater = [@@ -147,13 +150,13 @@ def matrices(tier: str, *, macos: bool) -> dict[str, Any]:{"os": LINUX, "runtime": "openclaw"},{"os": WINDOWS, "runtime": "openclaw"},]- if long_mac:+ if mac_lanes:e2e += [{"os": MACOS, "runtime": "hermes"}, {"os": MACOS, "runtime": "openclaw"}]if macos:e2e.append({"os": WINDOWS, "runtime": "hermes"})browser = list(base_os)- if long_mac:+ if mac_lanes:browser.append(MACOS)if macos:browser += [LINUX_ARM, WINDOWS_ARM, MACOS_INTEL]@@ -167,8 +170,8 @@ def matrices(tier: str, *, macos: bool) -> dict[str, Any]:"crypto_matrix": crypto,"e2e_matrix": e2e,"browser_os": browser,- # Every change proves the Apple Silicon bundle (four minutes); the- # wide matrix adds both Intel rows.+ # Skipped on a pull request; the queue proves the Apple Silicon+ # bundle, the wide matrix adds both Intel rows."desktop_legs": "all" if macos else ("primary" if mac_lanes else "none"),"linux_shards": list(range(1, LINUX_SHARDS + 1)),"windows_shards": list(range(1, WINDOWS_SHARDS + 1)),diff --git a/tests/unit/ci/test_agent_runtime_lane.py b/tests/unit/ci/test_agent_runtime_lane.pyindex 04151b5a26..e2daefa819 100644--- a/tests/unit/ci/test_agent_runtime_lane.py+++ b/tests/unit/ci/test_agent_runtime_lane.py@@ -104,12 +104,11 @@ def test_ci_runs_the_lane_on_every_os_and_the_gate_waits_for_it():jobs = yaml.safe_load((ROOT / ".github/workflows/ci.yml").read_text(encoding="utf-8"))["jobs"]assert "agent_runtimes" in jobs["detect"]["outputs"]assert LANE_JOBS <= set(jobs["gate"]["needs"])- # A pull request runs the unit suite on all three OSes. Before anything- # lands (the merge queue) both real runtimes have run on Linux and macOS- # and OpenClaw on Windows; the nightly and release runs add the Hermes- # install on Windows. ci.yml reads the plan.+ # Before anything lands (the merge queue) the unit suite has run on all+ # three OSes, both real runtimes on Linux and macOS and OpenClaw on+ # Windows; the nightly and release runs add the Hermes install on+ # Windows. ci.yml reads the plan.oses = {"ubuntu-latest", "windows-latest", "macos-latest"}- assert set(plan_run.plan("pull_request")["lane_os"]) == osesqueue = plan_run.plan("merge_group")assert set(queue["lane_os"]) == osesassert {(leg["os"], leg["runtime"]) for leg in queue["e2e_matrix"]} == {diff --git a/tests/unit/ci/test_pipeline_audit.py b/tests/unit/ci/test_pipeline_audit.pyindex a911448270..87848ff44f 100644--- a/tests/unit/ci/test_pipeline_audit.py+++ b/tests/unit/ci/test_pipeline_audit.py@@ -243,18 +243,15 @@ def test_merge_queue_runs_the_whole_suite_and_is_never_cancelled():# --------------------------------------------------------------------------- tiers-def test_a_pull_request_shows_macos_failures_with_short_legs_only():+def test_a_pull_request_never_takes_a_macos_runner():+ # Five macOS slots serve every run. With macOS legs on each push, pull+ # requests waited 6 to 25 minutes for a slot (2026-10-13): the macOS+ # proof is spent once per change, in the merge queue.sized = plan_run.plan("pull_request")- assert sized["tier"] == "pr" and sized["pr_mode"] and sized["mac_lanes"]- # One short macOS leg per affected lane: a macOS failure shows on the- # pull request, and the legs end before the Windows shards do.- assert sized["lane_os"] == ["ubuntu-latest", "windows-latest", "macos-latest"]- assert sized["desktop_legs"] == "primary"- assert {"os": "macos-latest", "python": "3.12"} in sized["updater_matrix"]- # The long macOS legs (real runtimes, real browser) run once per change, in the queue.- assert "macos" not in json.dumps([sized["e2e_matrix"], sized["browser_os"]])- # Nothing in jarvisctl, dragdrop or the installer smoke is macOS-specific.- assert sized["os_matrix"] == ["ubuntu-latest", "windows-latest"]+ assert sized["tier"] == "pr" and sized["pr_mode"] and not sized["mac_lanes"]+ assert sized["desktop_legs"] == "none"+ matrices = [v for k, v in sized.items() if k not in ("macos", "mac_lanes")]+ assert "macos" not in json.dumps(matrices)# The slowest real-runtime leg never runs per change.assert {"os": "windows-latest", "runtime": "hermes"} not in sized["e2e_matrix"]assert {"os": "windows-latest", "runtime": "openclaw"} in sized["e2e_matrix"]@@ -268,9 +265,12 @@ def test_nothing_macos_specific_lands_unproven():assert sized["lane_os"] == ["ubuntu-latest", "windows-latest", "macos-latest"]assert sized["desktop_legs"] == "primary"# A macOS failure stops a change in the queue instead of surfacing the- # morning after: the real runtimes and the real browser run on macOS- # before anything lands. Only the Hermes install on Windows (14 to 23- # minutes) stays in the nightly and release runs.+ # morning after: every macOS leg of the affected lanes, the real runtimes+ # and the real browser included, runs before anything lands. Only the+ # Hermes install on Windows (14 to 23 minutes) stays in the nightly and+ # release runs.+ assert {"os": "macos-latest", "python": "3.12"} in sized["updater_matrix"]+ assert any(leg["runner"] == "macos-15-intel" for leg in sized["crypto_matrix"])assert sized["browser_os"] == ["ubuntu-latest", "windows-latest", "macos-latest"]assert {(leg["os"], leg["runtime"]) for leg in sized["e2e_matrix"]} == {("ubuntu-latest", "hermes"),@@ -463,6 +463,36 @@ def test_a_red_nightly_reaches_someone_and_only_a_nightly_on_main_can_write_issuassert only_nightly == [{"os": "windows-latest", "runtime": "hermes"}]+def test_the_nightly_run_never_queues_its_macos_legs_ahead_of_a_merge_group():+ ci = workflows()["ci.yml"]+ # One leg at a time in every matrix that carries macOS legs: the nightly+ # run has no reader waiting, the merge groups sharing the five slots do.+ assert plan_run.plan("schedule")["matrix_parallel"] == 1+ for event in ("pull_request", "merge_group", "push"):+ assert plan_run.plan(event)["matrix_parallel"] == plan_run.UNTHROTTLED+ # A release tag run is a manual dispatch and stays at full width.+ release = plan_run.plan("workflow_dispatch", full_input=True, macos_input=True)+ assert release["matrix_parallel"] == plan_run.UNTHROTTLED+ throttle = "${{ fromJSON(needs.detect.outputs.matrix_parallel) }}"+ for job in (+ "tests-macos",+ "jarvisctl",+ "native-crypto-runtime",+ "realtime-contract",+ "updater",+ "dragdrop",+ "agent-runtimes-unit",+ "agent-runtimes-e2e",+ ):+ assert ci["jobs"][job]["strategy"]["max-parallel"] == throttle, job+ assert ci["jobs"]["detect"]["outputs"]["matrix_parallel"] == (+ "${{ steps.mode.outputs.matrix_parallel }}"+ )+ # Every matrix fits below the unthrottled limit.+ widest = max(len(v) for v in plan_run.plan("schedule").values() if isinstance(v, list))+ assert widest < plan_run.UNTHROTTLED++def test_only_a_complete_green_run_on_main_writes_the_duration_caches():ci = workflows()["ci.yml"]["jobs"]job = ci["duration-cache"]