Dodaios is an open-source agent OS for your desktop: it runs your coding agents, browser and apps by voice or chat, on your own machine. Formerly Personal Jarvis.
Python agent-orchestrationai-agentai-coding-assistantclaude-codecoding-agentcomputer-usedesktop-applocal-firstmcppersonal-jarvisvoice-assistantvoice-coding
ci(release): cut releases with a release bot app from a main-only environment (#462)
Ruben Lütkeca178e405c52
diff --git a/.github/workflows/release-cut.yml b/.github/workflows/release-cut.ymlindex 28360bf300..44d96d8326 100644--- a/.github/workflows/release-cut.yml+++ b/.github/workflows/release-cut.yml@@ -8,17 +8,29 @@# section (scripts/ci/cut_release.py: [Unreleased] notes, or the# Conventional Commits since the last tag),# 3. checks the version commit through a candidate PR, then merges it normally,-# 4. pushes the annotated tag and dispatches the three publishing workflows-# on it (PyPI, desktop installers, signed installer). Each of them starts-# with release-gate.yml, which waits for CI on the release commit.+# 4. pushes the annotated tag; the three publishing workflows (PyPI, desktop+# installers, signed installer) run on it. Each of them starts with+# release-gate.yml, which waits for CI on the release commit.# 5. publishes the GitHub Release with the notes and the resumable# personal-jarvis-src.tar.gz asset the install scripts fall back to. The# in-app updater follows releases/latest, so this is the step that reaches# users.# 6. runs release-smoke.yml against the published release (read-only).#-# Tags pushed with GITHUB_TOKEN fire no `push: tags` event, which is why step 4-# dispatches explicitly; workflow_dispatch is the documented exception.+# Release bot. The `cut` job runs in the `release-cut` environment, which only+# main may deploy to. That environment holds the private key of the release+# bot GitHub App (secret RELEASE_APP_PRIVATE_KEY, variable RELEASE_APP_ID). The+# job mints short-lived installation tokens from it to push the candidate+# branch, open and merge the candidate PR, and push the tag. The "release tags"+# ruleset lets only that app and repository admins create `v*` tags, and the+# `release-signing` and `pypi` environments admit only `v*` tags, so the+# signing keys and PyPI are reachable only from a tag the bot cut on main.+#+# A bot token, unlike GITHUB_TOKEN, fires the normal events: the candidate PR+# runs CI by itself and the new tag starts the publishers by itself. Without+# the bot (RELEASE_APP_ID unset) the job falls back to GITHUB_TOKEN and+# dispatches the publishers itself, because a GITHUB_TOKEN tag push fires no+# `push: tags` event; workflow_dispatch is the documented exception.name: Release cut@@ -50,6 +62,8 @@ jobs:cut:name: cut releaseif: github.ref == 'refs/heads/main'+ # Holds the release bot key; its deployment policy admits main only.+ environment: release-cutruns-on: ubuntu-latesttimeout-minutes: 360permissions:@@ -60,16 +74,25 @@ jobs:outputs:version: ${{ steps.bump.outputs.version || steps.resume.outputs.version }}env:- GH_TOKEN: ${{ secrets.INTEGRATION_TOKEN || github.token }}- TOKEN_IS_BOT: ${{ secrets.INTEGRATION_TOKEN == '' }}+ # Reads and dispatches use GITHUB_TOKEN. Each write step overrides+ # GH_TOKEN with a bot token minted right before it: an installation+ # token expires after one hour, and the CI waits between writes can take+ # longer than that.+ GH_TOKEN: ${{ github.token }}+ # true = no release bot: pushes and merges made with GITHUB_TOKEN start+ # no workflows, so this job dispatches the publishers itself.+ TOKEN_IS_BOT: ${{ vars.RELEASE_APP_ID == '' }}steps:- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2with:fetch-depth: 0- token: ${{ secrets.INTEGRATION_TOKEN || github.token }}+ # git pushes through gh (next step), which uses the step's GH_TOKEN.+ persist-credentials: false- uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0with:python-version: "3.11"+ - name: Let git push with the step's token+ run: gh auth setup-git --hostname github.com- name: main must be greenif: inputs.resume_sha == ''run: python scripts/ci/release_admit.py --ci-only --sha "$GITHUB_SHA" --repo "$GITHUB_REPOSITORY" --wait-minutes 0@@ -98,10 +121,21 @@ jobs:new="$(python scripts/ci/cut_release.py "${arg[@]}" --notes-out "$RUNNER_TEMP/notes.md" | tail -1)"echo "version=$new" >> "$GITHUB_OUTPUT"git diff --stat- - name: Check the version commit before landing and tagging+ - name: Mint a release bot token for the candidate+ id: candidate-token+ if: inputs.resume_sha == '' && vars.RELEASE_APP_ID != ''+ uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0+ with:+ client-id: ${{ vars.RELEASE_APP_ID }}+ private-key: ${{ secrets.RELEASE_APP_PRIVATE_KEY }}+ permission-contents: write+ permission-pull-requests: write+ - name: Open the candidate PR+ id: candidateif: inputs.resume_sha == ''env:V: ${{ steps.bump.outputs.version }}+ GH_TOKEN: ${{ steps.candidate-token.outputs.token || github.token }}run: |git config user.name "jarvis-release[bot]"git config user.email "41898282+github-actions[bot]@users.noreply.github.com"@@ -112,14 +146,23 @@ jobs:git push origin "HEAD:refs/heads/$candidate"# Only eligible events (here pull_request) satisfy branch protection.# A workflow_dispatch CI success is not accepted by GitHub for landing.+ # GITHUB_TOKEN may not open pull requests here; the release bot may.if ! gh pr create --base main --head "$candidate" --title "chore(release): publish v$V" \--body-file "$RUNNER_TEMP/notes.md"; thenecho "::notice::Repository policy may require a maintainer to open the candidate PR."echo "Open https://github.com/$GITHUB_REPOSITORY/compare/main...$candidate?expand=1" >> "$GITHUB_STEP_SUMMARY"fi+ echo "sha=$sha" >> "$GITHUB_OUTPUT"+ echo "branch=$candidate" >> "$GITHUB_OUTPUT"+ - name: Wait for the candidate CI gate+ if: inputs.resume_sha == ''+ env:+ CANDIDATE: ${{ steps.candidate.outputs.branch }}+ CANDIDATE_SHA: ${{ steps.candidate.outputs.sha }}+ run: |ci_run=""for attempt in {1..60}; do- ci_run="$(gh run list --workflow ci.yml --branch "$candidate" --commit "$sha" \+ ci_run="$(gh run list --workflow ci.yml --branch "$CANDIDATE" --commit "$CANDIDATE_SHA" \--event pull_request --json databaseId --jq '.[0].databaseId // empty')"[ -n "$ci_run" ] && breaksleep 5@@ -129,19 +172,36 @@ jobs:gh api -X POST "repos/$GITHUB_REPOSITORY/actions/runs/$ci_run/approve"figh run watch "$ci_run" --exit-status --interval 30- state="$(gh api "repos/$GITHUB_REPOSITORY/commits/$sha/check-runs?check_name=CI%20gate" \+ state="$(gh api "repos/$GITHUB_REPOSITORY/commits/$CANDIDATE_SHA/check-runs?check_name=CI%20gate" \--jq '(.check_runs | max_by(.id)) | if .status == "completed" then .conclusion else (.status // "missing") end')"[ "$state" = "success" ] || { echo "::error::Candidate CI gate is $state."; exit 1; }- gh pr merge "$candidate" --merge --match-head-commit "$sha"- merged="$(gh pr view "$candidate" --json mergeCommit --jq '.mergeCommit.oid')"+ - name: Mint a release bot token for landing+ id: land-token+ if: inputs.resume_sha == '' && vars.RELEASE_APP_ID != ''+ uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0+ with:+ client-id: ${{ vars.RELEASE_APP_ID }}+ private-key: ${{ secrets.RELEASE_APP_PRIVATE_KEY }}+ permission-contents: write+ permission-pull-requests: write+ - name: Land the candidate+ if: inputs.resume_sha == ''+ env:+ CANDIDATE: ${{ steps.candidate.outputs.branch }}+ CANDIDATE_SHA: ${{ steps.candidate.outputs.sha }}+ GH_TOKEN: ${{ steps.land-token.outputs.token || github.token }}+ DISPATCH_TOKEN: ${{ github.token }}+ run: |+ gh pr merge "$CANDIDATE" --merge --match-head-commit "$CANDIDATE_SHA"+ merged="$(gh pr view "$CANDIDATE" --json mergeCommit --jq '.mergeCommit.oid')"[[ "$merged" =~ ^[0-9a-f]{40}$ ]] || { echo "::error::Missing merge commit."; exit 1; }git fetch --no-tags origin main:refs/remotes/origin/maingit checkout --detach "$merged"- if [ "$TOKEN_IS_BOT" = "true" ]; then- gh workflow run ci.yml --ref main -f full=true -f include_macos=true- fi- - name: Tag the admitted release- id: tag+ # Always dispatch: a GITHUB_TOKEN merge starts no CI, and a bot+ # merge's push run shares the main-push group, where a newer push to+ # main drops it while it still waits for a runner.+ GH_TOKEN="$DISPATCH_TOKEN" gh workflow run ci.yml --ref main -f full=true -f include_macos=true+ - name: Admit the release commitenv:V: ${{ steps.bump.outputs.version || steps.resume.outputs.version }}run: |@@ -150,6 +210,23 @@ jobs:actual="$(python -c 'from scripts.ci.release_admit import versions; print(versions()[0])')"[ "$actual" = "$V" ] || { echo "::error::Merged package version changed."; exit 1; }python scripts/ci/release_admit.py --ci-only --sha "$sha" --repo "$GITHUB_REPOSITORY"+ - name: Mint a release bot token for the tag+ id: tag-token+ if: vars.RELEASE_APP_ID != ''+ uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0+ with:+ client-id: ${{ vars.RELEASE_APP_ID }}+ private-key: ${{ secrets.RELEASE_APP_PRIVATE_KEY }}+ permission-contents: write+ - name: Tag the admitted release+ id: tag+ env:+ V: ${{ steps.bump.outputs.version || steps.resume.outputs.version }}+ # The "release tags" ruleset lets only the release bot and admins+ # create v* tags.+ GH_TOKEN: ${{ steps.tag-token.outputs.token || github.token }}+ run: |+ sha="$(git rev-parse HEAD)"git config user.name "jarvis-release[bot]"git config user.email "41898282+github-actions[bot]@users.noreply.github.com"if git show-ref --verify --quiet "refs/tags/v$V"; then@@ -165,10 +242,10 @@ jobs:V: ${{ steps.bump.outputs.version || steps.resume.outputs.version }}TAG_PUSHED: ${{ steps.tag.outputs.pushed }}run: |- # GITHUB_TOKEN tag pushes do not start CI; dispatch the immutable tag.+ # ci.yml has no tag trigger; dispatch full CI on the immutable tag.gh workflow run ci.yml --ref "v$V" -f full=true -f include_macos=truepython scripts/ci/release_assets.py prepare --repo "$GITHUB_REPOSITORY" --tag "v$V"- # An integration token's new tag already triggered the publishers.+ # A new tag pushed by the release bot already started the publishers.# Dispatching twice races an immutable PyPI upload against itself.if [ "$TOKEN_IS_BOT" = "true" ] || [ "$TAG_PUSHED" != "true" ]; thenpython scripts/ci/release_assets.py dispatch --repo "$GITHUB_REPOSITORY" --tag "v$V"diff --git a/.github/zizmor.yml b/.github/zizmor.ymlindex 3fce53c102..ce0fa3518a 100644--- a/.github/zizmor.yml+++ b/.github/zizmor.yml@@ -16,10 +16,9 @@ rules:- flaky-tests.ymlartipacked:ignore:- # These jobs push commits or tags with git after checkout, so the- # checkout must keep its credential. Neither uploads the workspace.+ # This job pushes commits with git after checkout, so the checkout must+ # keep its credential. It does not upload the workspace.- merge-train.yml- - release-cut.ymlunpinned-uses:config:policies:diff --git a/docs/ci-pipeline.md b/docs/ci-pipeline.mdindex 4a3335dbc5..5c9589c353 100644--- a/docs/ci-pipeline.md+++ b/docs/ci-pipeline.md@@ -168,11 +168,19 @@ A release happens **only** when the maintainer asks for one.the Conventional Commits since the last tag) into a dated CHANGELOG section(`scripts/ci/cut_release.py`), lands the candidate through a CI-checked PR,tags the resulting merge commit, and dispatches full CI with macOS on that- immutable tag. It dispatches publishers when using `GITHUB_TOKEN`; a new- tag pushed with the optional integration token already starts them, so no- duplicate PyPI publication is dispatched. Resume support accepts an already- merged version commit. Repository PR policy may require the integration- token or a maintainer to open the candidate PR.+ immutable tag. Its `cut` job runs in the `release-cut` environment, which+ only `main` may deploy to and which holds the private key of the release+ bot GitHub App (secret `RELEASE_APP_PRIVATE_KEY`, variable+ `RELEASE_APP_ID`). Right before each write (candidate push and PR, merge,+ tag push) the job mints a fresh installation token with+ `actions/create-github-app-token`; CI waits use `GITHUB_TOKEN`, because an+ installation token lasts one hour. Bot pushes fire the normal events: the+ candidate PR runs CI by itself and the new tag starts the publishers, so no+ duplicate PyPI publication is dispatched. Without the bot+ (`RELEASE_APP_ID` unset) the job falls back to `GITHUB_TOKEN`, dispatches+ the publishers itself, and a maintainer must open the candidate PR, since+ `GITHUB_TOKEN` may not open pull requests here. Resume support accepts an+ already merged version commit.* **`release-gate.yml`** is the first job of `release.yml` (PyPI),`desktop-installers.yml` and `sign-installer.yml`. It admits a tag onlywhen tag, versions and CHANGELOG agree, the commit is on main, and@@ -190,8 +198,8 @@ A release happens **only** when the maintainer asks for one.its checksum, and verifies those uploads before publishing. The release-cut`publish` job is the normal finalizer. `release-finalize.yml` shares itsper-tag lock but only fires on its own (`workflow_run`) when a person or the- integration token started the publishers: publisher runs dispatched with- `GITHUB_TOKEN` raise no `workflow_run` event, so with the default token it+ release bot started the publishers: publisher runs dispatched with+ `GITHUB_TOKEN` raise no `workflow_run` event, so without the bot itstays a manual retry path. Finalization canalso be retried manually from main with an existing draft tag. Producersnever edit release visibility or replace existing asset names. Before andiff --git a/scripts/ci/check_workflow_policy.py b/scripts/ci/check_workflow_policy.pyindex d3ab236805..40d6afd914 100644--- a/scripts/ci/check_workflow_policy.py+++ b/scripts/ci/check_workflow_policy.py@@ -52,6 +52,25 @@ def audit(workflows: dict[str, dict]) -> list[str]:provenance = signing["jobs"]["provenance"]["with"]if provenance.get("upload-assets") is not False or "draft-release" in provenance:findings.append("sign-installer.yml: provenance must only produce a workflow artifact")+ findings += _release_bot_findings(workflows)+ return findings+++def _release_bot_findings(workflows: dict[str, dict]) -> list[str]:+ """Only release-cut's main-only job may hold the key that creates v* tags."""+ findings = []+ for filename, workflow in workflows.items():+ for name, job in workflow.get("jobs", {}).items():+ if filename == "release-cut.yml" and name == "cut":+ continue+ if "RELEASE_APP_PRIVATE_KEY" in yaml.safe_dump(job):+ findings.append(f"{filename}/{name}: only release-cut may use the release bot key")+ cut = workflows["release-cut.yml"]["jobs"]["cut"]+ if cut.get("environment") != "release-cut":+ findings.append("release-cut.yml: the cut job must run in the release-cut environment")+ tag = next((s for s in cut["steps"] if s.get("name") == "Tag the admitted release"), {})+ if "steps.tag-token.outputs.token" not in str(tag.get("env", {}).get("GH_TOKEN", "")):+ findings.append("release-cut.yml: push the release tag with the release bot token")return findingsdiff --git a/tests/unit/ci/test_pipeline_audit.py b/tests/unit/ci/test_pipeline_audit.pyindex d6a9e86da4..8e55aea4f9 100644--- a/tests/unit/ci/test_pipeline_audit.py+++ b/tests/unit/ci/test_pipeline_audit.py@@ -379,12 +379,55 @@ def test_workflow_security_policy_and_mutation_regressions():lambda w: w["stargazer-map.yml"]["jobs"]["verify"]["steps"][0].update({"uses": "actions/checkout@v4"}),+ lambda w: w["release-cut.yml"]["jobs"]["cut"].pop("environment"),+ lambda w: _release_cut_step(w, "Tag the admitted release")["env"].update(+ {"GH_TOKEN": "${{ github.token }}"}+ ),+ lambda w: w["ci.yml"]["jobs"]["gate"]["steps"][0].update(+ {"env": {"KEY": "${{ secrets.RELEASE_APP_PRIVATE_KEY }}"}}+ ),):changed = copy.deepcopy(clean)mutate(changed)assert check_workflow_policy.audit(changed)+def _release_cut_step(definitions: dict, name: str) -> dict:+ steps = definitions["release-cut.yml"]["jobs"]["cut"]["steps"]+ return next(step for step in steps if step.get("name") == name)+++def test_release_cut_writes_with_fresh_bot_tokens_and_reads_with_github_token():+ definitions = workflows()+ cut = definitions["release-cut.yml"]["jobs"]["cut"]+ assert cut["environment"] == "release-cut"+ assert cut["env"]["GH_TOKEN"] == "${{ github.token }}" # noqa: S105 - expression+ assert cut["env"]["TOKEN_IS_BOT"] == "${{ vars.RELEASE_APP_ID == '' }}" # noqa: S105+ checkout = cut["steps"][0]+ assert checkout["uses"].startswith("actions/checkout@")+ assert checkout["with"]["persist-credentials"] is False+ # Every write step mints its own token right before it (one-hour lifetime).+ for writer, minter in (+ ("Open the candidate PR", "candidate-token"),+ ("Land the candidate", "land-token"),+ ("Tag the admitted release", "tag-token"),+ ):+ names = [step.get("name") for step in cut["steps"]]+ assert cut["steps"][names.index(writer) - 1].get("id") == minter+ token = _release_cut_step(definitions, writer)["env"]["GH_TOKEN"]+ assert token == f"${{{{ steps.{minter}.outputs.token || github.token }}}}"+ minters = [s for s in cut["steps"] if s.get("id", "").endswith("-token")]+ assert len(minters) == 3+ for step in minters:+ assert step["uses"].startswith("actions/create-github-app-token@")+ assert "vars.RELEASE_APP_ID != ''" in step["if"]+ assert step["with"]["private-key"] == "${{ secrets.RELEASE_APP_PRIVATE_KEY }}"+ assert "permission-workflows" not in step["with"]+ # Long CI waits never hold a bot token.+ for reader in ("Wait for the candidate CI gate", "Admit the release commit"):+ assert "GH_TOKEN" not in _release_cut_step(definitions, reader).get("env", {})++def test_all_shards_consume_the_same_detect_output_including_partial_reruns():ci = workflows()["ci.yml"]["jobs"]for name in ("tests-linux", "tests-windows", "tests-macos"):@@ -399,8 +442,8 @@ def test_all_shards_consume_the_same_detect_output_including_partial_reruns():def test_release_ci_targets_tag_with_all_platforms():- steps = workflows()["release-cut.yml"]["jobs"]["cut"]["steps"]- dispatch = next(s["run"] for s in steps if s.get("name") == "Dispatch the publishing workflows on the tag")+ step = _release_cut_step(workflows(), "Dispatch the publishing workflows on the tag")+ dispatch = step["run"]assert 'ci.yml --ref "v$V" -f full=true -f include_macos=true' in dispatch@@ -414,7 +457,8 @@ def test_release_cut_lands_through_protection_and_tags_the_merge_commit():cut = workflows()["release-cut.yml"]["jobs"]["cut"]commands = "\n".join(step.get("run", "") for step in cut["steps"])assert "git push origin HEAD:main" not in commands- assert 'gh pr merge "$candidate" --merge --match-head-commit "$sha"' in commands+ merge = 'gh pr merge "$CANDIDATE" --merge --match-head-commit "$CANDIDATE_SHA"'+ assert merge in commandsassert 'git checkout --detach "$merged"' in commandsassert '[ "$TOKEN_IS_BOT" = "true" ] || [ "$TAG_PUSHED" != "true" ]' in commandsdiff --git a/tests/unit/ci/test_release_cut_workflow.py b/tests/unit/ci/test_release_cut_workflow.pyindex f7158336c4..629b83e8c3 100644--- a/tests/unit/ci/test_release_cut_workflow.py+++ b/tests/unit/ci/test_release_cut_workflow.py@@ -106,6 +106,9 @@ def _exercise("RELEASE_REF": "refs/tags/v1.2.3","TAG_EXISTS": "1","TAG_SHA": _SHA,+ "TAG_PUSHED": "true",+ "CANDIDATE": "release-cut/v1.2.3-77-2",+ "CANDIDATE_SHA": _SHA,**overrides,},capture_output=True,@@ -148,6 +151,49 @@ def test_release_qualification_fails_on_a_failing_auth_contract(tmp_path):assert not (tmp_path / "summary.txt").exists()+_CANDIDATE_TO_TAG = [+ "Open the candidate PR",+ "Wait for the candidate CI gate",+ "Land the candidate",+ "Admit the release commit",+ "Tag the admitted release",+]+_MAIN_CI = "gh workflow run ci.yml --ref main -f full=true -f include_macos=true"+_DISPATCH = "Dispatch the publishing workflows on the tag"+_PUBLISHERS = "python scripts/ci/release_assets.py dispatch --repo example/project --tag v1.2.3"++[email protected]("token_is_bot", ["true", "false"])+def test_landing_always_dispatches_ci_for_the_merge_commit(tmp_path, token_is_bot):+ # A bot merge's push run can be dropped from the main-push group while it+ # still queues; the dispatched run has its own group.+ result, commands = _exercise(tmp_path, _CANDIDATE_TO_TAG, TOKEN_IS_BOT=token_is_bot)+ assert result.returncode == 0, result.stdout + result.stderr+ assert commands.count(_MAIN_CI) == 1+ assert commands.index(_MAIN_CI) < commands.index("git tag -a v1.2.3 -m v1.2.3")+++ "token_is_bot,tag_pushed,dispatched",+ [+ # GITHUB_TOKEN tag push fires no tag events: dispatch the publishers.+ ("true", "true", True),+ # The release bot's new tag already started them: never twice.+ ("false", "true", False),+ # The tag existed before this run (resume): nothing started them.+ ("false", "false", True),+ ("true", "false", True),+ ],+)+def test_publishers_start_exactly_once(tmp_path, token_is_bot, tag_pushed, dispatched):+ result, commands = _exercise(+ tmp_path, [_DISPATCH], TOKEN_IS_BOT=token_is_bot, TAG_PUSHED=tag_pushed+ )+ assert result.returncode == 0, result.stdout + result.stderr+ assert commands[0] == "gh workflow run ci.yml --ref v1.2.3 -f full=true -f include_macos=true"+ assert (_PUBLISHERS in commands) is dispatched++@pytest.mark.parametrize("overrides,success",[@@ -161,10 +207,7 @@ def test_release_qualification_fails_on_a_failing_auth_contract(tmp_path):def test_only_a_green_candidate_can_land_and_publish(tmp_path, overrides, success):result, commands = _exercise(tmp_path,- [- "Check the version commit before landing and tagging",- "Tag the admitted release",- ],+ _CANDIDATE_TO_TAG,**overrides,)assert (result.returncode == 0) is success, result.stdout + result.stderr@@ -200,6 +243,7 @@ def test_resume_requires_merged_current_version_and_admission(tmp_path, overridetmp_path,["Select an already merged version commit",+ "Admit the release commit","Tag the admitted release",],**overrides,