Skip to content

The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.

Allow overlapping Desktop builds with guarded publication (#618)

Deec136b1fbe024
diff --git a/.github/workflows/release-desktop.yml b/.github/workflows/release-desktop.yml
index be8a302d5..069f231bc 100644
--- a/.github/workflows/release-desktop.yml
+++ b/.github/workflows/release-desktop.yml
@@ -59,16 +59,19 @@ on:
description: 'Manifest to write. Leave as-is for a real release; metadata-test.json for a dry run.'
required: false
default: harness/desktop/metadata.json
+ test_publication:
+ description: 'Only test publication against disposable fixture objects; no app build or product release'
+ type: boolean
+ default: false
permissions:
contents: read
id-token: write # only used by the Workload Identity Federation variant of the auth steps below
concurrency:
- # Two releases overlapping would interleave their manifest writes. Never cancelled in progress:
- # a half-published release leaves the manifest describing one platform's new build beside
- # another's old one.
- group: desktop-release
+ # Different versions build concurrently. Only publish+verification takes the
+ # shared manifest lock below; a whole-workflow lock added 4m35s to 1.2.47.
+ group: desktop-release-${{ github.ref }}-${{ inputs.version }}
cancel-in-progress: false
env:
@@ -89,6 +92,7 @@ jobs:
# Resolve and validate before anything expensive starts. A malformed tag fails here in seconds
# rather than 25 minutes into a notarized build.
version:
+ if: ${{ !inputs.test_publication }}
runs-on: ubuntu-latest
outputs:
version: ${{ steps.resolve.outputs.version }}
@@ -118,11 +122,12 @@ jobs:
echo "scratch_prefix=harness/desktop/.ci/${GITHUB_RUN_ID}" >> "$GITHUB_OUTPUT"
echo "Publishing **$ver** to \`$meta\`" >> "$GITHUB_STEP_SUMMARY"
- # Uploading is the last thing each build job does, ~25 minutes in. Proving the credential now
- # means a broken one costs seconds. Runs beside `version`, so it is free in wall clock.
+ # Reject duplicate/older versions and broken credentials before expensive builds.
preflight:
+ needs: version
runs-on: ubuntu-latest
steps:
+ - uses: actions/checkout@v7
# GCP auth, WITHOUT a long-lived key — the same pattern release.yml and production-be-build.yaml
# already run. A service-account JSON key is a bearer credential with no expiry: it is only as
# safe as every place it has ever been copied to. Workload Identity Federation replaces it with
@@ -143,6 +148,18 @@ jobs:
workload_identity_provider: ${{ secrets.GCP_WORKLOAD_IDENTITY_PROVIDER }}
service_account: ${{ secrets.GCP_SERVICE_ACCOUNT }}
- uses: google-github-actions/setup-gcloud@v2
+ - name: Reject a version already published or superseded
+ env:
+ VERSION: ${{ needs.version.outputs.version }}
+ METADATA_PATH: ${{ needs.version.outputs.metadata_path }}
+ run: |
+ set -euo pipefail
+ args=()
+ if [[ "$METADATA_PATH" != harness/desktop/metadata.json ]]; then
+ args+=(--allow-initialize)
+ fi
+ python3 scripts/publish-desktop-manifest.py "$VERSION" \
+ "gs://${GCS_BUCKET}/${METADATA_PATH}" --check-only "${args[@]}"
- name: Verify GCS write access
run: |
set -euo pipefail
@@ -307,10 +324,21 @@ jobs:
publish:
needs: [version, build-macos, build-linux]
runs-on: ubuntu-latest
+ timeout-minutes: 8
+ concurrency:
+ # Include verification: another release must not advance the live manifest
+ # between this publication and its six-download check. GCS's generation
+ # condition also protects against writers outside this workflow.
+ # Keep the historical group so a release already using the older workflow
+ # still serializes with this publisher during the transition.
+ group: desktop-release
+ cancel-in-progress: false
env:
METADATA_PATH: ${{ needs.version.outputs.metadata_path }}
SCRATCH_PREFIX: ${{ needs.version.outputs.scratch_prefix }}
+ VERSION: ${{ needs.version.outputs.version }}
steps:
+ - uses: actions/checkout@v7
- name: Authenticate to GCP (workload identity federation)
uses: google-github-actions/auth@v2
with:
@@ -323,76 +351,23 @@ jobs:
set -euo pipefail
mkdir -p parts
gcloud storage cp "gs://${GCS_BUCKET}/${SCRATCH_PREFIX}/*.json" parts/
- echo "::group::scratch manifests"; ls -la parts; echo "::endgroup::"
-
- src="$(mktemp)"; dst="$(mktemp)"
- if ! gcloud storage cp "gs://${GCS_BUCKET}/${METADATA_PATH}" "$src" 2>/dev/null; then
- echo " (no existing manifest at ${METADATA_PATH} — creating a new one)"
- printf '{}' > "$src"
+ args=()
+ if [[ "$METADATA_PATH" != harness/desktop/metadata.json ]]; then
+ args+=(--allow-initialize)
fi
- # Paths through argv, NEVER piped into this heredoc — the heredoc claims stdin, so a pipe
- # would be silently dropped and the merge would blank the manifest.
- python3 - "$src" "$dst" parts <<'PY'
- import json, os, sys
-
- src, dst, parts = sys.argv[1:4]
- try:
- with open(src) as f:
- raw = f.read()
- data = json.loads(raw) if raw.strip() else {}
- except (OSError, json.JSONDecodeError):
- data = {}
- if not isinstance(data, dict):
- data = {}
-
- merged = 0
- for name in sorted(os.listdir(parts)):
- if not name.endswith(".json"):
- continue
- with open(os.path.join(parts, name)) as f:
- part = json.load(f)
- if not isinstance(part, dict):
- sys.exit(f"error: {name} is not a JSON object")
- for key, entry in part.items():
- data[key] = entry
- merged += 1
- print(f" {key} -> {entry.get('version')}")
-
- # An empty merge would quietly re-upload the manifest unchanged and report success, which
- # is indistinguishable from a release that published nothing.
- if merged == 0:
- sys.exit("error: no manifest entries found — did the build jobs upload anything?")
-
- with open(dst, "w") as f:
- json.dump(data, f, indent=2)
- f.write("\n")
- PY
- gcloud storage cp --content-type=application/json \
- --cache-control="no-cache, no-store, must-revalidate" \
- "$dst" "gs://${GCS_BUCKET}/${METADATA_PATH}"
+ python3 scripts/publish-desktop-manifest.py "$VERSION" \
+ "gs://${GCS_BUCKET}/${METADATA_PATH}" --parts parts "${args[@]}"
echo "Published to \`${METADATA_PATH}\`" >> "$GITHUB_STEP_SUMMARY"
- # Start as soon as the public manifest is live, beside release-page creation.
- # A green workflow is the six-download verification receipt; do not download them
- # all again locally just to finish the release task.
- verify:
- needs: [version, publish]
- runs-on: ubuntu-latest
- timeout-minutes: 5
- permissions:
- contents: read
- steps:
- - uses: actions/checkout@v7
+ # Same worker and publication lock: no extra runner startup and no chance
+ # of another workflow changing the live version during this verification.
- name: Verify all public artifacts as the updater
- env:
- VERSION: ${{ needs.version.outputs.version }}
- METADATA_PATH: ${{ needs.version.outputs.metadata_path }}
run: |
python3 scripts/verify-desktop-release.py "$VERSION" \
--manifest-url "https://storage.googleapis.com/${GCS_BUCKET}/${METADATA_PATH}" \
--receipt desktop-release-verification.json
- uses: actions/upload-artifact@v4
- if: always()
+ if: ${{ always() && hashFiles('desktop-release-verification.json') != '' }}
with:
name: desktop-release-verification
path: desktop-release-verification.json
@@ -486,7 +461,7 @@ jobs:
# written theirs. `if: always()` here means the prefix is cleared whatever happened upstream.
cleanup:
needs: [version, build-macos, build-linux, publish]
- if: always()
+ if: ${{ always() && needs.version.result == 'success' }}
runs-on: ubuntu-latest
steps:
- name: Authenticate to GCP (workload identity federation)
@@ -499,3 +474,19 @@ jobs:
env:
SCRATCH_PREFIX: ${{ needs.version.outputs.scratch_prefix }}
run: gcloud storage rm -r "gs://${GCS_BUCKET}/${SCRATCH_PREFIX}" || true
+
+ # This mode exercises actual GCS generation preconditions using tiny fixture
+ # artifacts under its own run prefix. All product build/publish jobs are skipped.
+ publication-test:
+ if: ${{ inputs.test_publication }}
+ runs-on: ubuntu-latest
+ timeout-minutes: 10
+ steps:
+ - uses: actions/checkout@v7
+ - uses: google-github-actions/auth@v2
+ with:
+ workload_identity_provider: ${{ secrets.GCP_WORKLOAD_IDENTITY_PROVIDER }}
+ service_account: ${{ secrets.GCP_SERVICE_ACCOUNT }}
+ - uses: google-github-actions/setup-gcloud@v2
+ - name: Exercise publication against a disposable GCS prefix
+ run: python3 scripts/check-desktop-publication-gcs.py "$GCS_BUCKET"
diff --git a/desktop/scripts/upload-desktop-linux.sh b/desktop/scripts/upload-desktop-linux.sh
index d3fa890a9..925d15c1d 100755
--- a/desktop/scripts/upload-desktop-linux.sh
+++ b/desktop/scripts/upload-desktop-linux.sh
@@ -133,11 +133,12 @@ gcloud storage --help >/dev/null 2>&1 || {
exit 1
}
-# gcs_cp <src> <dst> [cache-control] [content-type] — either side may be gs:// or a local path or `-`.
+# gcs_cp <src> <dst> [cache-control] [content-type] [generation-match].
gcs_cp() {
- local src="$1" dst="$2" cc="${3:-}" ct="${4:-}" args=(storage cp)
+ local src="$1" dst="$2" cc="${3:-}" ct="${4:-}" generation="${5:-}" args=(storage cp)
if [ -n "$cc" ]; then args+=("--cache-control=$cc"); fi
if [ -n "$ct" ]; then args+=("--content-type=$ct"); fi
+ if [ -n "$generation" ]; then args+=("--if-generation-match=$generation"); fi
gcloud "${args[@]}" "$src" "$dst"
}
@@ -255,7 +256,7 @@ echo " dest: gs://${GCS_BUCKET}/${GCS_PATH}"
# Immutable per-version path — see the CDN_ASSET_BASE_URL note near the top of this script. Long
# max-age here is what actually lets the CDN cache it instead of hitting GCS on every install/update.
gcs_refuse_republish "gs://${GCS_BUCKET}/${GCS_PATH}"
-gcs_cp "$OUTPUT" "gs://${GCS_BUCKET}/${GCS_PATH}" "public, max-age=31536000, immutable"
+gcs_cp "$OUTPUT" "gs://${GCS_BUCKET}/${GCS_PATH}" "public, max-age=31536000, immutable" "" 0
echo ">> merging manifest: gs://${GCS_BUCKET}/${METADATA_PATH} (${OTA_KEY})"
SRC="$(mktemp)"; DST="$(mktemp)" # removed by cleanup() on EXIT
diff --git a/desktop/scripts/upload-desktop.sh b/desktop/scripts/upload-desktop.sh
index 476377065..3157758ee 100755
--- a/desktop/scripts/upload-desktop.sh
+++ b/desktop/scripts/upload-desktop.sh
@@ -132,11 +132,12 @@ gcloud storage --help >/dev/null 2>&1 || {
exit 1
}
-# gcs_cp <src> <dst> [cache-control] [content-type] — either side may be gs:// or a local path or `-`.
+# gcs_cp <src> <dst> [cache-control] [content-type] [generation-match].
gcs_cp() {
- local src="$1" dst="$2" cc="${3:-}" ct="${4:-}" args=(storage cp)
+ local src="$1" dst="$2" cc="${3:-}" ct="${4:-}" generation="${5:-}" args=(storage cp)
if [ -n "$cc" ]; then args+=("--cache-control=$cc"); fi
if [ -n "$ct" ]; then args+=("--content-type=$ct"); fi
+ if [ -n "$generation" ]; then args+=("--if-generation-match=$generation"); fi
gcloud "${args[@]}" "$src" "$dst"
}
@@ -318,10 +319,10 @@ echo ">> uploading release $VER"
# max-age here is what actually lets the CDN cache these instead of hitting GCS on every install/update.
echo " zip: gs://${GCS_BUCKET}/${GCS_PATH} ($SIZE bytes, sha256=$SHA)"
gcs_refuse_republish "gs://${GCS_BUCKET}/${GCS_PATH}"
-gcs_cp "$ZIP" "gs://${GCS_BUCKET}/${GCS_PATH}" "public, max-age=31536000, immutable"
+gcs_cp "$ZIP" "gs://${GCS_BUCKET}/${GCS_PATH}" "public, max-age=31536000, immutable" "" 0
echo " dmg: gs://${GCS_BUCKET}/${DMG_GCS_PATH} ($DMG_SIZE bytes, sha256=$DMG_SHA)"
gcs_refuse_republish "gs://${GCS_BUCKET}/${DMG_GCS_PATH}"
-gcs_cp "$DMG" "gs://${GCS_BUCKET}/${DMG_GCS_PATH}" "public, max-age=31536000, immutable"
+gcs_cp "$DMG" "gs://${GCS_BUCKET}/${DMG_GCS_PATH}" "public, max-age=31536000, immutable" "" 0
echo ">> merging manifest: gs://${GCS_BUCKET}/${METADATA_PATH} (${OTA_KEY}, ${DMG_KEY})"
SRC="$(mktemp)"; DST="$(mktemp)" # removed by cleanup() on EXIT
diff --git a/docs/performance/2026-10-02-desktop-release-overlap.md b/docs/performance/2026-10-02-desktop-release-overlap.md
new file mode 100644
index 000000000..66f24808c
--- /dev/null
+++ b/docs/performance/2026-10-02-desktop-release-overlap.md
@@ -0,0 +1,56 @@
+# Overlapping Desktop release builds
+
+The next measured opportunity after #612 is waiting behind another release.
+[1.2.47](https://github.com/autonomous-ai/openharness/actions/runs/36964579887)
+was requested at 04:26:45 UTC on October 2. Its first jobs started at 04:31:23,
+three seconds after [1.2.46](https://github.com/autonomous-ai/openharness/actions/runs/36964024308)
+finished. The earlier audit attributes 4m35s to the shared whole-workflow lock;
+the full trigger-to-first-job interval was 4m38s.
+
+## Change
+
+Different versions can now build concurrently. Only publication and public download
+verification hold the shared `desktop-release` lock. Retaining that lock name also
+coordinates with an older workflow already running during the transition. The
+remaining lock is a short publish/verify phase, rather than compilation and both
+notarization passes. GitHub runner availability can still cause queues.
+
+Publication validates all four platform files and all six expected entries, checks
+the version against every live Desktop entry, reads one exact GCS generation and
+writes only if that generation is still current. Duplicate or superseded versions
+fail before building when already known, or before the final manifest write if
+another release became live during the build. A corrupt or unreadable live manifest
+is never silently replaced with an empty object. Initializing a new scratch manifest
+is explicit and only an actual 404 is treated as absence.
+
+Each macOS ZIP/DMG and Linux AppImage upload also uses a create-only GCS condition.
+This closes the gap between checking that a versioned path does not exist and
+uploading to it. The updater's public version, full SHA-256 and size checks remain
+inside the publisher's lock. The release watcher accepts that exact successful
+step, while still supporting #612's separate verification job.
+
+These use [GCS generation preconditions](https://docs.cloud.google.com/storage/docs/request-preconditions)
+and [job concurrency](https://docs.github.com/en/actions/how-tos/write-workflows/choose-when-workflows-run/control-workflow-concurrency).
+Concurrency does not promise version order. A newer release that finishes first
+must remain live; a late older attempt is explicitly rejected. GitHub may also
+replace pending jobs in a busy concurrency group. Never label such an attempt as
+a successful publication or move an existing version tag to retry it.
+
+## Validation and expected effect
+
+The local process suite passes 40 tests, including the real publisher and actual
+artifact upload commands against a disposable object-store fixture. Cases cover
+complete publication, mixed/missing/wrong platform data, corrupt manifests,
+permission failures, explicit initialization, stale reads/writes, all live version
+comparisons and immutable artifact collisions. Workflow and shell syntax checks
+also pass. The workflow includes a mode to exercise these storage preconditions
+against actual GCS fixture objects without building or releasing an application.
+
+For a release overlapping another, this removes the requirement to wait for the
+previous build and notarization before starting. The historical 1.2.47 case had
+4m35s of avoidable waiting. That is an opportunity illustrated by an actual past
+queue, not a measured 4m35s reduction for every future release. A release with no
+overlap gets little benefit from this change; its Flutter compilation and Apple
+notarization still take their normal time. Publication and verification are now
+on one worker, followed by release-page creation; the small handoff changes must
+be included when measuring the next full release.
diff --git a/docs/performance/2026-10-02-release-process-audit.md b/docs/performance/2026-10-02-release-process-audit.md
index 6ad4f0fd1..53b3fb9f3 100644
--- a/docs/performance/2026-10-02-release-process-audit.md
+++ b/docs/performance/2026-10-02-release-process-audit.md
@@ -84,9 +84,11 @@ All 17 published versions were included: **1.2.33–1.2.35 and 1.2.37–1.2.50**
Attempt **1.2.36** was cancelled. This includes all ten releases initially requested
(1.2.41–1.2.50), not just the latest example.
-The confirmed 4m35s concurrency wait for 1.2.47 was behind 1.2.46. Desktop's global
-publication serialization remains because its manifest merge lacks a generation
-guard; removing that lock alone would risk mixed/older platform entries. It is a
+The confirmed 4m35s concurrency wait for 1.2.47 was behind 1.2.46. At the initial
+audit, Desktop's whole-workflow lock was retained because its manifest merge lacked
+a generation guard; removing that lock alone would risk mixed/older platform
+entries. The [publication follow-up](2026-10-02-desktop-release-overlap.md) adds
+those guards and narrows the lock to publication and verification. Queueing was a
secondary cost, not an explanation for the hours before tagging.
CLI coverage is **0.3.31–0.3.57**; web coverage is **1.3.14–1.3.25**. TUI logs
diff --git a/docs/validation-and-release.md b/docs/validation-and-release.md
index 80a09b126..995bdadac 100644
--- a/docs/validation-and-release.md
+++ b/docs/validation-and-release.md
@@ -151,15 +151,37 @@ tag and source through completion:
make release-desktop ARGS="--notes-file /path/to/reviewed-notes.md --wait"
```
-The workflow's `verify` job downloads all six public artifacts three at a time,
+The workflow's `publish` job downloads all six public artifacts three at a time,
using the updater's Dart user agent, and checks every version, full SHA-256, and
-size. It starts immediately after publication, alongside release-page creation.
+size. It starts immediately after publication on the same runner.
Each transfer has a total deadline; any failure makes the workflow fail and keeps
the JSON receipt in the `desktop-release-verification` artifact. A successful
verification satisfies that release's download checks: report completion instead
of downloading everything again locally. The watcher requires this job to pass
and checks the tag's full SHA, so another release's success cannot satisfy it.
+Different Desktop versions may build concurrently. The shared release lock covers
+publication and its public verification, so a second workflow cannot change the
+live version midway through that check. Versions are checked before building and
+again immediately before publication. A late older version fails as superseded;
+do not retry it over a newer live release. All four platform manifests must contain
+exactly the six expected entries for one version. Artifact writes are create-only,
+and the final manifest write requires the GCS generation read by the publisher.
+Concurrent external changes therefore cause a failure instead of being lost.
+
+For publication changes, run the local process suite and the disposable GCS
+contract check on the candidate branch:
+
+```bash
+gh workflow run release-desktop.yml --ref BRANCH -f version=0.0.0 -f test_publication=true
+```
+
+This mode skips all product build/release jobs. It uses tiny public fixture objects
+under `harness/desktop/.publication-check/<run>-<attempt>/`, checks actual GCS
+preconditions and downloads, and removes only that run's prefix. It exercises
+normal publication, duplicate/older versions, incomplete platform sets, concurrent
+manifest writes, and immutable artifact collisions. It cannot publish an app.
+
For a release made before that job existed, or to investigate a download failure:
```bash
diff --git a/scripts/check-desktop-publication-gcs.py b/scripts/check-desktop-publication-gcs.py
new file mode 100644
index 000000000..17bfbda3d
--- /dev/null
+++ b/scripts/check-desktop-publication-gcs.py
@@ -0,0 +1,95 @@
+#!/usr/bin/env python3
+"""Exercise real publication preconditions using tiny, run-owned GCS fixtures.
+
+Requires GitHub run/attempt IDs. Never accepts a production metadata path, and
+always removes only its own fixture prefix. No application is built or released.
+"""
+import json
+import os
+from pathlib import Path
+import re
+import subprocess
+import sys
+import tempfile
+
+sys.path.insert(0, str(Path(__file__).with_name("tests")))
+from desktop_publication_fixture import artifact_upload_script, make_parts, publisher
+
+
+def must_reject(operation):
+ try:
+ operation()
+ except (ValueError, RuntimeError):
+ return
+ raise AssertionError("unsafe publication unexpectedly succeeded")
+
+
+def main():
+ bucket, = sys.argv[1:]
+ run = os.environ.get("GITHUB_RUN_ID", "")
+ attempt = os.environ.get("GITHUB_RUN_ATTEMPT", "")
+ if not re.fullmatch(r"[a-z0-9._-]+", bucket) or not run.isdigit() or not attempt.isdigit():
+ raise ValueError("requires a bucket and numeric GitHub run/attempt IDs")
+ prefix = f"harness/desktop/.publication-check/{run}-{attempt}"
+ base = f"gs://{bucket}/{prefix}"
+ uri = f"{base}/metadata.json"
+ try:
+ with tempfile.TemporaryDirectory(prefix="desktop-publication-check-") as folder:
+ root = Path(folder)
+ parts = root / "parts"
+ for version in ("0.0.1", "0.0.3"):
+ entries = make_parts(parts, version, f"https://storage.googleapis.com/{bucket}/{prefix}/{version}")
+ files = [str(parts / key) for key in entries]
+ publisher.gcloud("storage", "cp", *files, f"{base}/{version}/", "--if-generation-match=0")
+ publisher.publish(uri, version, parts, allow_initialize=True)
+ receipt = publisher.verification.verify(version, f"https://storage.googleapis.com/{bucket}/{prefix}/metadata.json")
+ assert receipt["status"] == "passed", receipt
+ print(f"PASS: publish and verify all six {version} fixtures", flush=True)
+
+ live, generation = publisher.read_live(uri)
+ for version in ("0.0.2", "0.0.3"):
+ make_parts(parts, version, "https://fixture.invalid/never-uploaded")
+ must_reject(lambda: publisher.publish(uri, version, parts))
+ assert publisher.read_live(uri) == (live, generation)
+ print("PASS: late older and duplicate releases preserve the current version", flush=True)
+
+ make_parts(parts, "0.0.4", "https://fixture.invalid/never-uploaded")
+ (parts / "linux-arm64.json").unlink()
+ must_reject(lambda: publisher.publish(uri, "0.0.4", parts))
+ assert publisher.read_live(uri) == (live, generation)
+ print("PASS: incomplete platform sets never change the manifest", flush=True)
+
+ # An external writer wins after the publisher reads. The stale writer
+ # must get a real GCS precondition failure and preserve those changes.
+ concurrent = dict(live, fixture_concurrent_write=True)
+ publisher.write_manifest(uri, concurrent, generation)
+ must_reject(lambda: publisher.write_manifest(uri, live, generation))
+ assert publisher.read_live(uri)[0] == concurrent
+ print("PASS: stale generation cannot overwrite a concurrent writer", flush=True)
+
+ # Use the production shell helper AND each actual upload call site.
+ # Bypass the advisory exists check to exercise the atomic race guard.
+ payload = root / "artifact"
+ for script, variable in (("upload-desktop.sh", "ZIP"), ("upload-desktop.sh", "DMG"), ("upload-desktop-linux.sh", "OUTPUT")):
+ payload.write_text("original fixture bytes")
+ path = f"{prefix}/immutable-{variable}"
+ env = dict(os.environ, **{variable: str(payload)}, GCS_BUCKET=bucket, GCS_PATH=path, DMG_GCS_PATH=path)
+ command = ["bash", "-eu", "-c", artifact_upload_script(script, variable)]
+ subprocess.run(command, env=env, check=True, capture_output=True, text=True, timeout=90)
+ payload.write_text("replacement fixture bytes")
+ result = subprocess.run(command, env=env, capture_output=True, text=True, timeout=90)
+ assert result.returncode != 0, result.stdout
+ assert publisher.gcloud("storage", "cat", f"gs://{bucket}/{path}") == "original fixture bytes"
+ print("PASS: macOS ZIP/DMG and Linux upload commands refuse immutable overwrites", flush=True)
+ if summary := os.environ.get("GITHUB_STEP_SUMMARY"):
+ with open(summary, "a") as stream:
+ stream.write("Desktop publication contract passed against disposable GCS fixtures: six-platform publication/downloads, superseded and duplicate versions, incomplete builds, concurrent manifest writes, and immutable artifacts.\n")
+ finally:
+ # The prefix is constructed above from numeric run/attempt IDs, never user
+ # input or the production metadata_path. An empty/root cleanup is impossible.
+ publisher.gcloud("storage", "rm", "--recursive", base + "/")
+ return 0
+
+
+if __name__ == "__main__":
+ sys.exit(main())
diff --git a/scripts/publish-desktop-manifest.py b/scripts/publish-desktop-manifest.py
new file mode 100644
index 000000000..287daed4a
--- /dev/null
+++ b/scripts/publish-desktop-manifest.py
@@ -0,0 +1,135 @@
+#!/usr/bin/env python3
+"""Publish one complete Desktop version using a generation-conditional GCS write.
+
+Builds may overlap. A late older build, incomplete platform set, corrupt live
+manifest or concurrent writer must fail without replacing the live manifest.
+"""
+import argparse
+import importlib.util
+import json
+from pathlib import Path
+import re
+import subprocess
+import sys
+import tempfile
+
+# Use the same six-entry/schema contract as public download verification.
+spec = importlib.util.spec_from_file_location("desktop_verification", Path(__file__).with_name("verify-desktop-release.py"))
+verification = importlib.util.module_from_spec(spec)
+spec.loader.exec_module(verification)
+
+PARTS = {
+ "macos-intel.json": {"desktop-macos", "desktop-macos-dmg"},
+ "macos-apple-silicon.json": {"desktop-macos-arm64", "desktop-macos-arm64-dmg"},
+ "linux-x64.json": {"desktop-linux-x64"},
+ "linux-arm64.json": {"desktop-linux-arm64"},
+}
+
+
+def version_tuple(value):
+ if not isinstance(value, str) or not re.fullmatch(r"(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)", value):
+ raise ValueError(f"invalid Desktop version: {value!r}")
+ return tuple(map(int, value.split(".")))
+
+
+def gcloud(*args):
+ result = subprocess.run(["gcloud", *args], capture_output=True, text=True, timeout=90)
+ if result.returncode:
+ raise RuntimeError(result.stderr.strip() or "gcloud failed")
+ return result.stdout
+
+
+def read_live(uri, allow_initialize=False):
+ if not re.fullmatch(r"gs://[a-z0-9._-]+/[A-Za-z0-9._/-]+", uri) or ".." in uri.split("/"):
+ raise ValueError("manifest must be a literal gs://bucket/object path")
+ try:
+ description = json.loads(gcloud("storage", "objects", "describe", uri, "--format=json"))
+ except RuntimeError as error:
+ # Permission/transport failures are never treated as an absent manifest.
+ if allow_initialize and re.search(r"\b404\b", str(error)):
+ return {}, "0"
+ raise
+ generation = str(description.get("generation", ""))
+ if not re.fullmatch(r"[1-9][0-9]*", generation):
+ raise ValueError("GCS returned no valid manifest generation")
+ # Read exactly the generation described; a change in between fails closed.
+ live = json.loads(gcloud("storage", "cat", f"{uri}#{generation}"))
+ if not isinstance(live, dict):
+ raise ValueError("live manifest must be an object")
+ return live, generation
+
+
+def require_new_version(version, live, allow_initialize=False):
+ requested = version_tuple(version)
+ found = False
+ for key, entry in live.items():
+ if not key.startswith("desktop-"):
+ continue
+ found = True
+ if not isinstance(entry, dict):
+ raise ValueError(f"{key}: invalid live entry")
+ current = entry.get("version")
+ if requested <= version_tuple(current):
+ raise ValueError(f"{version} is not newer than live {key} {current}; this release is superseded or already published")
+ if not found and not allow_initialize:
+ raise ValueError("live manifest has no Desktop versions; initialization requires --allow-initialize")
+
+
+def read_parts(directory, version):
+ files = {path.name for path in directory.glob("*.json")}
+ if files != set(PARTS):
+ raise ValueError(f"expected four platform manifests; missing={sorted(set(PARTS) - files)}, extra={sorted(files - set(PARTS))}")
+ entries = {}
+ for name, keys in PARTS.items():
+ part = json.loads((directory / name).read_text())
+ if not isinstance(part, dict) or set(part) != keys:
+ raise ValueError(f"{name}: expected exactly {sorted(keys)}")
+ entries.update(part)
+ verification.entries_for_version(entries, version)
+ return entries
+
+
+def write_manifest(uri, manifest, generation):
+ with tempfile.TemporaryDirectory(prefix="desktop-manifest-") as folder:
+ path = Path(folder) / "metadata.json"
+ path.write_text(json.dumps(manifest, indent=2) + "\n")
+ gcloud("storage", "cp", str(path), uri, f"--if-generation-match={generation}",
+ "--content-type=application/json", "--cache-control=no-cache, no-store, must-revalidate")
+
+
+def publish(uri, version, parts, allow_initialize=False):
+ entries = read_parts(parts, version)
+ live, generation = read_live(uri, allow_initialize)
+ require_new_version(version, live, allow_initialize)
+ merged = dict(live, **entries)
+ write_manifest(uri, merged, generation)
+ return merged
+
+
+def main(argv=None):
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument("version")
+ parser.add_argument("manifest")
+ parser.add_argument("--parts", type=Path)
+ parser.add_argument("--check-only", action="store_true")
+ parser.add_argument("--allow-initialize", action="store_true", help="only for an explicitly new scratch manifest")
+ args = parser.parse_args(argv)
+ if not args.check_only and args.parts is None:
+ parser.error("publication requires --parts")
+ try:
+ version_tuple(args.version)
+ if args.check_only:
+ live, _ = read_live(args.manifest, args.allow_initialize)
+ require_new_version(args.version, live, args.allow_initialize)
+ print(f"Desktop {args.version} is newer than the live manifest")
+ else:
+ publish(args.manifest, args.version, args.parts, args.allow_initialize)
+ print(f"Published all six Desktop {args.version} entries to {args.manifest}")
+ except (OSError, ValueError, RuntimeError, subprocess.TimeoutExpired) as error:
+ print(f"error: {error}", file=sys.stderr)
+ return 1
+ return 0
+
+
+if __name__ == "__main__":
+ sys.exit(main())
diff --git a/scripts/tests/desktop_publication_fixture.py b/scripts/tests/desktop_publication_fixture.py
new file mode 100644
index 000000000..38dbd85c7
--- /dev/null
+++ b/scripts/tests/desktop_publication_fixture.py
@@ -0,0 +1,33 @@
+"""Shared fixture data for local contract tests and the disposable GCS check."""
+import hashlib
+import importlib.util
+import json
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[2]
+spec = importlib.util.spec_from_file_location("desktop_publisher", ROOT / "scripts/publish-desktop-manifest.py")
+publisher = importlib.util.module_from_spec(spec)
+spec.loader.exec_module(publisher)
+
+
+def make_parts(directory, version, base_url):
+ directory.mkdir(parents=True, exist_ok=True)
+ entries = {}
+ for name, keys in publisher.PARTS.items():
+ part = {}
+ for key in sorted(keys):
+ payload = f"Publication contract fixture: {version} {key}\n".encode()
+ part[key] = dict(version=version, url=f"{base_url}/{key}",
+ size=len(payload), sha256=hashlib.sha256(payload).hexdigest())
+ (directory / key).write_bytes(payload)
+ (directory / name).write_text(json.dumps(part))
+ entries.update(part)
+ return entries
+
+
+def artifact_upload_script(script_name, variable):
+ """Exercise the actual helper AND upload invocation, without building an app."""
+ source = (ROOT / "desktop/scripts" / script_name).read_text()
+ function = "gcs_cp() {" + source.split("gcs_cp() {", 1)[1].split("\n}", 1)[0] + "\n}\n"
+ invocation = next(line for line in source.splitlines() if line.startswith(f'gcs_cp "${variable}" '))
+ return function + invocation + "\n"
diff --git a/scripts/tests/test_desktop_publication.py b/scripts/tests/test_desktop_publication.py
new file mode 100644
index 000000000..1b023b79e
--- /dev/null
+++ b/scripts/tests/test_desktop_publication.py
@@ -0,0 +1,182 @@
+"""Execute the real publisher/upload commands against a disposable object store."""
+import json
+import os
+from pathlib import Path
+import subprocess
+import sys
+import tempfile
+import unittest
+
+from desktop_publication_fixture import ROOT, artifact_upload_script, make_parts, publisher
+
+URI = "gs://fixture-bucket/harness/desktop/metadata.json"
+GCLOUD = r'''#!/usr/bin/env python3
+import json, os, sys
+from pathlib import Path
+path = Path(os.environ['FAKE_CLOUD_STATE'])
+state = json.loads(path.read_text())
+args = sys.argv[1:]
+state.setdefault('calls', []).append(args)
+objects = state['objects']
+def save(): path.write_text(json.dumps(state))
+def fail(message):
+ save()
+ print(message, file=sys.stderr)
+ sys.exit(1)
+if args[:3] == ['storage', 'objects', 'describe']:
+ if os.environ.get('FAKE_READ_ERROR'): fail(os.environ['FAKE_READ_ERROR'])
+ obj = objects.get(args[3])
+ if obj is None: fail('HTTPError 404: object not found')
+ print(json.dumps({'generation': str(obj['generation'])}))
+elif args[:2] == ['storage', 'cat']:
+ uri, generation = args[2].rsplit('#', 1)
+ obj = objects[uri]
+ if os.environ.get('FAKE_READ_RACE'):
+ obj['generation'] += 1
+ obj['data'] = '{"external": "preserved"}'
+ if int(generation) != obj['generation']: fail('HTTPError 404: old generation gone')
+ print(obj['data'])
+elif args[:2] == ['storage', 'cp']:
+ source, target = [a for a in args[2:] if not a.startswith('--')]
+ obj = objects.get(target)
+ if os.environ.get('FAKE_WRITE_RACE') and target.endswith('/metadata.json'):
+ obj['generation'] += 1
+ obj['data'] = '{"external": "preserved"}'
+ expected = next((a.split('=', 1)[1] for a in args if a.startswith('--if-generation-match=')), None)
+ actual = obj['generation'] if obj else 0
+ if expected is not None and int(expected) != actual: fail('HTTPError 412: generation mismatch')
+ objects[target] = {'generation': actual + 1, 'data': Path(source).read_text()}
+else:
+ fail('unexpected gcloud arguments: ' + repr(args))
+save()
+'''
+
+
+class DesktopPublicationTests(unittest.TestCase):
+ def setUp(self):
+ self.tmp = tempfile.TemporaryDirectory()
+ self.addCleanup(self.tmp.cleanup)
+ self.root = Path(self.tmp.name)
+ self.parts = self.root / "parts"
+ self.entries = make_parts(self.parts, "1.2.2", "https://fixture.invalid/1.2.2")
+ bindir = self.root / "bin"
+ bindir.mkdir()
+ client = bindir / "gcloud"
+ client.write_text(GCLOUD)
+ client.chmod(0o755)
+ self.state_path = self.root / "cloud.json"
+ self.env = dict(os.environ, PATH=f"{bindir}:{os.environ['PATH']}", FAKE_CLOUD_STATE=str(self.state_path))
+ self.seed()
+
+ def seed(self, version="1.2.1"):
+ live = {key: dict(entry, version=version) for key, entry in self.entries.items()}
+ live["other-product"] = {"keep": True}
+ self.state_path.write_text(json.dumps(dict(objects={URI: dict(generation=7, data=json.dumps(live))})))
+
+ def state(self):
+ return json.loads(self.state_path.read_text())
+
+ def run_publisher(self, version="1.2.2", flags=(), env=None):
+ result = subprocess.run([sys.executable, str(ROOT / "scripts/publish-desktop-manifest.py"),
+ version, URI, "--parts", str(self.parts), *flags],
+ env=dict(self.env, **(env or {})), capture_output=True, text=True, timeout=15)
+ return result, self.state()
+
+ def test_complete_manifest_is_written_once_and_preserves_other_keys(self):
+ result, state = self.run_publisher()
+ self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
+ live = json.loads(state["objects"][URI]["data"])
+ self.assertEqual(live["other-product"], {"keep": True})
+ self.assertEqual({key: live[key] for key in self.entries}, self.entries)
+ writes = [call for call in state["calls"] if call[:2] == ["storage", "cp"]]
+ self.assertEqual(len(writes), 1)
+ self.assertEqual(state["objects"][URI]["generation"], 8)
+
+ def test_duplicate_and_superseded_versions_fail_in_preflight_and_publication(self):
+ for live_version in ("1.2.2", "1.2.3", "1.10.0"):
+ for flags in ((), ("--check-only",)):
+ with self.subTest(live_version=live_version, flags=flags):
+ self.seed(live_version)
+ result, state = self.run_publisher(flags=flags)
+ self.assertNotEqual(result.returncode, 0)
+ self.assertIn("not newer", result.stderr)
+ self.assertEqual(state["objects"][URI]["generation"], 7)
+
+ def test_all_live_desktop_versions_are_checked(self):
+ state = self.state()
+ live = json.loads(state["objects"][URI]["data"])
+ live["desktop-linux-arm64"]["version"] = "1.2.9"
+ state["objects"][URI]["data"] = json.dumps(live)
+ self.state_path.write_text(json.dumps(state))
+ result, state = self.run_publisher()
+ self.assertNotEqual(result.returncode, 0)
+ self.assertEqual(state["objects"][URI]["generation"], 7)
+
+ def test_missing_extra_wrong_platform_and_mixed_version_parts_fail_before_gcs(self):
+ for change in ("missing", "extra", "wrong-platform", "mixed-version", "bad-hash", "bad-size"):
+ with self.subTest(change=change):
+ self.seed()
+ make_parts(self.parts, "1.2.2", "https://fixture.invalid/1.2.2")
+ path = self.parts / "linux-arm64.json"
+ part = json.loads(path.read_text())
+ if change == "missing":
+ path.unlink()
+ elif change == "extra":
+ (self.parts / "extra.json").write_text("{}")
+ else:
+ if change == "wrong-platform": part = {"desktop-linux-x64": part["desktop-linux-arm64"]}
+ if change == "mixed-version": part["desktop-linux-arm64"]["version"] = "1.2.1"
+ if change == "bad-hash": part["desktop-linux-arm64"]["sha256"] = "invalid"
+ if change == "bad-size": part["desktop-linux-arm64"]["size"] = False
+ path.write_text(json.dumps(part))
+ result, state = self.run_publisher()
+ self.assertNotEqual(result.returncode, 0)
+ self.assertEqual(state.get("calls", []), [])
+ (self.parts / "extra.json").unlink(missing_ok=True)
+
+ def test_concurrent_writes_and_reads_do_not_replace_external_changes(self):
+ for variable in ("FAKE_WRITE_RACE", "FAKE_READ_RACE"):
+ with self.subTest(variable=variable):
+ self.seed()
+ result, state = self.run_publisher(env={variable: "1"})
+ self.assertNotEqual(result.returncode, 0)
+ self.assertEqual(json.loads(state["objects"][URI]["data"]), {"external": "preserved"})
+
+ def test_initialization_is_explicit_and_permission_errors_fail_closed(self):
+ for flags, success in (((), False), (("--allow-initialize",), True)):
+ self.state_path.write_text(json.dumps({"objects": {}}))
+ result, state = self.run_publisher(flags=flags)
+ self.assertEqual(result.returncode == 0, success, result.stderr)
+ self.seed()
+ result, state = self.run_publisher(flags=("--allow-initialize",), env={"FAKE_READ_ERROR": "HTTPError 403: access denied"})
+ self.assertNotEqual(result.returncode, 0)
+ self.assertEqual(state["objects"][URI]["generation"], 7)
+
+ def test_corrupt_or_empty_live_manifest_is_not_silently_reset(self):
+ for payload in ("not json", "[]", "{}", '{"desktop-macos":{"version":"bad"}}'):
+ with self.subTest(payload=payload):
+ state = self.state()
+ state["objects"][URI] = dict(generation=7, data=payload)
+ self.state_path.write_text(json.dumps(state))
+ result, state = self.run_publisher()
+ self.assertNotEqual(result.returncode, 0)
+ self.assertEqual(state["objects"][URI]["data"], payload)
+
+ def test_real_artifact_upload_commands_cannot_overwrite_existing_bytes(self):
+ for script, variable in (("upload-desktop.sh", "ZIP"), ("upload-desktop.sh", "DMG"), ("upload-desktop-linux.sh", "OUTPUT")):
+ with self.subTest(script=script, variable=variable):
+ payload = self.root / "artifact"
+ payload.write_text("original bytes")
+ target = f"harness/desktop/1.2.2/{variable}"
+ env = dict(self.env, **{variable: str(payload)}, GCS_BUCKET="fixture-bucket", GCS_PATH=target, DMG_GCS_PATH=target)
+ command = ["bash", "-eu", "-c", artifact_upload_script(script, variable)]
+ first = subprocess.run(command, env=env, capture_output=True, text=True, timeout=5)
+ self.assertEqual(first.returncode, 0, first.stderr)
+ payload.write_text("replacement bytes")
+ second = subprocess.run(command, env=env, capture_output=True, text=True, timeout=5)
+ self.assertNotEqual(second.returncode, 0)
+ self.assertEqual(self.state()["objects"][f"gs://fixture-bucket/{target}"]["data"], "original bytes")
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/scripts/tests/test_watch_desktop_release.py b/scripts/tests/test_watch_desktop_release.py
index d2dd942b1..5f785bf07 100644
--- a/scripts/tests/test_watch_desktop_release.py
+++ b/scripts/tests/test_watch_desktop_release.py
@@ -37,6 +37,18 @@ class ReleaseWatchTests(unittest.TestCase):
with self.assertRaisesRegex(TimeoutError, "no release-desktop.yml run appeared"):
watch.watch(TAG, SHA)
+ def test_verification_inside_publish_must_have_a_successful_step(self):
+ for conclusion in ("success", "skipped", "failure"):
+ result = dict(DONE, jobs=[dict(name="publish", status="completed", conclusion="success", steps=[
+ dict(name="Verify all public artifacts as the updater", conclusion=conclusion)
+ ])])
+ with self.subTest(conclusion=conclusion), patch.object(watch, "gh_json", side_effect=[[RUN], result]):
+ if conclusion == "success":
+ self.assertEqual(watch.watch(TAG, SHA), RUN)
+ else:
+ with self.assertRaises(RuntimeError):
+ watch.watch(TAG, SHA)
+
def test_in_progress_run_is_rechecked_without_another_validation(self):
pending = dict(DONE, status="in_progress", conclusion="", jobs=[])
with patch.object(watch, "gh_json", side_effect=[[RUN], pending, DONE]) as gh, patch.object(watch.time, "sleep"):
diff --git a/scripts/watch-desktop-release.py b/scripts/watch-desktop-release.py
index 8d4466161..500734175 100644
--- a/scripts/watch-desktop-release.py
+++ b/scripts/watch-desktop-release.py
@@ -50,7 +50,19 @@ def watch(tag, sha, timeout=1800, appear_timeout=180):
if detail["status"] == "completed":
if detail["conclusion"] != "success":
raise RuntimeError(f"release {detail['conclusion']}: {run['url']}")
- if not any(job["name"] == "verify" and job["conclusion"] == "success" for job in detail["jobs"]):
+ # Older releases used a separate verify job. Newer ones verify
+ # inside publish's lock, so require that exact step to pass.
+ verified = any(
+ job["conclusion"] == "success" and (
+ job["name"] == "verify" or (
+ job["name"] == "publish" and any(
+ step["name"] == "Verify all public artifacts as the updater" and step["conclusion"] == "success"
+ for step in job.get("steps", [])
+ )
+ )
+ ) for job in detail["jobs"]
+ )
+ if not verified:
raise RuntimeError("run has no successful public-download verification; use verify-desktop-release.py for older releases")
release_url = run["url"].split("/actions/runs/", 1)[0] + "/releases/tag/" + tag
print(f"Published and verified: {release_url}", flush=True)