The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.
TypeScript 3d-modelingagent-frameworkagent-harnessai-agentscadclaude-codecodexcoding-agentsdeveloper-toolsesp32game-developmentmcp-servermulti-agentopen-hardwareopencodeopenharnesspcbroboticsterminaltmux
Compress the final macOS ZIP during DMG notarization (#668)
Deefce430890bf0
diff --git a/desktop/RELEASE.md b/desktop/RELEASE.mdindex a7cb16184..6eaece01f 100644--- a/desktop/RELEASE.md+++ b/desktop/RELEASE.md@@ -164,7 +164,9 @@ manifests are combined only after all macOS and Linux builds succeed, followed ball six public downloads. `macos-build-timings` records the bounded build and per-variant phases.App copies and DMG staging use independent APFS clones when available. The temporarynotarization ZIP uses faster compression; the downloadable ZIP retains normal compression-and is rebuilt from the stapled app. Signing and both notarization checks remain the same.+and is rebuilt from the stapled app while DMG notarization runs. Compression failures stop+publication; uploads wait for both artifacts and their checks. Packaging phase timings are+printed in the build log. Signing and both notarization checks remain the same.`scripts/publish-macos-variant.sh intel|apple-silicon <version>` remains the single-variant entrypoint. With `--no-build`, `APP_BUNDLE` selects an existing bundle; the uploader's `OUTPUT_DIR`@@ -258,10 +260,10 @@ For one macOS build, `publish-macos-variant.sh`:publishing anything.6. Packages the `.app` with `ditto -c -k --sequesterRsrc --keepParent` (keeps the bundle structure andextended attributes intact — a plain `zip` does not).-7. Notarizes the zip, staples the ticket into the `.app`, re-zips from the stapled bundle, and- asserts Gatekeeper accepts it (`spctl`).+7. Notarizes the zip, staples the ticket into the `.app`, and asserts Gatekeeper accepts it (`spctl`).8. Packages a `.dmg` from that same stapled bundle — a staging folder holding `Harness.app` plus an`/Applications` symlink, imaged with `hdiutil` — then signs, notarizes and staples the image too.+ The final ZIP is rebuilt during DMG notarization; both artifacts must finish and pass their checks.9. Uploads **both** artifacts with a year-long immutable `Cache-Control` (see "GCS layout").10. Download-merge-reuploads `metadata.json` in a single write, touching only that build's two keys.diff --git a/desktop/scripts/upload-desktop.sh b/desktop/scripts/upload-desktop.shindex 2db23fde8..08542a03f 100755--- a/desktop/scripts/upload-desktop.sh+++ b/desktop/scripts/upload-desktop.sh@@ -162,7 +162,15 @@ if [ "$DO_NOTARIZE" -eq 1 ]; thencommand -v xcrun >/dev/null 2>&1 || { echo "error: xcrun not found — install Xcode command line tools" >&2; exit 1; }fi-cleanup() { rm -f "${SRC:-}" "${DST:-}"; [ -n "${DMG_STAGE:-}" ] && rm -rf "$DMG_STAGE"; return 0; }+ZIP_PID=""+cleanup() {+ # Reap the compression worker even when DMG creation/notarization fails. The+ # coordinator's existing deadline still bounds this script and its children.+ [ -z "$ZIP_PID" ] || wait "$ZIP_PID" || true+ rm -f "${SRC:-}" "${DST:-}"+ [ -n "${DMG_STAGE:-}" ] && rm -rf "$DMG_STAGE"+ return 0+}trap cleanup EXIT# --- Step 1: resolve the version (source of truth = remote metadata.json) ---@@ -225,10 +233,18 @@ echo ">> packaging $ZIP"package_zip() {( cd "$(dirname "$APP_BUNDLE")" && ditto -c -k --sequesterRsrc --keepParent "$@" "$(basename "$APP_BUNDLE")" "$ZIP" )}+packaging_time() {+ local name="$1" started="$SECONDS" status=0+ shift+ "$@" || status=$?+ # Only fixed phase names and elapsed seconds: never arguments or credentials.+ echo ">> packaging timing: $name start=$started end=$SECONDS status=$status" >&2+ return "$status"+}if [ "$DO_NOTARIZE" -eq 1 ]; then# This upload is discarded after stapling. Spend less CPU compressing it;# the final downloadable ZIP below still uses ditto's normal compression.- package_zip --zlibCompressionLevel 1+ packaging_time notarization-zip package_zip --zlibCompressionLevel 1elsepackage_zipfi@@ -243,7 +259,7 @@ if [ "$DO_NOTARIZE" -eq 1 ]; then# `submit --wait` exits 0 once it has a TERMINAL status, even if that status is "Invalid" — a# rejected submission is not a tool failure as far as its own exit code is concerned, so the# verdict has to be read out of the response instead of trusted from $?.- NOTARY_JSON="$(xcrun notarytool submit "$ZIP" --keychain-profile "$NOTARY_PROFILE" --wait --output-format json)"+ NOTARY_JSON="$(packaging_time app-notarization xcrun notarytool submit "$ZIP" --keychain-profile "$NOTARY_PROFILE" --wait --output-format json)"echo "$NOTARY_JSON"NOTARY_STATUS="$(printf '%s' "$NOTARY_JSON" | python3 -c 'import json,sys; print(json.load(sys.stdin).get("status",""))' 2>/dev/null || true)"NOTARY_ID="$(printf '%s' "$NOTARY_JSON" | python3 -c 'import json,sys; print(json.load(sys.stdin).get("id",""))' 2>/dev/null || true)"@@ -255,10 +271,6 @@ if [ "$DO_NOTARIZE" -eq 1 ]; thenecho ">> stapling notarization ticket to $APP_BUNDLE"xcrun stapler staple "$APP_BUNDLE"- echo ">> re-packaging $ZIP with the stapled ticket"- rm -f "$ZIP"- package_zip-# Fails closed rather than silently shipping a build Gatekeeper would reject on someone else's Mac.spctl -a -vv --type execute "$APP_BUNDLE" || {echo "error: Gatekeeper assessment failed on the stapled bundle" >&2@@ -282,7 +294,7 @@ echo ">> packaging $DMG"rm -f "$DMG"ditto --clone "$APP_BUNDLE" "$DMG_STAGE/$(basename "$APP_BUNDLE")"ln -s /Applications "$DMG_STAGE/Applications"-hdiutil create -quiet -srcfolder "$DMG_STAGE" -volname "Harness" -fs HFS+ -format UDZO -ov "$DMG"+packaging_time dmg-create hdiutil create -quiet -srcfolder "$DMG_STAGE" -volname "Harness" -fs HFS+ -format UDZO -ov "$DMG"# Sign the image itself. The app inside is already signed and stapled; this is about the FILE the# browser hands the user — an unsigned disk image is what turns a clean install into a scary one.@@ -290,11 +302,19 @@ echo ">> signing $DMG ($SIGN_IDENTITY)"codesign --sign "$SIGN_IDENTITY" --timestamp "$DMG"if [ "$DO_NOTARIZE" -eq 1 ]; then+ # The stapled app is now immutable. Compress its final ZIP during the DMG's+ # network/service wait, after DMG compression has finished competing for CPU.+ # Neither artifact is uploaded until this worker and every DMG check pass.+ echo ">> re-packaging $ZIP with the stapled ticket during DMG notarization"+ rm -f "$ZIP"+ packaging_time final-zip package_zip &+ ZIP_PID=$!+# A second submission, and it cannot be avoided: stapling only attaches a ticket to the exact thing# that was submitted, so the zip's ticket does not cover this image. Apple has already seen this# app's cdhash from the first submission, so this pass is usually the quick one.echo ">> submitting the dmg for notarization (keychain profile: $NOTARY_PROFILE)"- DMG_NOTARY_JSON="$(xcrun notarytool submit "$DMG" --keychain-profile "$NOTARY_PROFILE" --wait --output-format json)"+ DMG_NOTARY_JSON="$(packaging_time dmg-notarization xcrun notarytool submit "$DMG" --keychain-profile "$NOTARY_PROFILE" --wait --output-format json)"echo "$DMG_NOTARY_JSON"# Same trap as the zip above: `--wait` exits 0 on any TERMINAL status, "Invalid" included, so the# verdict is read out of the JSON rather than inferred from $?.@@ -312,6 +332,12 @@ if [ "$DO_NOTARIZE" -eq 1 ]; thenecho "error: Gatekeeper assessment failed on the dmg" >&2exit 1}+ if ! packaging_time final-zip-wait wait "$ZIP_PID"; then+ ZIP_PID=""+ echo "error: final ZIP compression failed — nothing uploaded" >&2+ exit 1+ fi+ ZIP_PID=""elseecho ">> skipping dmg notarization (--no-notarize) — Developer ID signed only"fidiff --git a/scripts/tests/test_macos_upload.py b/scripts/tests/test_macos_upload.pynew file mode 100644index 000000000..20c479f02--- /dev/null+++ b/scripts/tests/test_macos_upload.py@@ -0,0 +1,193 @@+"""Exercise the real uploader with disposable signing/storage tool processes."""+import json+import os+from pathlib import Path+import subprocess+import sys+import tempfile+import unittest++ROOT = Path(__file__).resolve().parents[2]+UPLOADER = ROOT / "desktop/scripts/upload-desktop.sh"++TOOL = r'''+import json, os, shutil, sys, time+from pathlib import Path+root = Path(os.environ["UPLOAD_FIXTURE"])+tool = Path(sys.argv[0]).name+args = sys.argv[1:]+mode = os.environ.get("UPLOAD_FAILURE", "")+notarize = os.environ.get("UPLOAD_NOTARIZE", "1") == "1"+def mark(name):+ (root / name).touch()+def wait_for(name):+ end = time.monotonic() + 4+ while not (root / name).exists():+ if time.monotonic() >= end:+ raise RuntimeError("did not overlap: waiting for " + name)+ time.sleep(.01)+if tool == "curl":+ print('{}')+elif tool == "plutil":+ print('1.2.3')+elif tool == "ditto":+ source, target = map(Path, args[-2:])+ if "-c" not in args:+ if notarize:+ assert (source / 'ticket').exists()+ assert (root / 'app-approved').exists()+ shutil.copytree(source, target)+ (root / 'stage-path').write_text(str(target.parent))+ elif "--zlibCompressionLevel" in args:+ assert not (source / 'ticket').exists()+ target.write_text('unstapled submission')+ else:+ if notarize:+ assert (source / 'ticket').exists()+ assert (root / 'app-approved').exists()+ mark('zip-started')+ if notarize:+ wait_for('dmg-terminal')+ mark('zip-finished')+ if mode == 'zip':+ sys.exit(9)+ target.write_text('stapled final zip' if notarize else 'signed zip')+elif tool == "hdiutil":+ stage = Path(args[args.index('-srcfolder') + 1])+ if notarize:+ assert (stage / 'Harness.app/ticket').exists()+ if mode == 'dmg-create':+ sys.exit(8)+ Path(args[-1]).write_text('stapled app in dmg' if notarize else 'signed app in dmg')+elif tool == "xcrun":+ if args[:2] == ['notarytool', 'submit']:+ dmg = args[2].endswith('.dmg')+ if dmg:+ wait_for('zip-started')+ mark('dmg-notary-started')+ if mode in ('dmg-notary', 'dmg-tool'):+ mark('dmg-terminal')+ if mode == 'dmg-tool':+ sys.exit(7)+ invalid = mode == ('dmg-notary' if dmg else 'app-notary')+ print(json.dumps({'id': 'fixture', 'status': 'Invalid' if invalid else 'Accepted'}))+ elif args[:2] == ['stapler', 'staple']:+ target = Path(args[2])+ if target.is_dir():+ (target / 'ticket').touch()+ else:+ target.write_text(target.read_text() + ' and dmg ticket')+elif tool == "spctl":+ if args[-1].endswith('.dmg'):+ mark('dmg-terminal')+ if mode == 'dmg-gatekeeper':+ sys.exit(6)+ mark('dmg-approved')+ else:+ if mode == 'app-gatekeeper':+ sys.exit(6)+ mark('app-approved')+elif tool == "gcloud":+ if args[:2] == ['storage', 'ls']:+ sys.exit(1)+ if args[:2] == ['storage', 'cp']:+ source, target = args[-2:]+ if source.startswith('gs://'):+ sys.exit(1)+ assert (root / 'zip-finished').exists(), 'uploaded before compression finished'+ if notarize:+ assert (root / 'dmg-approved').exists(), 'uploaded before DMG verification'+ uploads = root / 'uploads'+ uploads.mkdir(exist_ok=True)+ destination = uploads / target.rsplit('/', 1)[-1]+ shutil.copyfile(source, destination)+elif tool not in ('flutter', 'codesign'):+ raise AssertionError('unexpected tool: ' + tool)+'''+++class MacOSUploadTests(unittest.TestCase):+ def setUp(self):+ self.temp = tempfile.TemporaryDirectory()+ self.addCleanup(self.temp.cleanup)+ self.root = Path(self.temp.name) / "fixture with spaces"+ self.root.mkdir()+ self.bin = self.root / "bin"+ self.bin.mkdir()+ for name in ("curl", "plutil", "ditto", "hdiutil", "xcrun", "spctl", "gcloud", "flutter", "codesign"):+ path = self.bin / name+ path.write_text(f"#!{sys.executable}\n" + TOOL)+ path.chmod(0o755)+ self.app = self.root / "Harness.app"+ self.app.mkdir()+ self.output = self.root / "output"+ self.env = dict(os.environ, PATH=str(self.bin) + os.pathsep + os.environ["PATH"],+ APP_BUNDLE=str(self.app), OUTPUT_DIR=str(self.output),+ UPLOAD_FIXTURE=str(self.root), GCS_BUCKET="fixture-bucket",+ GCS_PATH="fixture/Harness.zip", DMG_GCS_PATH="fixture/Harness.dmg",+ METADATA_PATH="fixture/metadata.json")++ def upload(self, failure="", notarize=True):+ options = [] if notarize else ["--no-notarize"]+ result = subprocess.run(["bash", str(UPLOADER), "--no-build", "1.2.3", *options],+ env=dict(self.env, UPLOAD_FAILURE=failure, UPLOAD_NOTARIZE=str(int(notarize))),+ text=True, capture_output=True, timeout=12)+ stage = self.root / "stage-path"+ if stage.exists():+ self.assertFalse(Path(stage.read_text()).exists(), "DMG staging directory leaked")+ return result++ def test_compression_and_dmg_notarization_overlap_but_upload_waits_for_both(self):+ result = self.upload()+ self.assertEqual(result.returncode, 0, result.stdout + result.stderr)+ self.assertTrue((self.root / "dmg-notary-started").exists())+ uploads = self.root / "uploads"+ self.assertEqual((uploads / "Harness.zip").read_text(), "stapled final zip")+ self.assertEqual((uploads / "Harness.dmg").read_text(), "stapled app in dmg and dmg ticket")+ self.assertEqual(set(json.loads((uploads / "metadata.json").read_text())),+ {"desktop-macos", "desktop-macos-dmg"})+ self.assertIn("packaging timing: final-zip ", result.stderr)+ self.assertIn("packaging timing: dmg-notarization ", result.stderr)++ def test_failed_zip_prevents_all_uploads(self):+ result = self.upload("zip")+ self.assertNotEqual(result.returncode, 0)+ self.assertIn("final ZIP compression failed", result.stderr)+ self.assertFalse((self.root / "uploads").exists())++ def test_dmg_failures_reap_the_compressor_before_returning(self):+ for failure in ("dmg-notary", "dmg-tool", "dmg-gatekeeper"):+ with self.subTest(failure=failure):+ # Each failure runs in its own disposable fixture.+ case = MacOSUploadTests()+ case.setUp()+ try:+ result = case.upload(failure)+ self.assertNotEqual(result.returncode, 0)+ self.assertTrue((case.root / "zip-finished").exists())+ self.assertFalse((case.root / "uploads").exists())+ finally:+ case.doCleanups()++ def test_app_rejections_stop_before_final_artifacts(self):+ for failure in ("app-notary", "app-gatekeeper", "dmg-create"):+ with self.subTest(failure=failure):+ case = MacOSUploadTests()+ case.setUp()+ try:+ result = case.upload(failure)+ self.assertNotEqual(result.returncode, 0)+ self.assertFalse((case.root / "zip-started").exists())+ self.assertFalse((case.root / "uploads").exists())+ finally:+ case.doCleanups()++ def test_explicit_no_notarize_keeps_its_existing_artifacts(self):+ result = self.upload(notarize=False)+ self.assertEqual(result.returncode, 0, result.stdout + result.stderr)+ self.assertFalse((self.root / "dmg-notary-started").exists())+ self.assertEqual((self.root / "uploads/Harness.zip").read_text(), "signed zip")+++if __name__ == "__main__":+ unittest.main()