Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.
TypeScript agentaiai-agentsall-in-onecrmcrm-systememaillinearmcpmessagingnotesnotionofficerustslackslack-alternativestartupstartup-toolstypescriptworkspace
perf(preview): parallelize the CI build phase into four lanes (#4979)
teob462913aed51
diff --git a/.github/workflows/preview-fly.yml b/.github/workflows/preview-fly.ymlindex 6b71a78d4a..1813de26c2 100644--- a/.github/workflows/preview-fly.yml+++ b/.github/workflows/preview-fly.yml@@ -11,7 +11,7 @@ name: Fly Previewjobs:deploy:if: github.event.pull_request.head.repo.full_name == github.repository && contains(github.event.pull_request.labels.*.name, 'preview')- runs-on: namespace-profile-linux-rust-ci;overrides.cache-tag=fly-preview+ runs-on: namespace-profile-linux-mid;overrides.cache-tag=fly-previewpermissions:contents: readpull-requests: write@@ -40,8 +40,6 @@ jobs:run: rm -rf target/*/debug/incremental target/debug/incremental- name: Set up Namespace Docker builderuses: namespacelabs/nscloud-setup-buildx-action@d059ed7184f0bc7c8b27e8810cea153d02bcc6dd- - name: Build compose service images (fail fast)- run: docker compose --project-directory . -p macro -f docker/docker-compose.yml build search sync_service websocket_service lexical_service ai_editing_worker- name: Setup Nixuses: './.github/actions/setup-nix'- name: Setup dev shell + web deps@@ -74,23 +72,140 @@ jobs:elseecho "::warning::Namespace remote sccache setup failed; using local cache fallback"fi- - name: Build service binaries+ - name: Setup flyctl+ uses: superfly/flyctl-actions/setup-flyctl@ed8efb33836e8b2096c7fd3ba1c8afe303ebbff1+ - name: Build artifacts (parallel lanes)run: |+ set -euo pipefail+ lanes="$RUNNER_TEMP/lanes"+ mkdir -p "$lanes"++ cat > "$lanes/cargo.sh" <<'LANE'+ set -euo pipefailcargo x zigbuildsccache --show-stats- - name: Build xtask_local (runs inside the preview VM)- run: cargo zigbuild --release --target x86_64-unknown-linux-gnu.2.36 -p xtask_local- - name: Build frontend bundle (same-origin)- run: bun run --bun build- env:- MODE: development- NODE_ENV: production- VITE_LOCAL_SERVERS: ALL- VITE_LOCAL_BACKEND_ORIGIN: same-origin- working-directory: apps/web- - name: Bake init snapshot- run: |+ cargo zigbuild --release --target x86_64-unknown-linux-gnu.2.36 -p xtask_local+ LANE++ cat > "$lanes/web.sh" <<'LANE'set -euo pipefail+ cd apps/web+ export MODE=development NODE_ENV=production+ export VITE_LOCAL_SERVERS=ALL VITE_LOCAL_BACKEND_ORIGIN=same-origin+ bun run --bun build+ LANE++ # Instant lane: everything Fly-side that needs no build output.+ # The template fork must fire within seconds of the job starting —+ # its ~2 min server-side hydration then finishes inside the compose+ # build. (It previously sat after the premirror pushes, which put+ # it at the END of the build phase and re-gated the deploy.)+ # Entirely best-effort: everything in the fly lane (app create,+ # fork kick, token mint) has an authoritative fallback in the+ # deploy step, so a transient Fly API failure must not fail the+ # build. A child `bash -e` (not a `|| `-guarded function: errexit+ # is suppressed inside those) keeps its internal fail-fast so a+ # failed app_id fetch can't mint a garbage token.+ cat > "$lanes/fly.sh" <<'LANE'+ set -euo pipefail+ bash -euo pipefail "$(dirname "$0")/fly-prep.sh" \+ || echo "fly prep failed (non-fatal): the deploy step falls back" >&2+ LANE++ cat > "$lanes/fly-prep.sh" <<'LANE'+ flyctl status --app "$APP_NAME" >/dev/null 2>&1 \+ || flyctl apps create "$APP_NAME" --org "$FLY_ORG" \+ || flyctl status --app "$APP_NAME" >/dev/null+ # Kick the template-volume fork (see the deploy step's+ # volume-state block, which stays the authoritative fallback).+ if ! flyctl volumes list --app "$APP_NAME" --json 2>/dev/null \+ | jq -e 'map(select(.name == "docker_data")) | length > 0' >/dev/null; then+ tpl_src=$(flyctl volumes list --app "$TEMPLATE_APP" --json 2>/dev/null \+ | jq -r '[.[] | select((.name | startswith("tpl")) and .state == "created" and .attached_machine_id == null)]+ | sort_by(.created_at) | last | .id // empty' 2>/dev/null) || tpl_src=+ if [ -n "$tpl_src" ]; then+ curl -sf -X POST "https://api.machines.dev/v1/apps/$APP_NAME/volumes" \+ -H "Authorization: Bearer $FLY_API_TOKEN" \+ -H "Content-Type: application/json" \+ -d "{\"name\":\"docker_data\",\"region\":\"ewr\",\"source_volume_id\":\"$tpl_src\"}" \+ | jq -r '"early template fork: \(.id // "failed") (\(.state // "-"))"' \+ || echo "early template fork failed; deploy step will fall back" >&2+ fi+ fi+ # Mint the machine's app-scoped read-only registry pull token+ # (macaroon attenuation, see the deploy step for the why) so the+ # deploy preamble doesn't pay the round-trips serially.+ app_id=$(curl -sf https://api.fly.io/graphql \+ -H "Authorization: Bearer $FLY_API_TOKEN" \+ -H "content-type: application/json" \+ -d "{\"query\":\"{ app(name: \\\"$APP_NAME\\\") { internalNumericId } }\"}" \+ | jq -re '.data.app.internalNumericId')+ now=$(date +%s)+ printf '[{"type":"Apps","body":{"apps":{"%s":"r"}}},{"type":"ValidityWindow","body":{"not_before":%d,"not_after":%d}}]' \+ "$app_id" $((now - 60)) $((now + 604800)) \+ | flyctl tokens attenuate > "$RUNNER_TEMP/registry-pull-token.next"+ chmod 600 "$RUNNER_TEMP/registry-pull-token.next"+ mv "$RUNNER_TEMP/registry-pull-token.next" "$RUNNER_TEMP/registry-pull-token"+ LANE++ cat > "$lanes/images.sh" <<'LANE'+ set -euo pipefail+ docker compose --project-directory . -p macro -f docker/docker-compose.yml build \+ search sync_service websocket_service lexical_service ai_editing_worker+ # The image BUILD above is fatal; the premirror below is a pure+ # optimization with an authoritative fallback (the deploy step's+ # mirror loop), so its failure only costs a slower push later.+ bash -euo pipefail "$(dirname "$0")/premirror.sh" \+ || echo "premirror failed (non-fatal): the deploy step mirrors authoritatively" >&2+ LANE++ # Premirror: content-addressed refs, identical scheme to the deploy+ # step's mirror loop, so its manifest-existence check skips these.+ # Best effort by design — anything skipped here (images only the+ # bake produces, like macro-local-runtime:dev) is pushed there.+ # The fly lane created the app long ago; the guard is a fallback.+ cat > "$lanes/premirror.sh" <<'LANE'+ flyctl status --app "$APP_NAME" >/dev/null 2>&1 \+ || flyctl apps create "$APP_NAME" --org "$FLY_ORG" \+ || flyctl status --app "$APP_NAME" >/dev/null+ flyctl auth docker+ registry="registry.fly.io/$APP_NAME"+ for img in $(docker compose --project-directory . -p macro \+ -f docker/docker-compose.yml config --images | sort -u) alpine:3; do+ if ! docker image inspect "$img" >/dev/null 2>&1 && ! docker pull "$img"; then+ echo "premirror: skipping $img (not local, not pullable)"+ continue+ fi+ id=$(docker image inspect -f '{{.Id}}' "$img")+ ref="$registry:img-$(echo "$img" | tr '/:' '__')-$(echo "$id" | cut -c8-19)"+ docker tag "$img" "$ref"+ if ! docker manifest inspect "$ref" >/dev/null 2>&1; then+ for _ in 1 2 3; do+ docker push "$ref" && break+ echo "premirror push of $ref failed, retrying" >&2+ sleep 15+ done+ fi+ done+ LANE++ # A cold `stack up --infra-only` runs the real init (migrate,+ # kickstart, Kafka topics, indices) and saves the content-addressed+ # snapshot the VM restores from. Infra only: the app services need+ # the Doppler-sourced env to boot, which this runner deliberately+ # lacks — the snapshot captures only the infra volumes anyway. Via+ # `just` (not the bare `cargo x` alias) because the justfile+ # enables the `local-stack` feature the Kafka provisioning needs.+ # The store lives on the cache volume for a zero-copy hit; every+ # snapshot is also stored as one tar in Namespace artifact storage+ # keyed by the same hash, so cache-volume onboarding misses+ # download the exact snapshot instead of paying for a full init.+ # Artifact access is an optimization: failures fall back to the+ # volume/cold-bake behavior. The final `snapshot --json` line lands+ # in $RUNNER_TEMP for the stage step.+ cat > "$lanes/bake.sh" <<'LANE'+ set -euo pipefail+ export CARGO_TARGET_DIR="$PWD/target/bake"mkdir -p "$MACRO_STACK_SNAPSHOT_DIR"status=$(just stack snapshot --json | tail -n 1)key=$(echo "$status" | jq -r '.key')@@ -108,7 +223,7 @@ jobs:rm -rf "$restored"mkdir -p "$restored"if nsc artifact download "$artifact" "$archive" \- && tar -xf "$archive" -C "$restored" \+ && tar -xf "$archive" -C "$restored" \&& [ -f "$restored/$key/manifest.json" ]; thenrm -rf "$root/$key"cp -a "$restored/$key" "$root/$key"@@ -151,6 +266,42 @@ jobs:|| echo "warning: failed to upload init snapshot artifact" >&2fiecho "$status" > "$RUNNER_TEMP/preview-snapshot.json"+ LANE++ declare -a pids=() names=()+ for name in fly cargo web images bake; do+ # setsid: each lane leads its own process group so fail-fast+ # can kill the whole tree (cargo/docker/bun children), not just+ # the wrapper shell.+ setsid bash "$lanes/$name.sh" > "$lanes/$name.log" 2>&1 &+ pids+=($!)+ names+=("$name")+ done+ fail=+ for _ in "${pids[@]}"; do+ # Bare `wait -n`: any child; explicit pids would error once a+ # job has already been reaped.+ if ! wait -n; then+ fail=1+ break+ fi+ done+ if [ -n "$fail" ]; then+ # Negative pids: signal each lane's whole process group.+ kill -- "${pids[@]/#/-}" 2>/dev/null || true+ wait || true+ fi+ for i in "${!names[@]}"; do+ echo "::group::lane: ${names[$i]}"+ cat "$lanes/${names[$i]}.log" || true+ echo "::endgroup::"+ done+ if [ -n "$fail" ]; then+ echo "a build lane failed (first failure aborts the rest); see lane logs above" >&2+ exit 1+ fi+ env:+ FLY_ORG: ${{ vars.FLY_ORG || secrets.FLY_ORG }}- name: Dump stack diagnosticsif: failure()run: |@@ -222,8 +373,6 @@ jobs:cp -p infra/preview/entrypoint.sh "$ctx/entrypoint.sh"cp -p infra/preview/Dockerfile "$ctx/Dockerfile"cp -p infra/preview/update.Dockerfile "$ctx/update.Dockerfile"- - name: Setup flyctl- uses: superfly/flyctl-actions/setup-flyctl@ed8efb33836e8b2096c7fd3ba1c8afe303ebbff1- name: Deploy to Flyrun: |set -euo pipefail@@ -250,27 +399,27 @@ jobs:# warm /var/lib/docker layer store published by a previous# successful deploy (see the "Publish template volume" step).# Measured cold, an empty volume costs ~11 min of image pulls at- # boot; a fork costs a ~2.5 min server-side hydration that runs- # concurrently with the mirror push below (the gate before- # machine creation waits for it). Cross-app forks need the- # Machines API: flyctl resolves volume ids app-locally. The boot- # manifest check makes the volume a pure cache, so every failure- # path here just falls back to the old empty-volume behavior.- fork_pending=- if ! flyctl volumes list --app "$APP_NAME" --json 2>/dev/null \- | jq -e 'map(select(.name == "docker_data")) | length > 0' >/dev/null; then+ # boot. The images lane normally kicked the fork minutes ago (its+ # ~2 min server-side hydration overlaps the whole build phase);+ # this block is the authoritative fallback for a missing or+ # failed early fork. Cross-app forks need the Machines API:+ # flyctl resolves volume ids app-locally. The boot manifest check+ # makes the volume a pure cache, so every failure path here just+ # falls back to the old empty-volume behavior.+ vol_state=$(flyctl volumes list --app "$APP_NAME" --json 2>/dev/null \+ | jq -r '[.[] | select(.name == "docker_data")][0].state // empty') || vol_state=+ if [ -z "$vol_state" ]; thentpl_src=$(flyctl volumes list --app "$TEMPLATE_APP" --json 2>/dev/null \| jq -r '[.[] | select((.name | startswith("tpl")) and .state == "created" and .attached_machine_id == null)]| sort_by(.created_at) | last | .id // empty' 2>/dev/null) || tpl_src=if [ -n "$tpl_src" ]; then- fork_state=$(curl -sf -X POST "https://api.machines.dev/v1/apps/$APP_NAME/volumes" \+ vol_state=$(curl -sf -X POST "https://api.machines.dev/v1/apps/$APP_NAME/volumes" \-H "Authorization: Bearer $FLY_API_TOKEN" \-H "Content-Type: application/json" \-d "{\"name\":\"docker_data\",\"region\":\"ewr\",\"source_volume_id\":\"$tpl_src\"}" \- | jq -r '.state // empty') || fork_state=- if [ -n "$fork_state" ]; then- echo "seeding docker_data from template volume $tpl_src (state: $fork_state)"- fork_pending=1+ | jq -r '.state // empty') || vol_state=+ if [ -n "$vol_state" ]; then+ echo "seeding docker_data from template volume $tpl_src (state: $vol_state)"elseecho "template fork failed; falling back to an empty volume" >&2fi@@ -285,15 +434,21 @@ jobs:# secrets, so scope matters). The org deploy token can't mint new# tokens (createLimitedAccessToken is not authorized), but macaroon# attenuation is pure client-side crypto: append an Apps caveat- # (numeric app id, mask "r") and a validity window to our own token.- app_id=$(curl -sf https://api.fly.io/graphql \- -H "Authorization: Bearer $FLY_API_TOKEN" \- -H "content-type: application/json" \- -d "{\"query\":\"{ app(name: \\\"$APP_NAME\\\") { internalNumericId } }\"}" \- | jq -re '.data.app.internalNumericId')- now=$(date +%s)- pull_token=$(printf '[{"type":"Apps","body":{"apps":{"%s":"r"}}},{"type":"ValidityWindow","body":{"not_before":%d,"not_after":%d}}]' \- "$app_id" $((now - 60)) $((now + 604800)) | flyctl tokens attenuate)+ # (numeric app id, mask "r") and a validity window to our own+ # token. The images lane normally minted it already; fall back to+ # minting inline when its file is absent.+ if [ -s "$RUNNER_TEMP/registry-pull-token" ]; then+ pull_token=$(cat "$RUNNER_TEMP/registry-pull-token")+ else+ app_id=$(curl -sf https://api.fly.io/graphql \+ -H "Authorization: Bearer $FLY_API_TOKEN" \+ -H "content-type: application/json" \+ -d "{\"query\":\"{ app(name: \\\"$APP_NAME\\\") { internalNumericId } }\"}" \+ | jq -re '.data.app.internalNumericId')+ now=$(date +%s)+ pull_token=$(printf '[{"type":"Apps","body":{"apps":{"%s":"r"}}},{"type":"ValidityWindow","body":{"not_before":%d,"not_after":%d}}]' \+ "$app_id" $((now - 60)) $((now + 604800)) | flyctl tokens attenuate)+ fisnapshot_key=$(jq -r '.key' "$RUNNER_TEMP/preview-snapshot.json")runtime_key=$(jq -r '.runtime_key' preview-ctx/deployment.json)@@ -334,6 +489,20 @@ jobs:fiecho "preview deployment mode: $mode (snapshot $snapshot_key)"+ # Keep the machine in demand for the WHOLE deploy, hot path+ # included: fly-proxy suspends a machine ~7 min after its last+ # proxied request, and SSH via hallpass is not proxy traffic — a+ # hot update with no browser traffic got its VM suspended+ # mid-session (the frozen `flyctl ssh console` then ate 52 min of+ # runner until the job timeout). Any response status counts as+ # traffic, and a request also resumes a suspended machine.+ (while :; do+ curl -s -o /dev/null --max-time 10 "https://$APP_NAME.fly.dev/" || true+ sleep 15+ done) &+ keepalive=$!+ trap 'kill "$keepalive" 2>/dev/null || true' EXIT+flyctl auth dockerregistry="registry.fly.io/$APP_NAME"# Mirror every stack image into the app's registry repo instead of@@ -356,10 +525,9 @@ jobs:--env-file "$envfile" \config --images | sort -u)echo "mirroring images:" $images- docker pull alpine:3+ docker image inspect alpine:3 >/dev/null 2>&1 || docker pull alpine:3mkdir -p preview-ctx/preload: > preview-ctx/preload/manifest.txt- : > "$RUNNER_TEMP/push-refs.txt"for img in $images alpine:3; do# Images the bake didn't leave in the daemon (snapshot cache hit,# or app-layer-only images like proxy/mailpit) get pulled here.@@ -371,10 +539,14 @@ jobs:ref="$registry:img-$(echo "$img" | tr '/:' '__')-$(echo "$id" | cut -c8-19)"docker tag "$img" "$ref"echo "$id $img $ref" >> preview-ctx/preload/manifest.txt- if ! docker manifest inspect "$ref" >/dev/null 2>&1; then- echo "$ref" >> "$RUNNER_TEMP/push-refs.txt"- fidone+ # The existence probes are pure network round-trips (and after the+ # premirror lane, almost all of them hit) — run them in parallel+ # instead of ~1s each serially.+ awk '{print $3}' preview-ctx/preload/manifest.txt \+ | xargs -P 8 -I {} sh -c \+ 'docker manifest inspect "{}" >/dev/null 2>&1 || echo "{}"' \+ > "$RUNNER_TEMP/push-refs.txt"echo "pushing $(wc -l < "$RUNNER_TEMP/push-refs.txt") of $(wc -l < preview-ctx/preload/manifest.txt) images"# The Fly registry occasionally aborts large uploads ("s3aws:# append to zero-size path unsupported") — retry; layers that made@@ -412,8 +584,11 @@ jobs:--app "$APP_NAME" --machine "$machine_id" --mode 0600 \|| hot_rc=$?if [ "$hot_rc" = 0 ]; then+ # timeout backstop: a suspended-mid-session VM freezes the ssh+ # client forever (measured: 52 min of hang). 10 min is triple+ # a normal hot apply; on expiry we fall back to rehydrate.hot_command="/srv/macro/bin/hot-update '$update_image' /tmp/macro-registry-token"- flyctl ssh console --app "$APP_NAME" --machine "$machine_id" \+ timeout 600 flyctl ssh console --app "$APP_NAME" --machine "$machine_id" \--command "$hot_command" || hot_rc=$?fiif [ "$hot_rc" = 0 ]; then@@ -429,12 +604,16 @@ jobs:flyctl secrets set --app "$APP_NAME" --stage \"DOPPLER_TOKEN=$DOPPLER_PREVIEW_TOKEN" \"REGISTRY_PULL_TOKEN=$pull_token"- if [ -n "$fork_pending" ]; then- # The mirror push above bought most of the hydration time; the- # volume must reach "created" before a machine can mount it. A+ if [ -z "$vol_state" ]; then+ # No volume and no fork landed (no template, or both fork+ # attempts failed): the old empty-volume path.+ flyctl volumes create docker_data --app "$APP_NAME" --region ewr --size 40 --yes+ elif [ "$vol_state" != created ]; then+ # A fork is still hydrating (usually already finished: the+ # images lane kicked it before the cargo lane even ended). The+ # volume must reach "created" before a machine can mount it; a# fork that never lands is replaced by an empty volume rather# than failing the deploy.- vol_state=for _ in $(seq 1 60); dovol_state=$(flyctl volumes list --app "$APP_NAME" --json \| jq -r '[.[] | select(.name == "docker_data")][0].state // empty')@@ -449,9 +628,6 @@ jobs:[ -z "$vol_id" ] || flyctl volumes destroy "$vol_id" --app "$APP_NAME" --yes || trueflyctl volumes create docker_data --app "$APP_NAME" --region ewr --size 40 --yesfi- elif ! flyctl volumes list --app "$APP_NAME" --json \- | jq -e 'map(select(.name == "docker_data")) | length > 0' >/dev/null; then- flyctl volumes create docker_data --app "$APP_NAME" --region ewr --size 40 --yesfi# Machines created before the volume existed can't take the mounts# config update. A destroyed/partial machine is recreated by deploy.@@ -464,27 +640,15 @@ jobs:image="$registry:${{ github.sha }}"docker build -t "$image" preview-ctxpush_image "$image"- # Keep the machine in demand while it boots: with zero inbound- # traffic, fly-proxy counts a health-failing machine as excess- # capacity and suspends it mid-boot (observed mid-docker-load).- # Any response status counts as traffic, and a request also- # resumes an already-suspended machine.- (while :; do- curl -s -o /dev/null --max-time 10 "https://$APP_NAME.fly.dev/" || true- sleep 15- done) &- keepalive=$!# First boot does real work before 8090 opens (pulling the stack# images, snapshot restore, compose up, FusionAuth's JVM) —# give the health check more runway than flyctl's default wait.- rc=0+ # The step-wide keepalive above holds the machine out of suspend.flyctl deploy --app "$APP_NAME" \--config infra/preview/fly.toml \--image "$image" \--wait-timeout 1800 \- --yes || rc=$?- kill "$keepalive" 2>/dev/null || true- exit "$rc"+ --yesenv:FLY_ORG: ${{ vars.FLY_ORG || secrets.FLY_ORG }}DOPPLER_PREVIEW_TOKEN: ${{ secrets.DOPPLER_PREVIEW_TOKEN }}diff --git a/infra/preview/entrypoint.sh b/infra/preview/entrypoint.shindex 37893aad2d..8234216251 100644--- a/infra/preview/entrypoint.sh+++ b/infra/preview/entrypoint.sh@@ -34,6 +34,17 @@ if ! docker info >/dev/null 2>&1; thenfilog "dockerd ready in $(($(date +%s) - t_boot))s"+# A template-forked volume carries the source deploy's containers; any with a+# restart policy auto-start with dockerd and can answer :8090 with the OLD+# stack — fly-proxy then reports the machine healthy before the real stack+# exists, and a visitor gets the stale app followed by a teardown blackout.+# Kill them before anything can serve; `stack up` recreates properly.+stale=$(docker ps -q)+if [ -n "$stale" ]; then+ log "killing $(echo "$stale" | wc -l) auto-started containers from the volume source"+ echo "$stale" | xargs -r docker kill >/dev/null 2>&1 || true+fi+# Pull the stack images from the app's Fly registry repo (CI mirrored them# there at deploy time). /var/lib/docker is a persistent volume, so the# manifest (image ID per tag) lets redeploys skip every image already in thediff --git a/tooling/xtask/crates/xtask_workflows/src/workflows/preview_fly.rs b/tooling/xtask/crates/xtask_workflows/src/workflows/preview_fly.rsindex 53415d7f2c..5e93e83d1a 100644--- a/tooling/xtask/crates/xtask_workflows/src/workflows/preview_fly.rs+++ b/tooling/xtask/crates/xtask_workflows/src/workflows/preview_fly.rs@@ -73,11 +73,15 @@ fn deploy() -> Job {"github.event.pull_request.head.repo.full_name == github.repository && \contains(github.event.pull_request.labels.*.name, 'preview')",))- // Dedicated cache volume pool (nix store + cargo target + snapshots):- // see PREVIEW_CACHE_TAG for why sharing the check/test jobs' pool was- // measured cold on both layers (different --target = disjoint sccache- // keys, and their volumes never carry a cargo target dir).- .runs_on(runners::Runner::RustCi.with_cache_tag(vars::PREVIEW_CACHE_TAG))+ // linux-mid for the extra cores: five build lanes (zigbuild, compose+ // image builds, frontend, snapshot bake) genuinely contend on the+ // 8-core rust-ci shape. The cache volume follows the explicit tag,+ // not the profile, so this keeps the dedicated preview pool (nix+ // store + cargo target + snapshots) it has always used: see+ // PREVIEW_CACHE_TAG for why sharing the check/test jobs' pool was+ // measured cold on both layers (different --target = disjoint+ // sccache keys, and their volumes never carry a cargo target dir).+ .runs_on(runners::Runner::Mid.with_cache_tag(vars::PREVIEW_CACHE_TAG))// Backstop against hangs: worst honest case is ~15 min cold builds +// ~6 min cold mirror push + the 30 min flyctl wait cap. Anything past// an hour is a stuck step, not a slow deploy (a hung log fetch once@@ -124,52 +128,15 @@ fn deploy() -> Job {"d059ed7184f0bc7c8b27e8810cea153d02bcc6dd",), // v0.0.23)- // Fail fast: the Docker-built images are the most fragile part of a- // fresh bring-up (stale local images mask their rot) and need nothing- // from nix/cargo — build them before the expensive toolchain setup so- // a broken Dockerfile fails in ~2 minutes, not ~12. This is also the- // ONLY thing that builds them now: the infra-only bake never starts- // the app layer, so the registry mirror ships exactly what's built here.- .add_step(Step::new("Build compose service images (fail fast)").run(- "docker compose --project-directory . -p macro -f docker/docker-compose.yml build \- search sync_service websocket_service lexical_service ai_editing_worker",- )).add_step(steps::setup_nix()).add_step(steps::setup_reqs_web("Setup dev shell + web deps", false)).add_step(steps::configure_namespace_sccache(vars::PREVIEW_SCCACHE_NAME,))- .add_step(Step::new("Build service binaries").run(indoc::indoc! {r#"- cargo x zigbuild- sccache --show-stats- "#}))- .add_step(- // xtask_local, not the dependency-free `xtask` launcher (which- // just re-invokes cargo — useless in the VM). Deliberately WITHOUT- // the `local-stack` feature: the VM restores the baked snapshot,- // whose Kafka volume already carries the topics, so it never- // provisions Kafka — and skipping rdkafka means no- // dynamically-linked librdkafka to ship into the VM. zigbuild,- // not a host build: the nix dev shell links the /nix/store ELF- // interpreter, which doesn't exist in the VM image (exit 127,- // "required file not found").- Step::new("Build xtask_local (runs inside the preview VM)").run(- "cargo zigbuild --release --target x86_64-unknown-linux-gnu.2.36 -p xtask_local",- ),- )- .add_step(- Step::new("Build frontend bundle (same-origin)")- .run("bun run --bun build")- .working_directory(xtask_paths::repo_dir!("apps/web"))- .add_env(("MODE", "development"))- .add_env(("NODE_ENV", "production"))- .add_env(("VITE_LOCAL_SERVERS", "ALL"))- .add_env(("VITE_LOCAL_BACKEND_ORIGIN", "same-origin")),- )- .add_step(bake_snapshot())+ .add_step(setup_flyctl())+ .add_step(build_artifacts()).add_step(dump_stack_diagnostics()).add_step(stage_context())- .add_step(setup_flyctl()).add_step(deploy_to_fly()).add_step(dump_fly_diagnostics()).add_step(dump_boot_timings())@@ -206,87 +173,264 @@ fn dump_boot_timings() -> Step<Run> {"#})}-/// A cold `stack up --infra-only` on the runner runs the real init (migrate,-/// kickstart, Kafka topics, indices) and saves the content-addressed snapshot-/// the VM restores from. Infra only: the app services need the Doppler-sourced-/// env to boot, which this runner deliberately lacks — the snapshot captures-/// only the infra volumes anyway. Via `just` (not the bare `cargo x` alias)-/// because the justfile enables the `local-stack` feature the Kafka-/// provisioning needs.+/// Every build artifact in one step, five lanes in parallel. The lanes are+/// independent by construction:///-/// The store lives on the cache volume (MACRO_STACK_SNAPSHOT_DIR) for a-/// zero-copy hit. Every content-addressed snapshot is also stored as one tar in-/// Namespace artifact storage, keyed by the same hash. Cache-volume onboarding-/// misses therefore download the exact snapshot instead of paying for a full-/// init again. Artifact access is an optimization: download/upload failures-/// fall back to the existing volume/cold-bake behavior.+/// - `fly`: app create, template-volume fork kick, and pull-token mint —+/// nothing here needs build output, and the fork's ~2 min server-side+/// hydration must start within seconds to finish inside the compose+/// build instead of gating machine creation.+/// - `cargo`: service binaries then xtask_local, sequential because they+/// share the workspace target dir (cargo's build-dir lock would serialize+/// them anyway). xtask_local deliberately WITHOUT the `local-stack`+/// feature (the VM restores the baked snapshot so it never provisions+/// Kafka; skipping rdkafka means no dynamically-linked librdkafka in the+/// VM) and via zigbuild, not a host build (the nix dev shell links the+/// /nix/store ELF interpreter, which doesn't exist in the VM image).+/// - `web`: the same-origin frontend bundle.+/// - `images`: the compose-built app images (the most fragile part of a+/// fresh bring-up — a broken Dockerfile still fails the step early via+/// fail-fast wait), then a best-effort premirror push to the app's Fly+/// registry so the network transfer overlaps the cargo lane instead of+/// serializing after it. The deploy step's mirror loop stays+/// authoritative: its manifest-existence check skips whatever was+/// premirrored here and pushes the rest (e.g. the bake-built runtime+/// image).+/// - `bake`: the init snapshot (see the lane script header for details),+/// under its own CARGO_TARGET_DIR so its host xtask build doesn't take+/// the workspace build-dir lock the cargo lane holds.///-/// The final `snapshot --json` line lands in $RUNNER_TEMP for the stage step,-/// which ships exactly that key's directory.-fn bake_snapshot() -> Step<Run> {- Step::new("Bake init snapshot").run(indoc::indoc! {r#"- set -euo pipefail- mkdir -p "$MACRO_STACK_SNAPSHOT_DIR"- status=$(just stack snapshot --json | tail -n 1)- key=$(echo "$status" | jq -r '.key')- root=$(echo "$status" | jq -r '.root')- artifact="macro-preview/init-snapshots/${key}.tar"- archive="$RUNNER_TEMP/init-snapshot-${key}.tar"- artifact_hit=-- # The Namespace cache-volume pool has an onboarding period where a- # runner can receive an empty fork. Artifact storage is the durable- # fallback: restore into a temporary directory and only publish it to- # the live store after validating the embedded manifest through xtask.- if ! echo "$status" | jq -e '.present' >/dev/null; then- restored="$RUNNER_TEMP/restored-init-snapshot"- rm -rf "$restored"- mkdir -p "$restored"- if nsc artifact download "$artifact" "$archive" \- && tar -xf "$archive" -C "$restored" \- && [ -f "$restored/$key/manifest.json" ]; then- rm -rf "$root/$key"- cp -a "$restored/$key" "$root/$key"+/// Lanes log to files and are dumped as collapsible groups afterwards;+/// `wait -n` fails the step on the first lane failure and kills the rest.+fn build_artifacts() -> Step<Run> {+ Step::new("Build artifacts (parallel lanes)")+ .run(indoc::indoc! {r#"+ set -euo pipefail+ lanes="$RUNNER_TEMP/lanes"+ mkdir -p "$lanes"++ cat > "$lanes/cargo.sh" <<'LANE'+ set -euo pipefail+ cargo x zigbuild+ sccache --show-stats+ cargo zigbuild --release --target x86_64-unknown-linux-gnu.2.36 -p xtask_local+ LANE++ cat > "$lanes/web.sh" <<'LANE'+ set -euo pipefail+ cd apps/web+ export MODE=development NODE_ENV=production+ export VITE_LOCAL_SERVERS=ALL VITE_LOCAL_BACKEND_ORIGIN=same-origin+ bun run --bun build+ LANE++ # Instant lane: everything Fly-side that needs no build output.+ # The template fork must fire within seconds of the job starting —+ # its ~2 min server-side hydration then finishes inside the compose+ # build. (It previously sat after the premirror pushes, which put+ # it at the END of the build phase and re-gated the deploy.)+ # Entirely best-effort: everything in the fly lane (app create,+ # fork kick, token mint) has an authoritative fallback in the+ # deploy step, so a transient Fly API failure must not fail the+ # build. A child `bash -e` (not a `|| `-guarded function: errexit+ # is suppressed inside those) keeps its internal fail-fast so a+ # failed app_id fetch can't mint a garbage token.+ cat > "$lanes/fly.sh" <<'LANE'+ set -euo pipefail+ bash -euo pipefail "$(dirname "$0")/fly-prep.sh" \+ || echo "fly prep failed (non-fatal): the deploy step falls back" >&2+ LANE++ cat > "$lanes/fly-prep.sh" <<'LANE'+ flyctl status --app "$APP_NAME" >/dev/null 2>&1 \+ || flyctl apps create "$APP_NAME" --org "$FLY_ORG" \+ || flyctl status --app "$APP_NAME" >/dev/null+ # Kick the template-volume fork (see the deploy step's+ # volume-state block, which stays the authoritative fallback).+ if ! flyctl volumes list --app "$APP_NAME" --json 2>/dev/null \+ | jq -e 'map(select(.name == "docker_data")) | length > 0' >/dev/null; then+ tpl_src=$(flyctl volumes list --app "$TEMPLATE_APP" --json 2>/dev/null \+ | jq -r '[.[] | select((.name | startswith("tpl")) and .state == "created" and .attached_machine_id == null)]+ | sort_by(.created_at) | last | .id // empty' 2>/dev/null) || tpl_src=+ if [ -n "$tpl_src" ]; then+ curl -sf -X POST "https://api.machines.dev/v1/apps/$APP_NAME/volumes" \+ -H "Authorization: Bearer $FLY_API_TOKEN" \+ -H "Content-Type: application/json" \+ -d "{\"name\":\"docker_data\",\"region\":\"ewr\",\"source_volume_id\":\"$tpl_src\"}" \+ | jq -r '"early template fork: \(.id // "failed") (\(.state // "-"))"' \+ || echo "early template fork failed; deploy step will fall back" >&2+ fi+ fi+ # Mint the machine's app-scoped read-only registry pull token+ # (macaroon attenuation, see the deploy step for the why) so the+ # deploy preamble doesn't pay the round-trips serially.+ app_id=$(curl -sf https://api.fly.io/graphql \+ -H "Authorization: Bearer $FLY_API_TOKEN" \+ -H "content-type: application/json" \+ -d "{\"query\":\"{ app(name: \\\"$APP_NAME\\\") { internalNumericId } }\"}" \+ | jq -re '.data.app.internalNumericId')+ now=$(date +%s)+ printf '[{"type":"Apps","body":{"apps":{"%s":"r"}}},{"type":"ValidityWindow","body":{"not_before":%d,"not_after":%d}}]' \+ "$app_id" $((now - 60)) $((now + 604800)) \+ | flyctl tokens attenuate > "$RUNNER_TEMP/registry-pull-token.next"+ chmod 600 "$RUNNER_TEMP/registry-pull-token.next"+ mv "$RUNNER_TEMP/registry-pull-token.next" "$RUNNER_TEMP/registry-pull-token"+ LANE++ cat > "$lanes/images.sh" <<'LANE'+ set -euo pipefail+ docker compose --project-directory . -p macro -f docker/docker-compose.yml build \+ search sync_service websocket_service lexical_service ai_editing_worker+ # The image BUILD above is fatal; the premirror below is a pure+ # optimization with an authoritative fallback (the deploy step's+ # mirror loop), so its failure only costs a slower push later.+ bash -euo pipefail "$(dirname "$0")/premirror.sh" \+ || echo "premirror failed (non-fatal): the deploy step mirrors authoritatively" >&2+ LANE++ # Premirror: content-addressed refs, identical scheme to the deploy+ # step's mirror loop, so its manifest-existence check skips these.+ # Best effort by design — anything skipped here (images only the+ # bake produces, like macro-local-runtime:dev) is pushed there.+ # The fly lane created the app long ago; the guard is a fallback.+ cat > "$lanes/premirror.sh" <<'LANE'+ flyctl status --app "$APP_NAME" >/dev/null 2>&1 \+ || flyctl apps create "$APP_NAME" --org "$FLY_ORG" \+ || flyctl status --app "$APP_NAME" >/dev/null+ flyctl auth docker+ registry="registry.fly.io/$APP_NAME"+ for img in $(docker compose --project-directory . -p macro \+ -f docker/docker-compose.yml config --images | sort -u) alpine:3; do+ if ! docker image inspect "$img" >/dev/null 2>&1 && ! docker pull "$img"; then+ echo "premirror: skipping $img (not local, not pullable)"+ continue+ fi+ id=$(docker image inspect -f '{{.Id}}' "$img")+ ref="$registry:img-$(echo "$img" | tr '/:' '__')-$(echo "$id" | cut -c8-19)"+ docker tag "$img" "$ref"+ if ! docker manifest inspect "$ref" >/dev/null 2>&1; then+ for _ in 1 2 3; do+ docker push "$ref" && break+ echo "premirror push of $ref failed, retrying" >&2+ sleep 15+ done+ fi+ done+ LANE++ # A cold `stack up --infra-only` runs the real init (migrate,+ # kickstart, Kafka topics, indices) and saves the content-addressed+ # snapshot the VM restores from. Infra only: the app services need+ # the Doppler-sourced env to boot, which this runner deliberately+ # lacks — the snapshot captures only the infra volumes anyway. Via+ # `just` (not the bare `cargo x` alias) because the justfile+ # enables the `local-stack` feature the Kafka provisioning needs.+ # The store lives on the cache volume for a zero-copy hit; every+ # snapshot is also stored as one tar in Namespace artifact storage+ # keyed by the same hash, so cache-volume onboarding misses+ # download the exact snapshot instead of paying for a full init.+ # Artifact access is an optimization: failures fall back to the+ # volume/cold-bake behavior. The final `snapshot --json` line lands+ # in $RUNNER_TEMP for the stage step.+ cat > "$lanes/bake.sh" <<'LANE'+ set -euo pipefail+ export CARGO_TARGET_DIR="$PWD/target/bake"+ mkdir -p "$MACRO_STACK_SNAPSHOT_DIR"status=$(just stack snapshot --json | tail -n 1)+ key=$(echo "$status" | jq -r '.key')+ root=$(echo "$status" | jq -r '.root')+ artifact="macro-preview/init-snapshots/${key}.tar"+ archive="$RUNNER_TEMP/init-snapshot-${key}.tar"+ artifact_hit=++ # The Namespace cache-volume pool has an onboarding period where a+ # runner can receive an empty fork. Artifact storage is the durable+ # fallback: restore into a temporary directory and only publish it to+ # the live store after validating the embedded manifest through xtask.+ if ! echo "$status" | jq -e '.present' >/dev/null; then+ restored="$RUNNER_TEMP/restored-init-snapshot"+ rm -rf "$restored"+ mkdir -p "$restored"+ if nsc artifact download "$artifact" "$archive" \+ && tar -xf "$archive" -C "$restored" \+ && [ -f "$restored/$key/manifest.json" ]; then+ rm -rf "$root/$key"+ cp -a "$restored/$key" "$root/$key"+ status=$(just stack snapshot --json | tail -n 1)+ if echo "$status" | jq -e '.present' >/dev/null; then+ artifact_hit=1+ echo "init snapshot artifact hit: $status"+ else+ rm -rf "$root/$key"+ echo "downloaded init snapshot failed validation; baking cold" >&2+ fi+ else+ echo "init snapshot artifact miss; baking cold" >&2+ fi+ fi+if echo "$status" | jq -e '.present' >/dev/null; then- artifact_hit=1- echo "init snapshot artifact hit: $status"+ echo "init snapshot cache hit: $status"+ cargo run --quiet --manifest-path Cargo.toml \+ -p xtask_local --features local-stack -- gen-compose+ # The cold path builds the runtime image inside `stack up`; on a+ # hit nothing else does, and the registry mirror needs it in the+ # local daemon (its pull fallback only covers public images —+ # macro-local-runtime:dev is not on Docker Hub).+ cargo run --quiet --manifest-path Cargo.toml \+ -p xtask_local --features local-stack -- runtime-imageelse- rm -rf "$root/$key"- echo "downloaded init snapshot failed validation; baking cold" >&2+ just stack up --infra-only --no-doppler --no-build+ status=$(just stack snapshot --json | tail -n 1)+ echo "$status" | jq -e '.present' >/dev/null+ fi++ # Seed artifact storage from either a volume hit or a cold bake. Avoid+ # uploading another version when the artifact already exists; a race+ # between concurrent PRs is harmless and must not fail the preview.+ if [ -z "$artifact_hit" ] && ! nsc artifact describe "$artifact" >/dev/null 2>&1; then+ snap_dir=$(echo "$status" | jq -r '.dir')+ tar -cf "$archive" -C "$root" "$(basename "$snap_dir")"+ nsc artifact upload "$archive" "$artifact" --expires_in 720h \+ || echo "warning: failed to upload init snapshot artifact" >&2+ fi+ echo "$status" > "$RUNNER_TEMP/preview-snapshot.json"+ LANE++ declare -a pids=() names=()+ for name in fly cargo web images bake; do+ # setsid: each lane leads its own process group so fail-fast+ # can kill the whole tree (cargo/docker/bun children), not just+ # the wrapper shell.+ setsid bash "$lanes/$name.sh" > "$lanes/$name.log" 2>&1 &+ pids+=($!)+ names+=("$name")+ done+ fail=+ for _ in "${pids[@]}"; do+ # Bare `wait -n`: any child; explicit pids would error once a+ # job has already been reaped.+ if ! wait -n; then+ fail=1+ break+ fi+ done+ if [ -n "$fail" ]; then+ # Negative pids: signal each lane's whole process group.+ kill -- "${pids[@]/#/-}" 2>/dev/null || true+ wait || true+ fi+ for i in "${!names[@]}"; do+ echo "::group::lane: ${names[$i]}"+ cat "$lanes/${names[$i]}.log" || true+ echo "::endgroup::"+ done+ if [ -n "$fail" ]; then+ echo "a build lane failed (first failure aborts the rest); see lane logs above" >&2+ exit 1fi- else- echo "init snapshot artifact miss; baking cold" >&2- fi- fi-- if echo "$status" | jq -e '.present' >/dev/null; then- echo "init snapshot cache hit: $status"- cargo run --quiet --manifest-path Cargo.toml \- -p xtask_local --features local-stack -- gen-compose- # The cold path builds the runtime image inside `stack up`; on a- # hit nothing else does, and the registry mirror needs it in the- # local daemon (its pull fallback only covers public images —- # macro-local-runtime:dev is not on Docker Hub).- cargo run --quiet --manifest-path Cargo.toml \- -p xtask_local --features local-stack -- runtime-image- else- just stack up --infra-only --no-doppler --no-build- status=$(just stack snapshot --json | tail -n 1)- echo "$status" | jq -e '.present' >/dev/null- fi-- # Seed artifact storage from either a volume hit or a cold bake. Avoid- # uploading another version when the artifact already exists; a race- # between concurrent PRs is harmless and must not fail the preview.- if [ -z "$artifact_hit" ] && ! nsc artifact describe "$artifact" >/dev/null 2>&1; then- snap_dir=$(echo "$status" | jq -r '.dir')- tar -cf "$archive" -C "$root" "$(basename "$snap_dir")"- nsc artifact upload "$archive" "$artifact" --expires_in 720h \- || echo "warning: failed to upload init snapshot artifact" >&2- fi- echo "$status" > "$RUNNER_TEMP/preview-snapshot.json""#})+ .add_env(("FLY_ORG", "${{ vars.FLY_ORG || secrets.FLY_ORG }}"))}/// When the bake dies (typically the backend health gate), the answer is in@@ -410,27 +554,27 @@ fn deploy_to_fly() -> Step<Run> {# warm /var/lib/docker layer store published by a previous# successful deploy (see the "Publish template volume" step).# Measured cold, an empty volume costs ~11 min of image pulls at- # boot; a fork costs a ~2.5 min server-side hydration that runs- # concurrently with the mirror push below (the gate before- # machine creation waits for it). Cross-app forks need the- # Machines API: flyctl resolves volume ids app-locally. The boot- # manifest check makes the volume a pure cache, so every failure- # path here just falls back to the old empty-volume behavior.- fork_pending=- if ! flyctl volumes list --app "$APP_NAME" --json 2>/dev/null \- | jq -e 'map(select(.name == "docker_data")) | length > 0' >/dev/null; then+ # boot. The images lane normally kicked the fork minutes ago (its+ # ~2 min server-side hydration overlaps the whole build phase);+ # this block is the authoritative fallback for a missing or+ # failed early fork. Cross-app forks need the Machines API:+ # flyctl resolves volume ids app-locally. The boot manifest check+ # makes the volume a pure cache, so every failure path here just+ # falls back to the old empty-volume behavior.+ vol_state=$(flyctl volumes list --app "$APP_NAME" --json 2>/dev/null \+ | jq -r '[.[] | select(.name == "docker_data")][0].state // empty') || vol_state=+ if [ -z "$vol_state" ]; thentpl_src=$(flyctl volumes list --app "$TEMPLATE_APP" --json 2>/dev/null \| jq -r '[.[] | select((.name | startswith("tpl")) and .state == "created" and .attached_machine_id == null)]| sort_by(.created_at) | last | .id // empty' 2>/dev/null) || tpl_src=if [ -n "$tpl_src" ]; then- fork_state=$(curl -sf -X POST "https://api.machines.dev/v1/apps/$APP_NAME/volumes" \+ vol_state=$(curl -sf -X POST "https://api.machines.dev/v1/apps/$APP_NAME/volumes" \-H "Authorization: Bearer $FLY_API_TOKEN" \-H "Content-Type: application/json" \-d "{\"name\":\"docker_data\",\"region\":\"ewr\",\"source_volume_id\":\"$tpl_src\"}" \- | jq -r '.state // empty') || fork_state=- if [ -n "$fork_state" ]; then- echo "seeding docker_data from template volume $tpl_src (state: $fork_state)"- fork_pending=1+ | jq -r '.state // empty') || vol_state=+ if [ -n "$vol_state" ]; then+ echo "seeding docker_data from template volume $tpl_src (state: $vol_state)"elseecho "template fork failed; falling back to an empty volume" >&2fi@@ -445,15 +589,21 @@ fn deploy_to_fly() -> Step<Run> {# secrets, so scope matters). The org deploy token can't mint new# tokens (createLimitedAccessToken is not authorized), but macaroon# attenuation is pure client-side crypto: append an Apps caveat- # (numeric app id, mask "r") and a validity window to our own token.- app_id=$(curl -sf https://api.fly.io/graphql \- -H "Authorization: Bearer $FLY_API_TOKEN" \- -H "content-type: application/json" \- -d "{\"query\":\"{ app(name: \\\"$APP_NAME\\\") { internalNumericId } }\"}" \- | jq -re '.data.app.internalNumericId')- now=$(date +%s)- pull_token=$(printf '[{"type":"Apps","body":{"apps":{"%s":"r"}}},{"type":"ValidityWindow","body":{"not_before":%d,"not_after":%d}}]' \- "$app_id" $((now - 60)) $((now + 604800)) | flyctl tokens attenuate)+ # (numeric app id, mask "r") and a validity window to our own+ # token. The images lane normally minted it already; fall back to+ # minting inline when its file is absent.+ if [ -s "$RUNNER_TEMP/registry-pull-token" ]; then+ pull_token=$(cat "$RUNNER_TEMP/registry-pull-token")+ else+ app_id=$(curl -sf https://api.fly.io/graphql \+ -H "Authorization: Bearer $FLY_API_TOKEN" \+ -H "content-type: application/json" \+ -d "{\"query\":\"{ app(name: \\\"$APP_NAME\\\") { internalNumericId } }\"}" \+ | jq -re '.data.app.internalNumericId')+ now=$(date +%s)+ pull_token=$(printf '[{"type":"Apps","body":{"apps":{"%s":"r"}}},{"type":"ValidityWindow","body":{"not_before":%d,"not_after":%d}}]' \+ "$app_id" $((now - 60)) $((now + 604800)) | flyctl tokens attenuate)+ fisnapshot_key=$(jq -r '.key' "$RUNNER_TEMP/preview-snapshot.json")runtime_key=$(jq -r '.runtime_key' preview-ctx/deployment.json)@@ -494,6 +644,20 @@ fn deploy_to_fly() -> Step<Run> {fiecho "preview deployment mode: $mode (snapshot $snapshot_key)"+ # Keep the machine in demand for the WHOLE deploy, hot path+ # included: fly-proxy suspends a machine ~7 min after its last+ # proxied request, and SSH via hallpass is not proxy traffic — a+ # hot update with no browser traffic got its VM suspended+ # mid-session (the frozen `flyctl ssh console` then ate 52 min of+ # runner until the job timeout). Any response status counts as+ # traffic, and a request also resumes a suspended machine.+ (while :; do+ curl -s -o /dev/null --max-time 10 "https://$APP_NAME.fly.dev/" || true+ sleep 15+ done) &+ keepalive=$!+ trap 'kill "$keepalive" 2>/dev/null || true' EXIT+flyctl auth dockerregistry="registry.fly.io/$APP_NAME"# Mirror every stack image into the app's registry repo instead of@@ -516,10 +680,9 @@ fn deploy_to_fly() -> Step<Run> {--env-file "$envfile" \config --images | sort -u)echo "mirroring images:" $images- docker pull alpine:3+ docker image inspect alpine:3 >/dev/null 2>&1 || docker pull alpine:3mkdir -p preview-ctx/preload: > preview-ctx/preload/manifest.txt- : > "$RUNNER_TEMP/push-refs.txt"for img in $images alpine:3; do# Images the bake didn't leave in the daemon (snapshot cache hit,# or app-layer-only images like proxy/mailpit) get pulled here.@@ -531,10 +694,14 @@ fn deploy_to_fly() -> Step<Run> {ref="$registry:img-$(echo "$img" | tr '/:' '__')-$(echo "$id" | cut -c8-19)"docker tag "$img" "$ref"echo "$id $img $ref" >> preview-ctx/preload/manifest.txt- if ! docker manifest inspect "$ref" >/dev/null 2>&1; then- echo "$ref" >> "$RUNNER_TEMP/push-refs.txt"- fidone+ # The existence probes are pure network round-trips (and after the+ # premirror lane, almost all of them hit) — run them in parallel+ # instead of ~1s each serially.+ awk '{print $3}' preview-ctx/preload/manifest.txt \+ | xargs -P 8 -I {} sh -c \+ 'docker manifest inspect "{}" >/dev/null 2>&1 || echo "{}"' \+ > "$RUNNER_TEMP/push-refs.txt"echo "pushing $(wc -l < "$RUNNER_TEMP/push-refs.txt") of $(wc -l < preview-ctx/preload/manifest.txt) images"# The Fly registry occasionally aborts large uploads ("s3aws:# append to zero-size path unsupported") — retry; layers that made@@ -572,8 +739,11 @@ fn deploy_to_fly() -> Step<Run> {--app "$APP_NAME" --machine "$machine_id" --mode 0600 \|| hot_rc=$?if [ "$hot_rc" = 0 ]; then+ # timeout backstop: a suspended-mid-session VM freezes the ssh+ # client forever (measured: 52 min of hang). 10 min is triple+ # a normal hot apply; on expiry we fall back to rehydrate.hot_command="/srv/macro/bin/hot-update '$update_image' /tmp/macro-registry-token"- flyctl ssh console --app "$APP_NAME" --machine "$machine_id" \+ timeout 600 flyctl ssh console --app "$APP_NAME" --machine "$machine_id" \--command "$hot_command" || hot_rc=$?fiif [ "$hot_rc" = 0 ]; then@@ -589,12 +759,16 @@ fn deploy_to_fly() -> Step<Run> {flyctl secrets set --app "$APP_NAME" --stage \"DOPPLER_TOKEN=$DOPPLER_PREVIEW_TOKEN" \"REGISTRY_PULL_TOKEN=$pull_token"- if [ -n "$fork_pending" ]; then- # The mirror push above bought most of the hydration time; the- # volume must reach "created" before a machine can mount it. A+ if [ -z "$vol_state" ]; then+ # No volume and no fork landed (no template, or both fork+ # attempts failed): the old empty-volume path.+ flyctl volumes create docker_data --app "$APP_NAME" --region ewr --size 40 --yes+ elif [ "$vol_state" != created ]; then+ # A fork is still hydrating (usually already finished: the+ # images lane kicked it before the cargo lane even ended). The+ # volume must reach "created" before a machine can mount it; a# fork that never lands is replaced by an empty volume rather# than failing the deploy.- vol_state=for _ in $(seq 1 60); dovol_state=$(flyctl volumes list --app "$APP_NAME" --json \| jq -r '[.[] | select(.name == "docker_data")][0].state // empty')@@ -609,9 +783,6 @@ fn deploy_to_fly() -> Step<Run> {[ -z "$vol_id" ] || flyctl volumes destroy "$vol_id" --app "$APP_NAME" --yes || trueflyctl volumes create docker_data --app "$APP_NAME" --region ewr --size 40 --yesfi- elif ! flyctl volumes list --app "$APP_NAME" --json \- | jq -e 'map(select(.name == "docker_data")) | length > 0' >/dev/null; then- flyctl volumes create docker_data --app "$APP_NAME" --region ewr --size 40 --yesfi# Machines created before the volume existed can't take the mounts# config update. A destroyed/partial machine is recreated by deploy.@@ -624,27 +795,15 @@ fn deploy_to_fly() -> Step<Run> {image="$registry:${{ github.sha }}"docker build -t "$image" preview-ctxpush_image "$image"- # Keep the machine in demand while it boots: with zero inbound- # traffic, fly-proxy counts a health-failing machine as excess- # capacity and suspends it mid-boot (observed mid-docker-load).- # Any response status counts as traffic, and a request also- # resumes an already-suspended machine.- (while :; do- curl -s -o /dev/null --max-time 10 "https://$APP_NAME.fly.dev/" || true- sleep 15- done) &- keepalive=$!# First boot does real work before 8090 opens (pulling the stack# images, snapshot restore, compose up, FusionAuth's JVM) —# give the health check more runway than flyctl's default wait.- rc=0+ # The step-wide keepalive above holds the machine out of suspend.flyctl deploy --app "$APP_NAME" \--config infra/preview/fly.toml \--image "$image" \--wait-timeout 1800 \- --yes || rc=$?- kill "$keepalive" 2>/dev/null || true- exit "$rc"+ --yes"#})// Accept the org slug from either a repo variable or a repo secret —// it's not sensitive, but people reasonably reach for secrets first.