Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

feat: fullstack preview (#4547)

teo7b254ab62dc2
diff --git a/.github/workflows/preview-fly-cleanup.yml b/.github/workflows/preview-fly-cleanup.yml
new file mode 100644
index 0000000000..ebe911e0bb
--- /dev/null
+++ b/.github/workflows/preview-fly-cleanup.yml
@@ -0,0 +1,21 @@
+# DO NOT EDIT — regenerate with `cargo x workflows` (from the repository root).
+# Source: tooling/xtask/crates/xtask_workflows/src/workflows/preview_fly.rs
+name: Fly Preview Cleanup
+'on':
+ pull_request:
+ types:
+ - closed
+ - unlabeled
+jobs:
+ destroy:
+ if: github.event.action == 'closed' || github.event.label.name == 'preview'
+ runs-on: namespace-profile-linux-tiny-no-cache
+ permissions:
+ contents: read
+ env:
+ FLY_API_TOKEN: ${{ secrets.FLY_API_TOKEN }}
+ steps:
+ - name: Setup flyctl
+ uses: superfly/flyctl-actions/setup-flyctl@ed8efb33836e8b2096c7fd3ba1c8afe303ebbff1
+ - name: Destroy app
+ run: flyctl apps destroy "macro-pr-${{ github.event.pull_request.number }}" --yes || true
diff --git a/.github/workflows/preview-fly.yml b/.github/workflows/preview-fly.yml
new file mode 100644
index 0000000000..35b24d53e3
--- /dev/null
+++ b/.github/workflows/preview-fly.yml
@@ -0,0 +1,465 @@
+# DO NOT EDIT — regenerate with `cargo x workflows` (from the repository root).
+# Source: tooling/xtask/crates/xtask_workflows/src/workflows/preview_fly.rs
+name: Fly Preview
+'on':
+ pull_request:
+ types:
+ - opened
+ - reopened
+ - synchronize
+ - labeled
+jobs:
+ deploy:
+ if: github.event.pull_request.head.repo.full_name == github.repository && contains(github.event.pull_request.labels.*.name, 'preview')
+ runs-on: namespace-profile-linux-rust-ci;overrides.cache-tag=fly-preview
+ permissions:
+ contents: read
+ pull-requests: write
+ env:
+ FLY_API_TOKEN: ${{ secrets.FLY_API_TOKEN }}
+ APP_NAME: macro-pr-${{ github.event.pull_request.number }}
+ MACRO_STACK_SNAPSHOT_DIR: /home/runner/.cache/macro-preview-snapshots
+ CARGO_INCREMENTAL: '0'
+ steps:
+ - name: Checkout
+ uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd
+ with:
+ clean: 'false'
+ - name: Mount Namespace cache volume
+ uses: namespacelabs/nscloud-cache-action@15799a6b54e5765f85b2aac25b3f0df43ed571c0
+ with:
+ cache: rust
+ path: |-
+ /nix
+ ${{ github.workspace }}/target
+ /home/runner/.cache/macro-preview-snapshots
+ continue-on-error: true
+ - name: Prune stale incremental caches
+ run: rm -rf target/*/debug/incremental target/debug/incremental
+ - name: Set up Namespace Docker builder
+ uses: namespacelabs/nscloud-setup-buildx-action@d059ed7184f0bc7c8b27e8810cea153d02bcc6dd
+ - name: Build compose service images (fail fast)
+ run: docker compose --project-directory . -p macro -f docker/docker-compose.yml build search sync_service websocket_service lexical_service ai_editing_worker
+ - name: Setup Nix
+ uses: './.github/actions/setup-nix'
+ - name: Setup dev shell + web deps
+ uses: './.github/actions/setup-reqs-web'
+ with:
+ cachix-auth-token: ${{ secrets.CACHIX_AUTH_TOKEN }}
+ - name: Configure Namespace remote sccache
+ run: |-
+ set -euo pipefail
+ env_file="$RUNNER_TEMP/namespace-sccache.env"
+ if nsc cache sccache setup --cache_name fly-preview > "$env_file"; then
+ cat "$env_file" >> "$GITHUB_ENV"
+ # Force the next compiler invocation to start a server with the new remote
+ # backend even if a setup hook happened to launch one already.
+ sccache --stop-server >/dev/null 2>&1 || true
+ else
+ echo "::warning::Namespace remote sccache setup failed; using local cache fallback"
+ fi
+ - name: Build service binaries
+ run: |
+ cargo x zigbuild
+ sccache --show-stats
+ - name: Build xtask_local (runs inside the preview VM)
+ run: cargo zigbuild --release --target x86_64-unknown-linux-gnu.2.36 -p xtask_local
+ - name: Build frontend bundle (same-origin)
+ run: bun run --bun build
+ env:
+ MODE: development
+ NODE_ENV: production
+ VITE_LOCAL_SERVERS: ALL
+ VITE_LOCAL_BACKEND_ORIGIN: same-origin
+ working-directory: apps/web
+ - name: Bake init snapshot
+ run: |
+ set -euo pipefail
+ mkdir -p "$MACRO_STACK_SNAPSHOT_DIR"
+ status=$(just stack snapshot --json | tail -n 1)
+ key=$(echo "$status" | jq -r '.key')
+ root=$(echo "$status" | jq -r '.root')
+ artifact="macro-preview/init-snapshots/${key}.tar"
+ archive="$RUNNER_TEMP/init-snapshot-${key}.tar"
+ artifact_hit=
+
+ # The Namespace cache-volume pool has an onboarding period where a
+ # runner can receive an empty fork. Artifact storage is the durable
+ # fallback: restore into a temporary directory and only publish it to
+ # the live store after validating the embedded manifest through xtask.
+ if ! echo "$status" | jq -e '.present' >/dev/null; then
+ restored="$RUNNER_TEMP/restored-init-snapshot"
+ rm -rf "$restored"
+ mkdir -p "$restored"
+ if nsc artifact download "$artifact" "$archive" \
+ && tar -xf "$archive" -C "$restored" \
+ && [ -f "$restored/$key/manifest.json" ]; then
+ rm -rf "$root/$key"
+ cp -a "$restored/$key" "$root/$key"
+ status=$(just stack snapshot --json | tail -n 1)
+ if echo "$status" | jq -e '.present' >/dev/null; then
+ artifact_hit=1
+ echo "init snapshot artifact hit: $status"
+ else
+ rm -rf "$root/$key"
+ echo "downloaded init snapshot failed validation; baking cold" >&2
+ fi
+ else
+ echo "init snapshot artifact miss; baking cold" >&2
+ fi
+ fi
+
+ if echo "$status" | jq -e '.present' >/dev/null; then
+ echo "init snapshot cache hit: $status"
+ cargo run --quiet --manifest-path Cargo.toml \
+ -p xtask_local --features local-stack -- gen-compose
+ # The cold path builds the runtime image inside `stack up`; on a
+ # hit nothing else does, and the registry mirror needs it in the
+ # local daemon (its pull fallback only covers public images —
+ # macro-local-runtime:dev is not on Docker Hub).
+ cargo run --quiet --manifest-path Cargo.toml \
+ -p xtask_local --features local-stack -- runtime-image
+ else
+ just stack up --infra-only --no-doppler --no-build
+ status=$(just stack snapshot --json | tail -n 1)
+ echo "$status" | jq -e '.present' >/dev/null
+ fi
+
+ # Seed artifact storage from either a volume hit or a cold bake. Avoid
+ # uploading another version when the artifact already exists; a race
+ # between concurrent PRs is harmless and must not fail the preview.
+ if [ -z "$artifact_hit" ] && ! nsc artifact describe "$artifact" >/dev/null 2>&1; then
+ snap_dir=$(echo "$status" | jq -r '.dir')
+ tar -cf "$archive" -C "$root" "$(basename "$snap_dir")"
+ nsc artifact upload "$archive" "$artifact" --expires_in 720h \
+ || echo "warning: failed to upload init snapshot artifact" >&2
+ fi
+ echo "$status" > "$RUNNER_TEMP/preview-snapshot.json"
+ - name: Dump stack diagnostics
+ if: failure()
+ run: |
+ docker compose -p macro ps --all || true
+ for svc in authentication-service proxy fusionauth postgres kafka \
+ connection_gateway document_storage_service email_service \
+ notification_service contacts_service; do
+ echo "==================== $svc ===================="
+ docker compose -p macro logs --no-color --tail 80 "$svc" 2>&1 || true
+ done
+ - name: Stage preview build context
+ run: |
+ set -euo pipefail
+ # The dev-shell env (BASH_ENV) exports a Nix LD_LIBRARY_PATH; host
+ # binaries like rsync then resolve Nix-store libs whose deps drag in
+ # Nix glibc mid-process and crash on glibc symbol versions. This step
+ # is pure file shuffling — drop it.
+ unset LD_LIBRARY_PATH
+ ctx=preview-ctx
+ rm -rf "$ctx"
+ mkdir -p "$ctx/repo" "$ctx/artifacts/binaries" "$ctx/bin"
+ # The minimal repo layout xtask reads at runtime — including every
+ # snapshot-key input, byte-identical to this checkout, so the VM's
+ # key matches the snapshot baked above.
+ rsync -a --relative --exclude node_modules \
+ docker/docker-compose.yml \
+ docker/docker-compose-databases.yml \
+ infra/stacks/fusionauth-instance/docker-compose.yml \
+ infra/local/nginx \
+ infra/local/opensearch \
+ infra/stacks/opensearch/helpers \
+ crates/macro_db_client/migrations \
+ "$ctx/repo/"
+ # Service binaries only — the target dir also holds gigabytes of
+ # build intermediates. `-p` everywhere: preserved mtimes mean files
+ # cargo didn't relink produce byte-identical Docker layers, so the
+ # registry push/pull skips them (the Dockerfile orders its COPYs
+ # stable → volatile for the same reason).
+ find target/x86_64-unknown-linux-gnu/debug \
+ -maxdepth 1 -type f -executable ! -name '*.d' ! -name '*.so' \
+ -exec cp -p {} "$ctx/artifacts/binaries/" \;
+ cp -a apps/web/dist "$ctx/artifacts/frontend-dist"
+ # Only the current key's snapshot — the volume store accumulates keys.
+ snap_dir=$(jq -r '.dir' "$RUNNER_TEMP/preview-snapshot.json")
+ mkdir -p "$ctx/artifacts/snapshots"
+ cp -a "$snap_dir" "$ctx/artifacts/snapshots/"
+ snapshot_key=$(jq -r '.key' "$RUNNER_TEMP/preview-snapshot.json")
+ runtime_key=$(cd "$ctx/repo" \
+ && find . -type f -print0 \
+ | sort -z \
+ | xargs -0 sha256sum \
+ | sha256sum \
+ | cut -d ' ' -f 1)
+ frontend_key=$(cd "$ctx/artifacts/frontend-dist" \
+ && find . -type f -print0 \
+ | sort -z \
+ | xargs -0 sha256sum \
+ | sha256sum \
+ | cut -d ' ' -f 1)
+ jq -n \
+ --arg snapshot_key "$snapshot_key" \
+ --arg runtime_key "$runtime_key" \
+ --arg frontend_key "$frontend_key" \
+ --arg commit "${{ github.sha }}" \
+ '{format: 1, snapshot_key: $snapshot_key, runtime_key: $runtime_key, frontend_key: $frontend_key, commit: $commit}' \
+ > "$ctx/deployment.json"
+ cp -p target/x86_64-unknown-linux-gnu/release/xtask_local "$ctx/bin/xtask"
+ cp -p infra/preview/hot-update.sh "$ctx/bin/hot-update"
+ cp -p infra/preview/entrypoint.sh "$ctx/entrypoint.sh"
+ cp -p infra/preview/Dockerfile "$ctx/Dockerfile"
+ cp -p infra/preview/update.Dockerfile "$ctx/update.Dockerfile"
+ - name: Setup flyctl
+ uses: superfly/flyctl-actions/setup-flyctl@ed8efb33836e8b2096c7fd3ba1c8afe303ebbff1
+ - name: Deploy to Fly
+ run: |
+ set -euo pipefail
+ if [ -z "$DOPPLER_PREVIEW_TOKEN" ]; then
+ echo "DOPPLER_PREVIEW_TOKEN secret is not set (a Doppler service token scoped to the preview config)" >&2
+ exit 1
+ fi
+ if [ -z "$FLY_ORG" ]; then
+ echo "FLY_ORG is not set (repo variable or secret with the Fly org slug)" >&2
+ exit 1
+ fi
+
+ app_existed=
+ if flyctl status --app "$APP_NAME" >/dev/null 2>&1; then
+ app_existed=1
+ else
+ # A transient status error must not turn an existing app into a
+ # hard create failure: create, then prove the app is reachable.
+ flyctl apps create "$APP_NAME" --org "$FLY_ORG" \
+ || flyctl status --app "$APP_NAME" >/dev/null
+ fi
+
+ # A read-only, app-scoped, time-boxed pull token so the machine's
+ # inner dockerd can pull the mirrored stack images from this app's
+ # registry repo — and nothing else (PR code can read machine
+ # secrets, so scope matters). The org deploy token can't mint new
+ # tokens (createLimitedAccessToken is not authorized), but macaroon
+ # attenuation is pure client-side crypto: append an Apps caveat
+ # (numeric app id, mask "r") and a validity window to our own token.
+ app_id=$(curl -sf https://api.fly.io/graphql \
+ -H "Authorization: Bearer $FLY_API_TOKEN" \
+ -H "content-type: application/json" \
+ -d "{\"query\":\"{ app(name: \\\"$APP_NAME\\\") { internalNumericId } }\"}" \
+ | jq -re '.data.app.internalNumericId')
+ now=$(date +%s)
+ pull_token=$(printf '[{"type":"Apps","body":{"apps":{"%s":"r"}}},{"type":"ValidityWindow","body":{"not_before":%d,"not_after":%d}}]' \
+ "$app_id" $((now - 60)) $((now + 604800)) | flyctl tokens attenuate)
+
+ snapshot_key=$(jq -r '.key' "$RUNNER_TEMP/preview-snapshot.json")
+ runtime_key=$(jq -r '.runtime_key' preview-ctx/deployment.json)
+ mode=bootstrap
+ machine_id=
+ if [ -n "$app_existed" ]; then
+ mode=rehydrate
+ machine_id=$(flyctl machine list --app "$APP_NAME" --json \
+ | jq -r '[.[] | select((.config.mounts // []) | length > 0)][0].id // empty')
+ if [ -n "$machine_id" ]; then
+ # A suspended preview needs demand before SSH is reachable.
+ curl -s -o /dev/null --max-time 10 "https://$APP_NAME.fly.dev/" || true
+ for _ in $(seq 1 12); do
+ marker=$(flyctl ssh console --app "$APP_NAME" --machine "$machine_id" \
+ --quiet --command 'if [ -f /var/lib/docker/.macro-preview/deployment.json ]; then cat /var/lib/docker/.macro-preview/deployment.json; else echo __NO_MARKER__; fi' \
+ 2>/dev/null || true)
+ if echo "$marker" | jq -e \
+ --arg key "$snapshot_key" \
+ --arg runtime "$runtime_key" \
+ '.format == 1 and .snapshot_key == $key and .runtime_key == $runtime and (.frontend_key | type == "string")' \
+ >/dev/null 2>&1; then
+ mode=hot
+ break
+ fi
+ # SSH succeeded and returned a marker (or proved it absent):
+ # incompatibility is real, not a wake-up race.
+ if [ "$marker" = __NO_MARKER__ ] \
+ || echo "$marker" | jq -e '.format != null' >/dev/null 2>&1; then
+ break
+ fi
+ sleep 5
+ done
+ fi
+ fi
+ echo "preview deployment mode: $mode (snapshot $snapshot_key)"
+
+ flyctl auth docker
+ registry="registry.fly.io/$APP_NAME"
+ # Mirror every stack image into the app's registry repo instead of
+ # baking docker-save tars into the VM image (which made every
+ # deploy re-ship ~5GB the machine's volume already had). Pushes
+ # dedup at the layer level against the repo, so a redeploy only
+ # uploads layers that actually changed; the entrypoint pulls with
+ # the same dedup against the machine's persistent layer store. The
+ # manifest maps image ID -> local tag -> registry ref so boots can
+ # skip images already present. alpine:3 rides along for the
+ # snapshot-restore helper container.
+ # On a snapshot cache hit only gen-compose ran, which doesn't write
+ # the env file; compose hard-fails on a missing --env-file, and the
+ # image names don't interpolate env vars anyway.
+ envfile=infra/local/generated/macro/local.generated.env
+ [ -f "$envfile" ] || : > "$envfile"
+ images=$(docker compose --project-directory . -p macro \
+ -f docker/docker-compose.yml \
+ -f infra/local/generated/macro/docker-compose.override.yml \
+ --env-file "$envfile" \
+ config --images | sort -u)
+ echo "mirroring images:" $images
+ docker pull alpine:3
+ mkdir -p preview-ctx/preload
+ : > preview-ctx/preload/manifest.txt
+ : > "$RUNNER_TEMP/push-refs.txt"
+ for img in $images alpine:3; do
+ # Images the bake didn't leave in the daemon (snapshot cache hit,
+ # or app-layer-only images like proxy/mailpit) get pulled here.
+ docker image inspect "$img" >/dev/null 2>&1 || docker pull "$img"
+ id=$(docker image inspect -f '{{.Id}}' "$img")
+ # Content-addressed tag: same image content = same ref, so a
+ # redeploy's push is skippable by a manifest existence check and
+ # concurrent PRs' deploys can never clobber each other's tags.
+ ref="$registry:img-$(echo "$img" | tr '/:' '__')-$(echo "$id" | cut -c8-19)"
+ docker tag "$img" "$ref"
+ echo "$id $img $ref" >> preview-ctx/preload/manifest.txt
+ if ! docker manifest inspect "$ref" >/dev/null 2>&1; then
+ echo "$ref" >> "$RUNNER_TEMP/push-refs.txt"
+ fi
+ done
+ echo "pushing $(wc -l < "$RUNNER_TEMP/push-refs.txt") of $(wc -l < preview-ctx/preload/manifest.txt) images"
+ # The Fly registry occasionally aborts large uploads ("s3aws:
+ # append to zero-size path unsupported") — retry; layers that made
+ # it are reused. Four pushes at a time keeps the first (cold) push
+ # moving without saturating the registry.
+ if [ -s "$RUNNER_TEMP/push-refs.txt" ]; then
+ xargs -P 4 -I {} sh -c '
+ for _ in 1 2 3; do
+ docker push "{}" && exit 0
+ echo "push of {} failed, retrying" >&2
+ sleep 15
+ done
+ exit 1' < "$RUNNER_TEMP/push-refs.txt"
+ fi
+
+ push_image() {
+ image_to_push=$1
+ for _ in 1 2 3; do
+ if docker push "$image_to_push"; then return 0; fi
+ echo "docker push failed for $image_to_push, retrying" >&2
+ sleep 15
+ done
+ return 1
+ }
+
+ if [ "$mode" = hot ]; then
+ update_image="$registry:update-${{ github.sha }}"
+ docker build -f preview-ctx/update.Dockerfile -t "$update_image" preview-ctx
+ push_image "$update_image"
+ token_file="$RUNNER_TEMP/registry-pull-token"
+ printf '%s' "$pull_token" > "$token_file"
+ chmod 600 "$token_file"
+ hot_rc=0
+ flyctl ssh sftp put "$token_file" /tmp/macro-registry-token \
+ --app "$APP_NAME" --machine "$machine_id" --mode 0600 \
+ || hot_rc=$?
+ if [ "$hot_rc" = 0 ]; then
+ hot_command="/srv/macro/bin/hot-update '$update_image' /tmp/macro-registry-token"
+ flyctl ssh console --app "$APP_NAME" --machine "$machine_id" \
+ --command "$hot_command" || hot_rc=$?
+ fi
+ if [ "$hot_rc" = 0 ]; then
+ echo "hot update completed without restarting the Fly machine"
+ exit 0
+ fi
+ echo "hot update failed (exit $hot_rc); falling back to full rehydrate" >&2
+ mode=rehydrate
+ fi
+
+ # Bootstrap/rehydrate refreshes app secrets and reconciles the
+ # persistent Docker volume before replacing the machine image.
+ flyctl secrets set --app "$APP_NAME" --stage \
+ "DOPPLER_TOKEN=$DOPPLER_PREVIEW_TOKEN" \
+ "REGISTRY_PULL_TOKEN=$pull_token"
+ if ! flyctl volumes list --app "$APP_NAME" --json \
+ | jq -e 'map(select(.name == "docker_data")) | length > 0' >/dev/null; then
+ flyctl volumes create docker_data --app "$APP_NAME" --region ewr --size 40 --yes
+ fi
+ # Machines created before the volume existed can't take the mounts
+ # config update. A destroyed/partial machine is recreated by deploy.
+ flyctl machine list --app "$APP_NAME" --json \
+ | jq -r '.[] | select((.config.mounts // []) | length == 0) | .id' \
+ | while read -r id; do
+ [ -z "$id" ] || flyctl machine destroy "$id" --app "$APP_NAME" --force || true
+ done
+
+ image="$registry:${{ github.sha }}"
+ docker build -t "$image" preview-ctx
+ push_image "$image"
+ # Keep the machine in demand while it boots: with zero inbound
+ # traffic, fly-proxy counts a health-failing machine as excess
+ # capacity and suspends it mid-boot (observed mid-docker-load).
+ # Any response status counts as traffic, and a request also
+ # resumes an already-suspended machine.
+ (while :; do
+ curl -s -o /dev/null --max-time 10 "https://$APP_NAME.fly.dev/" || true
+ sleep 15
+ done) &
+ keepalive=$!
+ # First boot does real work before 8090 opens (pulling the stack
+ # images, snapshot restore, compose up, FusionAuth's JVM) —
+ # give the health check more runway than flyctl's default wait.
+ rc=0
+ flyctl deploy --app "$APP_NAME" \
+ --config infra/preview/fly.toml \
+ --image "$image" \
+ --wait-timeout 1800 \
+ --yes || rc=$?
+ kill "$keepalive" 2>/dev/null || true
+ exit "$rc"
+ env:
+ FLY_ORG: ${{ vars.FLY_ORG || secrets.FLY_ORG }}
+ DOPPLER_PREVIEW_TOKEN: ${{ secrets.DOPPLER_PREVIEW_TOKEN }}
+ - name: Dump Fly diagnostics
+ if: failure()
+ run: |
+ flyctl machine list --app "$APP_NAME" || true
+ flyctl logs --app "$APP_NAME" --no-tail || true
+ - name: Dump boot timings
+ run: flyctl logs --app "$APP_NAME" --no-tail | grep -E '\[preview\]|✓' || true
+ - name: Comment preview URL
+ uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b
+ with:
+ github-token: ${{ secrets.GITHUB_TOKEN }}
+ script: |
+ const app = process.env.APP_NAME;
+ const url = `https://${app}.fly.dev`;
+ const marker = '<!-- fly-preview -->';
+ const body = [
+ marker,
+ `🚀 **Full-stack preview**: ${url}`,
+ '',
+ `- Suspended when idle — the first request wakes it (a few seconds).`,
+ `- Log in with any email; the passwordless code lands in [Mailpit](${url}/mailpit/).`,
+ `- Deployed from ${context.payload.pull_request.head.sha.slice(0, 7)}.`,
+ ].join('\n');
+ const { data: comments } = await github.rest.issues.listComments({
+ owner: context.repo.owner,
+ repo: context.repo.repo,
+ issue_number: context.issue.number,
+ per_page: 100,
+ });
+ const existing = comments.find(c => c.body && c.body.startsWith(marker));
+ if (existing) {
+ await github.rest.issues.updateComment({
+ owner: context.repo.owner,
+ repo: context.repo.repo,
+ comment_id: existing.id,
+ body,
+ });
+ } else {
+ await github.rest.issues.createComment({
+ owner: context.repo.owner,
+ repo: context.repo.repo,
+ issue_number: context.issue.number,
+ body,
+ });
+ }
+concurrency:
+ group: fly-preview-${{ github.event.pull_request.number }}
+ cancel-in-progress: true
diff --git a/Cargo.lock b/Cargo.lock
index 157d96cbf8..ce558ccacd 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -16281,6 +16281,7 @@ dependencies = [
"serde",
"serde_json",
"serde_yaml",
+ "sha2 0.10.9",
"strum",
"tokio",
"xtask_paths",
diff --git a/apps/web/src/lib/core/constant/servers.ts b/apps/web/src/lib/core/constant/servers.ts
index 4788dc81bd..62e2e29ed5 100644
--- a/apps/web/src/lib/core/constant/servers.ts
+++ b/apps/web/src/lib/core/constant/servers.ts
@@ -47,8 +47,18 @@ type Servers = Record<keyof typeof serverHostRemote, string>;
// direct-port behavior (unchanged). Declared BEFORE SERVER_HOSTS so it is
// initialized before SERVER_HOSTS evaluates selectLocalServers() at module load
// (these are consts in a temporal dead zone; the functions below are hoisted).
-const proxyOrigin: string | undefined = import.meta.env
+//
+// The special value 'same-origin' resolves to the origin the bundle is served
+// from, at runtime. The headless stack (`cargo x stack up`) builds with it so
+// the static bundle Caddy serves works unchanged on any host that reaches the
+// proxy — localhost, a tunnel URL, a preview domain. (globalThis.location
+// exists in both windows and workers.)
+const rawLocalBackendOrigin: string | undefined = import.meta.env
.VITE_LOCAL_BACKEND_ORIGIN;
+const proxyOrigin: string | undefined =
+ rawLocalBackendOrigin === 'same-origin'
+ ? globalThis.location?.origin
+ : rawLocalBackendOrigin;
const wsProxyOrigin = proxyOrigin?.replace(/^http/, 'ws');
export const SERVER_HOSTS: Servers =
diff --git a/docs/RUNNING_LOCALLY.md b/docs/RUNNING_LOCALLY.md
index a4d8b7dabe..7ad2680bdc 100644
--- a/docs/RUNNING_LOCALLY.md
+++ b/docs/RUNNING_LOCALLY.md
@@ -120,6 +120,40 @@ pick up a change there.
If you already started without `--build-aux-services` and suspect a stale
sync/lexical image, press `q` and restart with the flag.
+## Headless Mode (previews, agents, CI)
+
+`just stack` is the same stack without a terminal attached: no hotkey loop, no
+dev server. The frontend is built once (a dev-mode bundle with production
+optimizations) and served statically by the instance's Caddy proxy, so the whole
+product lives behind **one origin** and a finished `up` leaves only Docker
+containers running — nothing to babysit.
+
+```bash
+just stack up # bring everything up, print URLs, return
+just stack status --json # machine-readable state (containers, health, URLs)
+just stack update # rebuild + reload only changed services (the `r` hotkey)
+just stack update --frontend # also rebuild the frontend bundle
+just stack down # containers + volumes + state
+```
+
+All the `run_local` flags apply (`--instance`, `--no-doppler --env-file`,
+`--no-build`, `--binaries-dir`); CI can hand in a prebuilt bundle with
+`--frontend-dist`, and `--infra-only` stops after the infra bring-up + init
+(the CI bake mode — without Doppler the app services have no env to boot
+with, and the snapshot only captures infra volumes). The app is served at `<proxy>/app/` — the bundle resolves its
+backend from the origin it is served on, so the same stack works on localhost
+or behind a preview hostname without a rebuild.
+
+`stack up` also caches the expensive infra init. The first cold run migrates
+the DB, waits out the FusionAuth kickstart, and creates the search indices,
+then saves those volumes as a content-addressed **init snapshot** (keyed by the
+migrations, kickstart, index mappings, image pins, and container platform —
+stored under `infra/local/generated/.snapshots`). Later runs whose inputs match restore the
+snapshot and skip the init entirely; any input change is a cache miss and a
+normal full init. `just stack snapshot` shows the current key; `--no-snapshot`
+opts out. This is also what makes Fly previews boot fast — CI bakes the
+snapshot into the preview image (see `infra/preview/README.md`).
+
## Common Commands
Run local binaries against shared dev resources instead of a fully local stack:
diff --git a/infra/preview/Dockerfile b/infra/preview/Dockerfile
new file mode 100644
index 0000000000..d131641c65
--- /dev/null
+++ b/infra/preview/Dockerfile
@@ -0,0 +1,59 @@
+# The Fly preview machine image: the entire local stack inside one VM.
+#
+# Built by .github/workflows/preview-fly.yml from a STAGED context (not the
+# repo root) — CI lays out everything the machine needs and this file just
+# copies it in:
+#
+# repo/ minimal repo layout xtask reads (compose files, migrations,
+# index helpers) — MACRO_REPO_ROOT points here
+# artifacts/ binaries/ (zigbuild output), frontend-dist/ (same-origin
+# bundle), snapshots/ (baked init snapshot)
+# bin/ prebuilt orchestrator + the in-place hot-update driver
+# preload/ manifest of every image the stack runs (image ID -> local tag
+# -> registry ref); the entrypoint pulls them from the app's Fly
+# registry repo at boot, skipping whatever the machine's
+# persistent layer store already has
+#
+# Fly machines are full microVMs, so an inner Docker daemon runs the same
+# compose topology as `just run_local` — the preview IS the local stack.
+FROM ubuntu:24.04
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+ docker.io \
+ docker-compose-v2 \
+ docker-buildx \
+ ca-certificates \
+ curl \
+ gnupg \
+ jq \
+ && rm -rf /var/lib/apt/lists/*
+
+# Doppler CLI: the machine boots with a config-scoped service token
+# (DOPPLER_TOKEN, set as a Fly secret by the workflow) and the stack's env
+# layer pulls the preview secrets with it.
+RUN curl -Ls --tlsv1.2 --proto "=https" https://cli.doppler.com/install.sh | sh
+
+# One COPY per artifact, ordered stable → volatile: the staging step preserves
+# mtimes, so a layer whose inputs didn't change this deploy is byte-identical
+# to the previous image's — the registry push and the Fly host's pull both
+# skip it. Everything after the first changed layer re-ships, so the pieces
+# that churn every deploy (frontend bundle, service binaries) go last.
+COPY entrypoint.sh /srv/macro/entrypoint.sh
+COPY deployment.json /srv/macro/deployment.json
+COPY repo/ /srv/macro/repo/
+COPY bin/ /srv/macro/bin/
+COPY artifacts/snapshots/ /srv/macro/artifacts/snapshots/
+COPY preload/ /srv/macro/preload/
+COPY artifacts/frontend-dist/ /srv/macro/artifacts/frontend-dist/
+COPY artifacts/binaries/ /srv/macro/artifacts/binaries/
+RUN chmod +x /srv/macro/entrypoint.sh /srv/macro/bin/xtask /srv/macro/bin/hot-update
+
+# Point the prebuilt xtask at the staged layout (its compiled-in paths refer to
+# the CI checkout) and at the baked snapshot store.
+ENV MACRO_REPO_ROOT=/srv/macro/repo \
+ MACRO_STACK_SNAPSHOT_DIR=/srv/macro/artifacts/snapshots
+
+# The instance's single-origin proxy (fly.toml's internal_port).
+EXPOSE 8090
+
+ENTRYPOINT ["/srv/macro/entrypoint.sh"]
diff --git a/infra/preview/README.md b/infra/preview/README.md
new file mode 100644
index 0000000000..6b60a17d90
--- /dev/null
+++ b/infra/preview/README.md
@@ -0,0 +1,93 @@
+# Full-stack PR previews on Fly
+
+Label a PR `preview` and CI deploys the **entire stack** — every service,
+Postgres, OpenSearch, FusionAuth, LocalStack, the frontend — as one Fly app at
+`https://macro-pr-<N>.fly.dev`. The app suspends when idle (≈zero cost) and the
+fly-proxy wakes it on the next request. Closing the PR (or removing the label)
+destroys it.
+
+## How it works
+
+The preview is not a parallel deployment system: it is the local stack
+(`just stack up`, see `RUNNING_LOCALLY.md`) running inside one Fly machine.
+A Fly machine is a full microVM, so an inner Docker daemon runs the exact
+compose topology `run_local` uses. What differs is that **everything expensive
+is baked in CI** and the machine only restores:
+
+| Baked in CI | Restored on the machine |
+|---|---|
+| Service binaries (`cargo x zigbuild`) | bind-mounted into the runtime image |
+| Frontend bundle (`same-origin` build) | served statically by the instance Caddy |
+| Init snapshot (migrations + FusionAuth kickstart + search indices) | volumes restored, init skipped |
+| Every Docker image, mirrored to the app's Fly registry repo | `docker pull` at boot — layer-dedup'd against the machine's persistent layer store, nothing built |
+
+Boot on a fresh machine ≈ image pulls + snapshot restore + JVM startup
+(a couple of minutes); wake from suspend ≈ seconds. Pushes to the PR choose one
+of three paths from the durable deployment marker on the Docker volume:
+
+- **bootstrap** — no app/machine yet: create everything and restore the snapshot;
+- **hot update** — the snapshot key, staged-runtime hash, and update protocol
+ are unchanged: pull a layer-deduplicated artifact carrier, atomically replace
+ changed binaries/frontend, and restart only affected containers without
+ restarting the Fly machine or infra;
+- **rehydrate** — compatibility changed, or a hot update fails: fall back to
+ the immutable full-image deploy and snapshot restore.
+
+The URL remains stable across all three paths.
+
+The whole product sits behind the stack's single-origin Caddy proxy
+(`:8090`), so one `internal_port` covers the frontend, APIs, WebSockets, and
+Mailpit.
+
+## Logging in
+
+Login is passwordless (email codes) and preview email is captured, not sent:
+enter any address on the login screen, then open
+`https://macro-pr-<N>.fly.dev/mailpit/` to read the code. (Routing login codes
+through scoped SES so teammates get real emails is a planned follow-up.)
+
+## Secrets (Doppler)
+
+The stack's local plumbing (DBs, queues, auth identity) is code-owned and
+dummy, but integration secrets (AI keys etc.) come from Doppler, exactly like
+`run_local`. The workflow stages a **config-scoped service token** onto each
+preview app as the Fly secret `DOPPLER_TOKEN`; at boot the env layer detects
+the token and pulls that config (no `doppler login` involved), then overlays
+the code-owned `LocalEnv` on top — so Doppler supplies integration keys while
+local plumbing stays local. A preview machine can only ever read the one
+config its token is scoped to.
+
+## One-time setup
+
+1. Create a Fly org deploy token: `fly tokens create org -o <org>` →
+ repo secret `FLY_API_TOKEN`.
+2. Set the repo variable `FLY_ORG` to the org slug.
+3. In Doppler, create a preview config in the `local` project (e.g.
+ `lcl_preview` — start from `lcl_personal`, keep only what previews should
+ have) and a service token for it → repo secret `DOPPLER_PREVIEW_TOKEN`.
+4. Create the `preview` label.
+
+## Security posture
+
+- Previews only build from same-repo branches (the workflow refuses forks).
+- No real AWS: S3/SQS/DynamoDB are LocalStack, email is Mailpit. The
+ cloud-reachable credentials on a machine are the preview Doppler config's
+ contents — scope that config accordingly (assume PR code can read it) — and
+ `REGISTRY_PULL_TOKEN`, the CI deploy token attenuated (client-side macaroon
+ caveats) to read-only on this one preview app with a 7-day validity window:
+ it can pull the app's registry repo, nothing else.
+- URLs are public. Anyone with the link can use the preview (and read its
+ Mailpit). Don't put sensitive data in one. Edge auth (oauth2-proxy in front
+ of Caddy) is a planned hardening step.
+
+## Known costs & future optimizations
+
+- The stack images are mirrored into the per-PR app's registry repo, so the
+ first deploy of a PR pushes (and its first boot pulls) the full set; after
+ that both sides move only changed layers. There is no cross-PR sharing —
+ a shared registry (GHCR/Namespace) would fix that at the cost of new
+ org-level credentials.
+- The aux images (sync/websocket/lexical) rebuild in CI on every preview
+ deploy. Runner-level Docker layer caching covers most of it.
+- `cpus = 8, memory = 16gb` (shared) is a deliberate over-provision; measure
+ and shrink once a few previews have run.
diff --git a/infra/preview/entrypoint.sh b/infra/preview/entrypoint.sh
new file mode 100644
index 0000000000..9186fcaae9
--- /dev/null
+++ b/infra/preview/entrypoint.sh
@@ -0,0 +1,94 @@
+#!/usr/bin/env bash
+# Boot for the Fly preview machine: start the inner Docker daemon, pull the
+# stack images from the app's registry repo, bring the stack up from the baked
+# artifacts, then idle while the containers serve. Everything expensive
+# (compile, bundle, init) happened in CI — this script only restores and
+# starts.
+set -euo pipefail
+
+log() { echo "[preview] $*"; }
+
+t_boot=$(date +%s)
+# A hard loss during a hot update cannot run its EXIT trap. The full immutable
+# boot is the recovery boundary, so clear any abandoned volume-backed lock.
+state_dir=/var/lib/docker/.macro-preview
+mkdir -p "$state_dir"
+rmdir "$state_dir/update.lock" 2>/dev/null || true
+log "starting inner dockerd"
+dockerd >/var/log/dockerd.log 2>&1 &
+
+for _ in $(seq 1 120); do
+ docker info >/dev/null 2>&1 && break
+ sleep 1
+done
+if ! docker info >/dev/null 2>&1; then
+ log "dockerd failed to start:"
+ tail -n 50 /var/log/dockerd.log
+ exit 1
+fi
+log "dockerd ready in $(($(date +%s) - t_boot))s"
+
+# Pull the stack images from the app's Fly registry repo (CI mirrored them
+# there at deploy time). /var/lib/docker is a persistent volume, so the
+# manifest (image ID per tag) lets redeploys skip every image already in the
+# store, and pulls dedup at the layer level — only layers that actually
+# changed cross the network. Suspend/resume never re-runs this; only real
+# boots do. REGISTRY_PULL_TOKEN is an app-scoped deploy token staged as a Fly
+# secret by the workflow.
+if [ -f /srv/macro/preload/manifest.txt ]; then
+ if [ -n "${REGISTRY_PULL_TOKEN:-}" ]; then
+ docker login registry.fly.io -u x --password-stdin <<<"$REGISTRY_PULL_TOKEN"
+ fi
+ log "pulling stack images (parallel, warm store aware)"
+ t0=$(date +%s)
+ pids=""
+ while read -r id tag ref; do
+ have=$(docker image inspect -f '{{.Id}}' "$tag" 2>/dev/null || true)
+ if [ "$have" = "$id" ]; then
+ log "already present: $tag"
+ continue
+ fi
+ (docker pull -q "$ref" && docker tag "$ref" "$tag" && docker rmi "$ref" >/dev/null) &
+ pids="$pids $!"
+ done < /srv/macro/preload/manifest.txt
+ rc=0
+ for pid in $pids; do wait "$pid" || rc=1; done
+ [ "$rc" = 0 ] || { log "image pull failed"; exit 1; }
+ log "images pulled in $(($(date +%s) - t0))s"
+fi
+
+# With a DOPPLER_TOKEN (a config-scoped service token, injected as a Fly
+# secret) the env layer pulls the preview secrets; without one the stack runs
+# on the code-owned local dummies only.
+doppler_args=(--no-doppler)
+if [ -n "${DOPPLER_TOKEN:-}" ]; then
+ log "DOPPLER_TOKEN present — pulling preview secrets"
+ doppler_args=()
+fi
+
+log "bringing the stack up (snapshot restore)"
+t0=$(date +%s)
+/srv/macro/bin/xtask stack up \
+ "${doppler_args[@]}" \
+ --no-build \
+ --binaries-dir /srv/macro/artifacts/binaries \
+ --frontend-dist /srv/macro/artifacts/frontend-dist \
+ --json
+
+log "stack up took $(($(date +%s) - t0))s (boot total $(($(date +%s) - t_boot))s)"
+# The backend health gate is the longest stack-up phase; surface the auth
+# service's step timings plus its timestamped tail so the wait is attributable
+# from the CI boot-timings step (which greps [preview] out of `flyctl logs`).
+docker logs -t macro-authentication-service-1 2>&1 \
+ | grep 'authentication startup step' \
+ | sed 's/^/[preview][auth-startup] /' || true
+docker logs -t --tail 40 macro-authentication-service-1 2>&1 \
+ | sed 's/^/[preview][auth] /' || true
+
+# Commit the compatibility marker only after the restored stack is healthy.
+# It lives beside (not inside) Docker's own data on the persistent volume, so
+# CI can distinguish a hot-updatable machine from an absent/partial/old deploy.
+install -m 0644 /srv/macro/deployment.json "$state_dir/deployment.json.next"
+mv -f "$state_dir/deployment.json.next" "$state_dir/deployment.json"
+log "stack ready — proxy serving on :8090"
+exec sleep infinity
diff --git a/infra/preview/fly.toml b/infra/preview/fly.toml
new file mode 100644
index 0000000000..e49362343c
--- /dev/null
+++ b/infra/preview/fly.toml
@@ -0,0 +1,43 @@
+# Fly config for per-PR preview apps. The workflow overrides the app name
+# (`--app macro-pr-<N>`) and supplies the image (`--image`); this file carries
+# everything else. See infra/preview/README.md.
+app = "macro-preview-template"
+primary_region = "ewr"
+
+# Everything (frontend, APIs, WebSockets, Mailpit) is behind the stack's
+# single-origin Caddy proxy.
+[http_service]
+ internal_port = 8090
+ force_https = true
+ # The whole point: an idle preview suspends to ~zero cost and the fly-proxy
+ # wakes it on the next request.
+ auto_stop_machines = "suspend"
+ auto_start_machines = true
+ min_machines_running = 0
+
+ [[http_service.checks]]
+ interval = "15s"
+ timeout = "5s"
+ # First boot loads the baked images and restores the init snapshot before
+ # the backend answers — give it room.
+ grace_period = "5m"
+ method = "GET"
+ path = "/auth/health"
+
+# The inner Docker store must NOT live on the machine rootfs: Fly's rootfs is
+# an overlay (nested overlayfs forbids the whiteout device nodes layer
+# unpacking needs — every Debian-based image failed with EPERM) and has only
+# a few GB of headroom (ENOSPC mid-load). A volume is real ext4, properly
+# sized, and persists across redeploys — so later pushes reuse the layer
+# store. The deploy step creates the volume if it doesn't exist.
+[[mounts]]
+ source = "docker_data"
+ destination = "/var/lib/docker"
+
+[[vm]]
+ cpu_kind = "shared"
+ cpus = 8
+ memory = "16gb"
+
+[[restart]]
+ policy = "on-failure"
diff --git a/infra/preview/hot-update.sh b/infra/preview/hot-update.sh
new file mode 100644
index 0000000000..c31a1c79b6
--- /dev/null
+++ b/infra/preview/hot-update.sh
@@ -0,0 +1,122 @@
+#!/usr/bin/env bash
+# Apply a registry-carried artifact update to an already-running preview VM.
+# Exit 42 asks CI to fall back to a full image deploy/stack rehydrate.
+set -euo pipefail
+
+readonly REHYDRATE_EXIT=42
+readonly STATE_DIR=/var/lib/docker/.macro-preview
+readonly STATE_FILE="$STATE_DIR/deployment.json"
+
+log() { echo "[preview][hot-update] $*"; }
+rehydrate() { log "$*; requesting full rehydrate" >&2; exit "$REHYDRATE_EXIT"; }
+
+if [ "$#" -ne 2 ]; then
+ echo "usage: hot-update <registry-image> <registry-token-file>" >&2
+ exit 2
+fi
+update_image=$1
+token_file=$2
+
+mkdir -p "$STATE_DIR"
+if ! mkdir "$STATE_DIR/update.lock" 2>/dev/null; then
+ echo "another preview update is already running" >&2
+ exit 1
+fi
+
+staging=
+container=
+cleanup() {
+ rm -f "$token_file"
+ [ -z "$container" ] || docker rm -f "$container" >/dev/null 2>&1 || true
+ [ -z "$staging" ] || rm -rf "$staging"
+ rmdir "$STATE_DIR/update.lock" 2>/dev/null || true
+}
+trap cleanup EXIT
+
+[ -s "$token_file" ] || rehydrate "registry pull token is missing"
+[ -f "$STATE_FILE" ] || rehydrate "deployment state is missing"
+
+log "pulling artifact image $update_image"
+docker login registry.fly.io -u x --password-stdin < "$token_file" >/dev/null
+docker pull -q "$update_image"
+container=$(docker create "$update_image")
+staging=$(mktemp -d /srv/macro/.hot-update.XXXXXX)
+docker cp "$container:/update/." "$staging/"
+docker rm "$container" >/dev/null
+container=
+
+new_format=$(jq -er '.format' "$staging/deployment.json")
+new_snapshot=$(jq -er '.snapshot_key' "$staging/deployment.json")
+new_runtime=$(jq -er '.runtime_key' "$staging/deployment.json")
+new_frontend=$(jq -er '.frontend_key' "$staging/deployment.json")
+current_format=$(jq -er '.format' "$STATE_FILE")
+current_snapshot=$(jq -er '.snapshot_key' "$STATE_FILE")
+current_runtime=$(jq -er '.runtime_key' "$STATE_FILE")
+current_frontend=$(jq -er '.frontend_key' "$STATE_FILE")
+[ "$new_format" = "1" ] || rehydrate "unsupported update format $new_format"
+[ "$current_format" = "$new_format" ] || rehydrate "deployment format changed"
+[ "$current_snapshot" = "$new_snapshot" ] || rehydrate "snapshot key changed"
+[ "$current_runtime" = "$new_runtime" ] || rehydrate "runtime configuration changed"
+
+# CI mirrors every compose image and puts the desired IDs in the carrier.
+# Docker-built leaf services are safe to recreate in place. Any other image
+# change is infrastructure/runtime drift and must use the full rehydrate path.
+aux_changed=
+unsafe_images=
+while read -r expected_id tag registry_ref; do
+ current_id=$(docker image inspect -f '{{.Id}}' "$tag" 2>/dev/null || true)
+ [ "$current_id" = "$expected_id" ] && continue
+ case "$tag" in
+ *-ai_editing_worker|*-lexical_service|*-sync_service|*-websocket_service)
+ log "refreshing auxiliary image $tag"
+ docker pull -q "$registry_ref"
+ pulled_id=$(docker image inspect -f '{{.Id}}' "$registry_ref")
+ [ "$pulled_id" = "$expected_id" ] \
+ || rehydrate "registry image ID mismatch for $tag"
+ docker tag "$registry_ref" "$tag"
+ docker rmi "$registry_ref" >/dev/null
+ aux_changed=1
+ ;;
+ *) unsafe_images="$unsafe_images $tag" ;;
+ esac
+done < "$staging/manifest.txt"
+[ -z "$unsafe_images" ] || rehydrate "non-hot-updatable images changed:$unsafe_images"
+
+# SSH sessions don't inherit app secrets. Recover only the config-scoped
+# Doppler token from PID 1 without printing it, then let xtask regenerate the
+# local env file before recreating/restarting containers.
+doppler_token=$(tr '\0' '\n' < /proc/1/environ | sed -n 's/^DOPPLER_TOKEN=//p' | head -n 1)
+[ -z "$doppler_token" ] || export DOPPLER_TOKEN="$doppler_token"
+export MACRO_REPO_ROOT=/srv/macro/repo
+export MACRO_STACK_SNAPSHOT_DIR=/srv/macro/artifacts/snapshots
+
+apply_args=(
+ stack update
+ --binaries-dir "$staging/binaries"
+ --json
+)
+[ "$current_frontend" = "$new_frontend" ] \
+ || apply_args+=(--frontend-dist "$staging/frontend-dist")
+[ -z "$aux_changed" ] || apply_args+=(--recreate-aux-services)
+
+log "applying prebuilt artifacts"
+"$staging/xtask" "${apply_args[@]}"
+
+# Publish the new orchestrator and compatibility marker only after the stack is
+# healthy. A failed apply leaves the old marker, so the next run rehydrates.
+install -m 0755 "$staging/xtask" /srv/macro/bin/xtask.next
+mv -f /srv/macro/bin/xtask.next /srv/macro/bin/xtask
+install -m 0644 "$staging/deployment.json" "$STATE_FILE.next"
+mv -f "$STATE_FILE.next" "$STATE_FILE"
+
+# Retain the current carrier's layers for the next incremental pull, but remove
+# older carrier tags and their now-unreferenced layers from the persistent store.
+repository=${update_image%:*}
+while read -r old; do
+ case "$old" in
+ "$repository":update-*) [ "$old" = "$update_image" ] || docker rmi "$old" >/dev/null || true ;;
+ esac
+done < <(docker images "$repository" --format '{{.Repository}}:{{.Tag}}')
+docker image prune -f >/dev/null || true
+
+log "update complete"
diff --git a/infra/preview/update.Dockerfile b/infra/preview/update.Dockerfile
new file mode 100644
index 0000000000..28084f0e25
--- /dev/null
+++ b/infra/preview/update.Dockerfile
@@ -0,0 +1,13 @@
+# Registry-carried artifact bundle for a running preview machine. The image is
+# never started: hot-update.sh creates a stopped container and copies /update
+# out of it. Scratch keeps every layer limited to actual application artifacts,
+# and stable COPY ordering maximizes registry/inner-Docker dedup across pushes.
+FROM scratch
+
+COPY bin/xtask /update/xtask
+COPY preload/manifest.txt /update/manifest.txt
+COPY deployment.json /update/deployment.json
+COPY artifacts/frontend-dist/ /update/frontend-dist/
+COPY artifacts/binaries/ /update/binaries/
+
+CMD ["/update/xtask"]
diff --git a/services/authentication_service/src/main.rs b/services/authentication_service/src/main.rs
index d4e19b901e..6c907a211c 100644
--- a/services/authentication_service/src/main.rs
+++ b/services/authentication_service/src/main.rs
@@ -66,8 +66,10 @@ async fn main() -> anyhow::Result<()> {
MacroEntrypoint::default().init();
let env = Environment::new_or_prod();
+ // One SDK config is sufficient for every AWS client in this process.
+ let aws_config = macro_aws_config::get_macro_aws_config().await;
let secretsmanager_client = secretsmanager_client::SecretsManager::new(
- aws_sdk_secretsmanager::Client::new(&macro_aws_config::get_macro_aws_config().await),
+ aws_sdk_secretsmanager::Client::new(&aws_config),
);
// Parse our configuration from the environment.
@@ -173,7 +175,7 @@ async fn main() -> anyhow::Result<()> {
// `from_env` routes to local SMTP (Mailpit) when SMTP_HOST is set, else SES.
let ses_client = ses_client::Ses::from_env(
- aws_sdk_sesv2::Client::new(&macro_aws_config::get_macro_aws_config().await),
+ aws_sdk_sesv2::Client::new(&aws_config),
&config.environment.to_string(),
);
@@ -201,12 +203,10 @@ async fn main() -> anyhow::Result<()> {
};
tracing::trace!("initialized notification ingress service");
- let sqs_client = sqs_client::SQS::new(aws_sdk_sqs::Client::new(
- &macro_aws_config::get_macro_aws_config().await,
- ))
- .search_event_queue(&search_event_queue)
- .email_link_manager_queue(&link_manager_queue)
- .email_backfill_queue(&email_backfill_queue);
+ let sqs_client = sqs_client::SQS::new(aws_sdk_sqs::Client::new(&aws_config))
+ .search_event_queue(&search_event_queue)
+ .email_link_manager_queue(&link_manager_queue)
+ .email_backfill_queue(&email_backfill_queue);
tracing::trace!("initialized sqs client");
// Initialize analytics client with configured providers
diff --git a/tooling/just/xtask.just b/tooling/just/xtask.just
index 8e942d0d8c..c1e8d91dea 100644
--- a/tooling/just/xtask.just
+++ b/tooling/just/xtask.just
@@ -31,3 +31,11 @@ destroy_local *ARGS:
# Stop an instance's containers, keeping volumes (xtask).
stop_local *ARGS:
{{ xtask }} stop-local "$@"
+
+# Headless stack orchestration for previews, agents, and CI.
+# just stack up # full stack; static frontend served by the proxy
+# just stack update # rebuild + reload only what changed (the `r` hotkey)
+# just stack status --json # machine-readable state
+# just stack down # reclaim everything
+stack *ARGS:
+ {{ xtask }} stack "$@"
diff --git a/tooling/xtask/crates/xtask_local/Cargo.toml b/tooling/xtask/crates/xtask_local/Cargo.toml
index 979276ec5a..8bebb50fb3 100644
--- a/tooling/xtask/crates/xtask_local/Cargo.toml
+++ b/tooling/xtask/crates/xtask_local/Cargo.toml
@@ -22,6 +22,8 @@ clap = { workspace = true }
macro_aws_config = { path = "../../../../crates/macro_aws_config" }
macro_queues = { path = "../../../../crates/macro_queues" }
serde = { workspace = true }
+# Content-address the init snapshots (`stack up` restore/save keys).
+sha2 = { workspace = true }
tokio = { workspace = true }
# Source of truth for the declared Kafka topics (local-stack provisioning).
macro_event_topics = { path = "../../../../crates/macro_event_topics" }
diff --git a/tooling/xtask/crates/xtask_local/src/local/build/zigbuild.rs b/tooling/xtask/crates/xtask_local/src/local/build/zigbuild.rs
index 88d551b909..3e6bbb4680 100644
--- a/tooling/xtask/crates/xtask_local/src/local/build/zigbuild.rs
+++ b/tooling/xtask/crates/xtask_local/src/local/build/zigbuild.rs
@@ -98,9 +98,37 @@ fn base_command(ws: &Path, zig_cache: &Path, jobs: &str, target: Target) -> Comm
.env("ZIG_LOCAL_CACHE_DIR", zig_cache)
.env("RUSTC_WRAPPER", "sccache")
.env("XDG_CACHE_HOME", zig_cache);
+ if let Some(flag) = curl_include_flag() {
+ // First-found wins in cc's env chain (CFLAGS_<target> before CFLAGS),
+ // so set the target-specific var too in case something else exports a
+ // plain CFLAGS that would otherwise shadow ours — and vice versa.
+ let var = format!("CFLAGS_{}", target.triple.replace('-', "_"));
+ cmd.env(var, &flag).env("CFLAGS", &flag);
+ }
cmd
}
+/// librdkafka 2.12's bundled cmake build needs curl *headers* even with
+/// `WITH_CURL=0` (its config header always defines `WITH_OAUTHBEARER_OIDC` via
+/// `#cmakedefine01`, and rdkafka_conf.c guards the curl include with `#ifdef`
+/// instead of `#if`). Host builds get them from the Nix cc-wrapper, which
+/// reads `NIX_CFLAGS_COMPILE` — but `zig cc` is a raw clang that ignores it,
+/// so the dev shell's `curl.dev` never reaches the cross cmake build. Mine
+/// the wrapper flags for an include dir that actually carries `curl/curl.h`
+/// and hand it to cc/cmake explicitly; `-idirafter` keeps it at the lowest
+/// search priority so it cannot shadow zig's own libc headers.
+// Reading the ambient Nix shell env is the point here; there is no config
+// layer this could come from.
+#[allow(clippy::disallowed_methods)]
+fn curl_include_flag() -> Option<String> {
+ let flags = std::env::var("NIX_CFLAGS_COMPILE").ok()?;
+ flags
+ .split_whitespace()
+ .filter(|tok| tok.starts_with('/'))
+ .find(|dir| Path::new(dir).join("curl/curl.h").is_file())
+ .map(|dir| format!("-idirafter {dir}"))
+}
+
/// Fail with an actionable hint if the rust-std for `target` is not installed.
/// The repo's `rustup` is a no-op shim, so the only fix is adding the triple to
/// `rust-toolchain.toml` and re-entering `nix develop`.
diff --git a/tooling/xtask/crates/xtask_local/src/local/cli.rs b/tooling/xtask/crates/xtask_local/src/local/cli.rs
index aa3e7dc966..947906293e 100644
--- a/tooling/xtask/crates/xtask_local/src/local/cli.rs
+++ b/tooling/xtask/crates/xtask_local/src/local/cli.rs
@@ -42,6 +42,24 @@ enum Cmd {
ResetLocal(InstanceArgs),
/// Remove an instance's containers, networks, and volumes.
DestroyLocal(InstanceArgs),
+ /// Headless stack orchestration (previews, agents, CI) — no TTY, no
+ /// attached dev server; the proxy serves a static frontend bundle.
+ #[command(subcommand)]
+ Stack(StackCmd),
+}
+
+#[derive(Subcommand)]
+pub enum StackCmd {
+ /// Bring a full local stack up and return (only containers keep running).
+ Up(super::stack::UpArgs),
+ /// Rebuild binaries (and optionally the frontend) and reload what changed.
+ Update(super::stack::UpdateArgs),
+ /// Report the instance's containers, health, and URLs (`--json` for machines).
+ Status(super::stack::StatusArgs),
+ /// Tear the instance down: containers, volumes, and state.
+ Down(super::stack::DownArgs),
+ /// Report the init-snapshot key/location for this instance.
+ Snapshot(super::stack::SnapshotArgs),
}
#[derive(Args, Clone, Default)]
@@ -85,11 +103,10 @@ pub struct RunArgs {
pub env: EnvArgs,
#[command(flatten)]
pub build: BuildArgs,
- /// Do not start the frontend dev server.
+ /// Do not start or serve the frontend.
#[arg(long)]
pub no_frontend: bool,
- /// Stream subprocess output and show per-step timings (e.g. on `r`, the
- /// build vs reload split instead of one folded line).
+ /// Stream subprocess output and show per-step timings.
#[arg(long, short)]
pub verbose: bool,
}
@@ -160,5 +177,12 @@ fn run(cli: Cli) -> Result<()> {
Cmd::StopLocal(a) => super::stop(&a),
Cmd::ResetLocal(a) => super::reset(&a),
Cmd::DestroyLocal(a) => super::destroy(&a),
+ Cmd::Stack(cmd) => match cmd {
+ StackCmd::Up(a) => super::stack::up(Mode::Local, &a),
+ StackCmd::Update(a) => super::stack::update(&a),
+ StackCmd::Status(a) => super::stack::status(&a),
+ StackCmd::Down(a) => super::stack::down(&a),
+ StackCmd::Snapshot(a) => super::stack::snapshot_status(&a),
+ },
}
}
diff --git a/tooling/xtask/crates/xtask_local/src/local/env_layer.rs b/tooling/xtask/crates/xtask_local/src/local/env_layer.rs
index b691bbd83c..0d35e668da 100644
--- a/tooling/xtask/crates/xtask_local/src/local/env_layer.rs
+++ b/tooling/xtask/crates/xtask_local/src/local/env_layer.rs
@@ -17,7 +17,7 @@ use std::collections::BTreeMap;
use std::path::{Path, PathBuf};
use std::process::Command;
-use anyhow::{Context, Result};
+use anyhow::{Context, Result, bail};
use super::instance::Instance;
use super::{Mode, local_env};
@@ -171,23 +171,41 @@ fn pull_aws_credentials(env: &mut BTreeMap<String, String>) {
}
}
-/// Pull the named Doppler `local`-project config as a flat env overlay (mirrors
-/// the old `just get_environment`). Non-fatal: if Doppler is unavailable in local
-/// mode we proceed on defaults + env-file. Returns whether secrets were loaded.
+/// Pull Doppler secrets as a flat env overlay (mirrors the old
+/// `just get_environment`). Two modes:
+///
+/// - `DOPPLER_TOKEN` in the process env (CI, Fly preview machines) is a
+/// config-scoped service token: download WITHOUT `--project`/`--config` (the
+/// token pins them), and treat failure as fatal — whoever provisioned the
+/// token explicitly asked for these secrets, so proceeding without them
+/// would just move the breakage into the services.
+/// - Otherwise pull the named `local`-project config via the developer's
+/// `doppler login`. Non-fatal: if Doppler is unavailable in local mode we
+/// proceed on defaults + env-file.
+///
+/// Returns whether secrets were loaded.
+// xtask is host tooling, not a service reading APP_SECRETS_JSON, so reading the
+// process environment directly is correct here.
+#[allow(clippy::disallowed_methods)]
fn pull_doppler(config: &str, env: &mut BTreeMap<String, String>) -> Result<bool> {
- let output = Command::new("doppler")
- .args([
- "secrets",
- "download",
- "--project",
- "local",
- "--config",
- config,
- ])
- .args(["--format", "json", "--no-file"])
- .output();
+ let token_scoped = std::env::var("DOPPLER_TOKEN").is_ok_and(|v| !v.is_empty());
+ let mut cmd = Command::new("doppler");
+ cmd.args(["secrets", "download"]);
+ if !token_scoped {
+ cmd.args(["--project", "local", "--config", config]);
+ }
+ cmd.args(["--format", "json", "--no-file"]);
+ let output = cmd.output();
let output = match output {
Ok(o) if o.status.success() => o,
+ Ok(o) if token_scoped => bail!(
+ "doppler secrets download failed with DOPPLER_TOKEN set:\n{}",
+ String::from_utf8_lossy(&o.stderr).trim_end()
+ ),
+ Err(e) if token_scoped => {
+ return Err(anyhow::Error::from(e))
+ .context("running doppler with DOPPLER_TOKEN set (is the doppler CLI installed?)");
+ }
_ => return Ok(false),
};
let json: BTreeMap<String, serde_json::Value> =
diff --git a/tooling/xtask/crates/xtask_local/src/local/frontend.rs b/tooling/xtask/crates/xtask_local/src/local/frontend.rs
index bc4046e5df..4e0bc5f954 100644
--- a/tooling/xtask/crates/xtask_local/src/local/frontend.rs
+++ b/tooling/xtask/crates/xtask_local/src/local/frontend.rs
@@ -4,6 +4,7 @@
use std::io::Read;
use std::os::unix::process::CommandExt;
+use std::path::Path;
use std::process::{Child, Command, Stdio};
use std::sync::{Arc, Mutex};
use std::thread::JoinHandle;
@@ -23,6 +24,69 @@ pub fn url(instance: &Instance) -> String {
format!("http://localhost:{}/app", instance.port(Port::Frontend))
}
+/// Frontend URL when the proxy serves the static bundle (headless stacks): the
+/// app lives on the single proxy origin, not a dev-server port.
+pub fn static_url(instance: &Instance) -> String {
+ format!("{}/app/", proxy::url(instance))
+}
+
+/// Where the instance's static frontend build is staged. Mounted read-only into
+/// the proxy container at `/srv/frontend` (see `gen_compose::add_proxy_service`).
+/// Staged per instance — not served from `dist/` directly — so concurrent
+/// instances in one checkout can't clobber each other's running frontend, and a
+/// later `bun run build` can't mutate a live stack out from under itself.
+pub fn static_dir(instance: &Instance) -> std::path::PathBuf {
+ instance.artifact_dir().join("frontend")
+}
+
+/// Build the app bundle for headless serving and stage it into the instance
+/// dir. `prebuilt` skips the build and stages an existing dist (CI hands the
+/// binaries-style artifact straight in). The build mirrors `just build-dev`
+/// (dev-mode bundle, production optimizations), except the backend origin is
+/// the `same-origin` sentinel — resolved from `location.origin` at runtime — so
+/// the one bundle works on localhost and through any tunnel/preview hostname.
+pub fn build_static(stage: &Stage, instance: &Instance, prebuilt: Option<&Path>) -> Result<()> {
+ let dist = match prebuilt {
+ Some(dir) => dir.to_owned(),
+ None => {
+ let mut cmd = Command::new("bun");
+ cmd.current_dir(app_dir())
+ .args(["run", "--bun", "build"])
+ .env("MODE", "development")
+ .env("NODE_ENV", "production")
+ .env("VITE_LOCAL_SERVERS", "ALL")
+ .env("VITE_LOCAL_BACKEND_ORIGIN", "same-origin");
+ stage.run("Building frontend bundle", &mut cmd)?;
+ app_dir().join("dist")
+ }
+ };
+ if stage.is_dry_run() {
+ return Ok(());
+ }
+ if !dist.join("index.html").exists() {
+ anyhow::bail!(
+ "no index.html in {} — frontend build did not produce a bundle",
+ dist.display()
+ );
+ }
+ stage.run_step("Staging frontend bundle", || {
+ let staged = static_dir(instance);
+ if staged.exists() {
+ std::fs::remove_dir_all(&staged)
+ .with_context(|| format!("clearing {}", staged.display()))?;
+ }
+ instance.ensure_artifact_dir()?;
+ let status = Command::new("cp")
+ .arg("-a")
+ .arg(&dist)
+ .arg(&staged)
+ .status()
+ .context("running cp -a")?;
+ anyhow::ensure!(status.success(), "cp -a exited with {status}");
+ Ok(())
+ })
+}
+
/// The env the dev server runs with. Both local and dev point the whole app at
/// the local proxy origin (single backend origin); the modes differ only in
/// what the *services* behind the proxy talk to (local infra vs dev resources).
diff --git a/tooling/xtask/crates/xtask_local/src/local/fusionauth.rs b/tooling/xtask/crates/xtask_local/src/local/fusionauth.rs
index 80fa0cac3a..860898ad15 100644
--- a/tooling/xtask/crates/xtask_local/src/local/fusionauth.rs
+++ b/tooling/xtask/crates/xtask_local/src/local/fusionauth.rs
@@ -6,7 +6,7 @@ use std::process::Command;
use anyhow::{Context, Result};
use super::instance::{Instance, Port};
-use super::{gen_compose, kickstart, stage::Stage};
+use super::{gen_compose, identity, kickstart, stage::Stage};
/// The local populate-JWT lambda body, inlined into `kickstart.json`.
///
@@ -49,18 +49,30 @@ pub fn write_kickstart(instance: &Instance) -> Result<()> {
Ok(())
}
-/// Block until FusionAuth reports status Ok (it applies the kickstart on first
-/// boot against an empty DB, which is slow). Runs as a stage so it shows the
-/// spinner.
+/// Block until the kickstart has fully applied (first boot against an empty DB
+/// is slow). Runs as a stage so it shows the spinner.
+///
+/// Polls the kickstart's OWN artifacts — a tenant fetch authorized by the
+/// kickstart API key — NOT `/api/status`: FusionAuth reports status Ok while
+/// the kickstart is still applying, and the snapshot save stops the containers
+/// right after this wait. Gating on status alone once froze a mid-kickstart DB
+/// into a snapshot (no tenant), and every stack restored from it 500'd at
+/// login with `InvalidTenantIdException` — while the key never changed, so the
+/// bad snapshot was sticky. The fetch below succeeds only once the API key,
+/// the tenant, and the application all exist (the kickstart creates the
+/// application after the tenant).
pub fn wait_ready(stage: &Stage, instance: &Instance) -> Result<()> {
let url = format!(
- "http://localhost:{}/api/status",
- instance.port(Port::FusionAuth)
+ "http://localhost:{}/api/application/{}",
+ instance.port(Port::FusionAuth),
+ identity::APPLICATION_ID,
);
let script = format!(
- "for i in $(seq 1 90); do curl -fsS {url} 2>/dev/null | grep -q '\"status\":\"Ok\"' && exit 0; sleep 2; done; echo 'timed out waiting for FusionAuth'; exit 1"
+ "for i in $(seq 1 120); do curl -fsS -H 'Authorization: {key}' {url} >/dev/null 2>&1 && exit 0; sleep 2; done; \
+ echo 'timed out waiting for the FusionAuth kickstart'; exit 1",
+ key = identity::FUSIONAUTH_API_KEY,
);
let mut cmd = Command::new("bash");
cmd.arg("-lc").arg(script);
- stage.run("Waiting for FusionAuth", &mut cmd)
+ stage.run("Waiting for FusionAuth (kickstart)", &mut cmd)
}
diff --git a/tooling/xtask/crates/xtask_local/src/local/gen_compose.rs b/tooling/xtask/crates/xtask_local/src/local/gen_compose.rs
index 455f38b7e4..b6c0e847cd 100644
--- a/tooling/xtask/crates/xtask_local/src/local/gen_compose.rs
+++ b/tooling/xtask/crates/xtask_local/src/local/gen_compose.rs
@@ -53,7 +53,14 @@ pub fn caddyfile_path(instance: &Instance) -> PathBuf {
}
/// Build the override (typed model), apply the merge tags, and write it.
-pub fn generate(mode: Mode, instance: &Instance, binaries: &BinariesDir) -> Result<PathBuf> {
+/// `static_frontend` mounts the staged app bundle into the proxy (headless
+/// stacks serve the frontend from Caddy instead of a dev server).
+pub fn generate(
+ mode: Mode,
+ instance: &Instance,
+ binaries: &BinariesDir,
+ static_frontend: bool,
+) -> Result<PathBuf> {
let mut services: IndexMap<String, Option<dct::Service>> = IndexMap::new();
let mounts = binaries.compose_mounts();
@@ -80,12 +87,12 @@ pub fn generate(mode: Mode, instance: &Instance, binaries: &BinariesDir) -> Resu
// The reverse proxy is the frontend's single origin in every mode, and
// LocalStack runs in every mode (dev's `dev_personal` notification queue
// lives there too).
- add_proxy_service(&mut services, instance);
+ add_proxy_service(&mut services, instance, static_frontend);
add_localstack_service(&mut services, instance);
// The rest of the local infra (FusionAuth, Mailpit, per-instance Postgres/
// Redis/OpenSearch port remaps) only for the self-contained local stacks.
if mode.spec().runs_local_infra {
- add_local_infra(&mut services, instance);
+ add_local_infra(&mut services, instance, static_frontend);
}
let compose = dct::Compose {
@@ -138,19 +145,32 @@ fn add_localstack_service(
}
/// The single-origin reverse proxy (Caddy). Added in every mode — it's how the
-/// frontend reaches the local service containers.
-fn add_proxy_service(services: &mut IndexMap<String, Option<dct::Service>>, instance: &Instance) {
+/// frontend reaches the local service containers. Headless stacks also mount
+/// the staged app bundle so Caddy serves the frontend itself (see
+/// `proxy::FRONTEND_STATIC`).
+fn add_proxy_service(
+ services: &mut IndexMap<String, Option<dct::Service>>,
+ instance: &Instance,
+ static_frontend: bool,
+) {
let proxy_port = instance.port(Port::Proxy);
+ let mut volumes = vec![dct::Volumes::Simple(format!(
+ "{}:/etc/caddy/Caddyfile:ro",
+ caddyfile_path(instance).display()
+ ))];
+ if static_frontend {
+ volumes.push(dct::Volumes::Simple(format!(
+ "{}:/srv/frontend:ro",
+ super::frontend::static_dir(instance).display()
+ )));
+ }
services.insert(
"proxy".to_string(),
Some(dct::Service {
image: Some(CADDY_IMAGE.to_string()),
environment: kv(&[("PROXY_PORT", &proxy_port.to_string())]),
ports: dct::Ports::Short(vec![format!("{proxy_port}:{proxy_port}")]),
- volumes: vec![dct::Volumes::Simple(format!(
- "{}:/etc/caddy/Caddyfile:ro",
- caddyfile_path(instance).display()
- ))],
+ volumes,
networks: dct::Networks::Simple(vec!["services".to_string(), "databases".to_string()]),
..Default::default()
}),
@@ -160,7 +180,11 @@ fn add_proxy_service(services: &mut IndexMap<String, Option<dct::Service>>, inst
/// Add the local-only companion infra services (FusionAuth override + Mailpit)
/// and, for named instances, the remapped infra ports. LocalStack is added
/// separately (it runs in every mode), so it is not included here.
-fn add_local_infra(services: &mut IndexMap<String, Option<dct::Service>>, instance: &Instance) {
+fn add_local_infra(
+ services: &mut IndexMap<String, Option<dct::Service>>,
+ instance: &Instance,
+ static_frontend: bool,
+) {
// FusionAuth: repoint at the generated kickstart (volumes replaced via tag).
let mut fusionauth = dct::Service {
environment: kv(&[(
@@ -182,15 +206,21 @@ fn add_local_infra(services: &mut IndexMap<String, Option<dct::Service>>, instan
}
services.insert("fusionauth".to_string(), Some(fusionauth));
- // Mailpit (also on `auth` so FusionAuth can deliver passwordless codes).
+ // Mailpit is also on `auth` so FusionAuth can deliver passwordless codes.
+ // Only headless mode moves its UI under the proxy-friendly /mailpit root;
+ // attached run_local preserves the existing direct UI at port 8025.
+ let mut mailpit_env = vec![
+ ("MP_SMTP_AUTH_ACCEPT_ANY", "1"),
+ ("MP_SMTP_AUTH_ALLOW_INSECURE", "1"),
+ ];
+ if static_frontend {
+ mailpit_env.push(("MP_WEBROOT", "/mailpit"));
+ }
services.insert(
"mailpit".to_string(),
Some(dct::Service {
image: Some(MAILPIT_IMAGE.to_string()),
- environment: kv(&[
- ("MP_SMTP_AUTH_ACCEPT_ANY", "1"),
- ("MP_SMTP_AUTH_ALLOW_INSECURE", "1"),
- ]),
+ environment: kv(&mailpit_env),
ports: dct::Ports::Short(vec![
format!("{}:1025", instance.port(Port::MailpitSmtp)),
format!("{}:8025", instance.port(Port::MailpitUi)),
diff --git a/tooling/xtask/crates/xtask_local/src/local/local_env.rs b/tooling/xtask/crates/xtask_local/src/local/local_env.rs
index 007d1d1b24..221d724c45 100644
--- a/tooling/xtask/crates/xtask_local/src/local/local_env.rs
+++ b/tooling/xtask/crates/xtask_local/src/local/local_env.rs
@@ -235,16 +235,24 @@ impl ServiceAuthEnv {
/// kickstart (see [`identity`]). The OAuth redirect is the only per-instance bit.
struct FusionAuthEnv {
oauth_redirect_uri: String,
+ /// The auth service's own public origin (`BASE_URL`): OAuth callbacks and
+ /// email verification links are built on it. Same host-port convention as
+ /// [`identity::oauth_redirect_uri`]. Doppler supplies it for dev; the
+ /// code-owned env must too, or a `--no-doppler` stack's auth service dies
+ /// at startup on the missing required value.
+ base_url: String,
}
impl FusionAuthEnv {
fn for_instance(instance: &Instance) -> Self {
FusionAuthEnv {
oauth_redirect_uri: identity::oauth_redirect_uri(instance.port(Port::Auth)),
+ base_url: format!("http://localhost:{}", instance.port(Port::Auth)),
}
}
fn write(&self, env: &mut BTreeMap<String, String>) {
+ env.insert("BASE_URL".into(), self.base_url.clone());
env.insert(
"FUSIONAUTH_BASE_URL".into(),
"http://fusionauth:9011".into(),
diff --git a/tooling/xtask/crates/xtask_local/src/local/local_env/test.rs b/tooling/xtask/crates/xtask_local/src/local/local_env/test.rs
index 98d254afa6..d3aaa895e8 100644
--- a/tooling/xtask/crates/xtask_local/src/local/local_env/test.rs
+++ b/tooling/xtask/crates/xtask_local/src/local/local_env/test.rs
@@ -15,6 +15,7 @@ fn emits_required_keys() {
for key in [
"ENVIRONMENT",
"PORT",
+ "BASE_URL",
"DATABASE_URL",
"DATABASE_URL_READONLY",
"REDIS_URI",
diff --git a/tooling/xtask/crates/xtask_local/src/local/mailpit.rs b/tooling/xtask/crates/xtask_local/src/local/mailpit.rs
index 5a86bf91b5..24915e3349 100644
--- a/tooling/xtask/crates/xtask_local/src/local/mailpit.rs
+++ b/tooling/xtask/crates/xtask_local/src/local/mailpit.rs
@@ -4,7 +4,12 @@
use super::instance::{Instance, Port};
-/// The Mailpit web UI URL (host-facing).
-pub fn ui_url(instance: &Instance) -> String {
+/// The directly published Mailpit web UI used by attached `run_local`.
+pub fn direct_ui_url(instance: &Instance) -> String {
format!("http://localhost:{}", instance.port(Port::MailpitUi))
}
+
+/// The single-origin Mailpit route used by headless stacks and previews.
+pub fn proxy_ui_url(instance: &Instance) -> String {
+ format!("{}/mailpit/", super::proxy::url(instance))
+}
diff --git a/tooling/xtask/crates/xtask_local/src/local/mod.rs b/tooling/xtask/crates/xtask_local/src/local/mod.rs
index 39ae39cbe5..540af0be0b 100644
--- a/tooling/xtask/crates/xtask_local/src/local/mod.rs
+++ b/tooling/xtask/crates/xtask_local/src/local/mod.rs
@@ -34,6 +34,8 @@ pub mod mailpit;
pub mod opensearch;
pub mod proxy;
pub mod resources;
+pub mod snapshot;
+pub mod stack;
pub mod stage;
pub mod summary;
pub mod validate;
@@ -152,6 +154,16 @@ use stage::Stage;
const AUX_SERVICE_IMAGES: &[&str] = &["websocket_service", "sync_service", "lexical_service"];
+/// The artifact-driven preview updater can refresh every Docker-built app
+/// service. Keep this separate so enabling local aux rebuilds does not make the
+/// existing `run_local` edit loop rebuild the AI worker too.
+const PREVIEW_AUX_SERVICE_IMAGES: &[&str] = &[
+ "websocket_service",
+ "sync_service",
+ "lexical_service",
+ "ai_editing_worker",
+];
+
/// Bring up a Local or Dev stack and (unless `--no-frontend`) the frontend.
pub fn run_stack(mode: Mode, args: &cli::RunArgs) -> Result<()> {
if mode.spec().runs_local_infra {
@@ -166,6 +178,9 @@ pub fn run_stack(mode: Mode, args: &cli::RunArgs) -> Result<()> {
mode.label(),
instance.name()
));
+ if !stage.is_dry_run() {
+ stack::clear_state(&instance)?;
+ }
// `run_local`/`run_dev` are full delete + full create: tear the previous
// stack and ALL its stateful volumes down so the bring-up is always from a
@@ -183,7 +198,7 @@ pub fn run_stack(mode: Mode, args: &cli::RunArgs) -> Result<()> {
// Foreground: resolve env, build binaries + runtime image, generate the
// compose override / Caddyfile / kickstart. None of this touches the volumes
// or containers the teardown is removing, so it's safe to overlap.
- let (env, target) = prepare(&stage, mode, &instance, args)?;
+ let (env, target) = prepare(&stage, mode, &instance, args, false)?;
// Join the background teardown before we (re)create volumes + bring infra up,
// surfaced as a live spinner so it's clear what we're blocked on. It
@@ -206,7 +221,7 @@ pub fn run_stack(mode: Mode, args: &cli::RunArgs) -> Result<()> {
// start. The teardown means everything is freshly created each run, so
// otherwise the services race their backends on startup (no `macrodb`,
// DynamoDB/OpenSearch connection refused).
- bring_up_infra(&stage, mode, &instance, &env)?;
+ bring_up_infra(&stage, mode, &instance, &env, InfraInit::Full)?;
bring_up_app(&stage, mode, &instance, &env)?;
// No restart-to-reload step: the teardown means `up` always creates fresh
@@ -225,7 +240,13 @@ pub fn run_stack(mode: Mode, args: &cli::RunArgs) -> Result<()> {
frontend::start(&stage, &instance, mode)?
};
- summary::print(mode, &instance, &env);
+ summary::print(
+ mode,
+ &instance,
+ &env,
+ &frontend::url(&instance),
+ &mailpit::direct_ui_url(&instance),
+ );
match frontend {
// Interactive terminal: stay attached with a hotkey loop.
@@ -430,12 +451,14 @@ fn interact(
/// runtime image + service binaries, and generate the compose override /
/// Caddyfile / kickstart. Deliberately does NOT create the external
/// networks/volumes — that's done after the background teardown joins, since
-/// teardown removes them. Returns the resolved env + build target.
+/// teardown removes them. `static_frontend` wires the proxy to serve the staged
+/// app bundle (headless `stack up`). Returns the resolved env + build target.
fn prepare(
stage: &Stage,
mode: Mode,
instance: &Instance,
args: &cli::RunArgs,
+ static_frontend: bool,
) -> Result<(env_layer::ResolvedEnv, arch::Target)> {
let env = env_layer::resolve(
mode,
@@ -461,8 +484,8 @@ fn prepare(
// through the proxy in every mode), and — for the self-contained local
// stacks — the FusionAuth kickstart. (External networks/volumes are created
// by the caller after the background teardown joins.)
- gen_compose::generate(mode, instance, &binaries)?;
- proxy::write_caddyfile(instance, mode)?;
+ gen_compose::generate(mode, instance, &binaries, static_frontend)?;
+ proxy::write_caddyfile(instance, mode, static_frontend)?;
if mode.spec().runs_local_infra {
fusionauth::write_kickstart(instance)?;
}
@@ -500,6 +523,29 @@ fn recreate_aux_service_containers(
stage.run("Recreating auxiliary service containers", &mut up)
}
+fn recreate_preview_aux_service_containers(
+ stage: &Stage,
+ instance: &Instance,
+ env: &env_layer::ResolvedEnv,
+) -> Result<()> {
+ let mut up = compose_cmd(instance, env);
+ up.args(["up", "-d", "--force-recreate", "--no-deps"])
+ .args(PREVIEW_AUX_SERVICE_IMAGES);
+ stage.run("Recreating preview auxiliary services", &mut up)
+}
+
+/// How the local infra reaches its initialized state on bring-up.
+#[derive(Clone, Copy, PartialEq, Eq)]
+enum InfraInit {
+ /// Run the real init: migrate the DB, let FusionAuth apply its kickstart,
+ /// create the OpenSearch indices.
+ Full,
+ /// The volumes were restored from an init snapshot — the state already
+ /// exists, so the init steps are skipped (LocalStack is provisioned either
+ /// way; its state isn't volume-backed).
+ FromSnapshot,
+}
+
/// Bring up the backend infra the app services connect to at startup, and get it
/// fully ready before any of them start. The clean-slate teardown means
/// everything is freshly created each run, so this ordering is what stops the
@@ -511,6 +557,7 @@ fn bring_up_infra(
mode: Mode,
instance: &Instance,
env: &env_layer::ResolvedEnv,
+ init: InfraInit,
) -> Result<()> {
if stage.is_dry_run() {
return Ok(());
@@ -545,7 +592,7 @@ fn bring_up_infra(
localstack::provision(instance)
})?;
- if spec.migrates_db {
+ if spec.migrates_db && init == InfraInit::Full {
// Postgres is ready (`--wait`); create + migrate the freshly-wiped DB.
db::migrate(stage, instance)?;
}
@@ -553,11 +600,18 @@ fn bring_up_infra(
// Kafka is healthy (`--wait` gates on its broker healthcheck); create
// the event topics declared in `macro_event_topics` — the local
// equivalent of the MSK topic provisioning driven by the generated
- // `.github/kafka-cluster-topics.json`.
- stage.run_step("Creating Kafka topics", || kafka::provision(instance))?;
+ // `.github/kafka-cluster-topics.json`. Restored volumes already carry
+ // the topics (they live in the broker's data dir), so only a full init
+ // provisions — which also means a snapshot-restoring `stack up` (e.g.
+ // the preview VM) never needs the rdkafka-backed `local-stack` feature.
+ if init == InfraInit::Full {
+ stage.run_step("Creating Kafka topics", || kafka::provision(instance))?;
+ }
// Start FusionAuth on its own (impatient healthcheck → no `--wait`) and
- // poll it patiently until the kickstart has applied.
+ // poll it patiently until it's up. On a full init that wait covers the
+ // ~minute kickstart; on a snapshot restore the kickstart is already in
+ // the restored volumes, so this is just JVM boot.
let mut fa = compose_cmd(instance, env);
fa.arg("up").arg("-d").arg("fusionauth");
stage.run("Starting FusionAuth", &mut fa)?;
@@ -566,7 +620,10 @@ fn bring_up_infra(
// OpenSearch is up (`--wait`) but empty. Create the search indices +
// aliases (idempotent) so the unified search path works out of the box
// instead of 404ing on the missing `documents`/`chats`/… indices.
- opensearch::provision_indices(stage, instance, &env.merged)?;
+ // Restored volumes already contain them.
+ if init == InfraInit::Full {
+ opensearch::provision_indices(stage, instance, &env.merged)?;
+ }
}
Ok(())
}
@@ -743,7 +800,7 @@ pub fn gen_compose_only(args: &cli::InstanceArgs) -> Result<()> {
let instance = Instance::derive(args.instance.as_deref(), args.port_base)?;
let target = arch::detect()?;
let binaries = build::BinariesDir::TargetDir(workspace_root().join(target.debug_dir()));
- let path = gen_compose::generate(Mode::Local, &instance, &binaries)?;
+ let path = gen_compose::generate(Mode::Local, &instance, &binaries, false)?;
println!("{}", path.display());
Ok(())
}
diff --git a/tooling/xtask/crates/xtask_local/src/local/proxy.rs b/tooling/xtask/crates/xtask_local/src/local/proxy.rs
index 6d28de5473..913acd0174 100644
--- a/tooling/xtask/crates/xtask_local/src/local/proxy.rs
+++ b/tooling/xtask/crates/xtask_local/src/local/proxy.rs
@@ -22,29 +22,38 @@ pub fn url(instance: &Instance) -> String {
/// Write the instance Caddyfile and return its path. Both local and dev route
/// the frontend through this single origin to the local service containers; the
/// only difference is the static-file block (dev has no local LocalStack).
-pub fn write_caddyfile(instance: &Instance, mode: Mode) -> Result<PathBuf> {
+/// With `static_frontend` the proxy also serves the built app bundle at `/app`,
+/// making it the one origin for the whole product.
+pub fn write_caddyfile(instance: &Instance, mode: Mode, static_frontend: bool) -> Result<PathBuf> {
let path = caddyfile_path(instance);
if let Some(dir) = path.parent() {
std::fs::create_dir_all(dir)
.with_context(|| format!("creating proxy dir {}", dir.display()))?;
}
- std::fs::write(&path, caddyfile(mode))
+ std::fs::write(&path, caddyfile(mode, static_frontend))
.with_context(|| format!("writing {}", path.display()))?;
Ok(path)
}
/// Assemble the Caddyfile: the listener head, the generated per-service routes
/// (from the inventory), the special non-inventory routes, the mode's
-/// static-file block, then the tail. Service routes are identical across modes
-/// (they hit the local containers); only the static-file block differs.
-fn caddyfile(mode: Mode) -> String {
+/// static-file block, the optional static-frontend block, then the tail.
+/// Service routes are identical across modes (they hit the local containers);
+/// only the static-file and frontend blocks differ.
+fn caddyfile(mode: Mode, static_frontend: bool) -> String {
let static_block = if mode.spec().static_files_via_localstack {
STATIC_FILE_LOCAL
} else {
STATIC_FILE_DEV
};
+ let mailpit_block = if static_frontend && mode.spec().runs_local_infra {
+ MAILPIT_ROUTE
+ } else {
+ ""
+ };
+ let frontend_block = if static_frontend { FRONTEND_STATIC } else { "" };
format!(
- "{CADDY_HEAD}{routes}{SPECIAL_ROUTES}{static_block}{CADDY_TAIL}",
+ "{CADDY_HEAD}{routes}{SPECIAL_ROUTES}{mailpit_block}{static_block}{frontend_block}{CADDY_TAIL}",
routes = service_routes()
)
}
@@ -112,6 +121,14 @@ const SPECIAL_ROUTES: &str = r#" @websocket path /websocket /websocket/*
uri strip_prefix /sync
reverse_proxy sync-service:8787
}
+"#;
+
+const MAILPIT_ROUTE: &str = r#" # Mailpit serves itself under /mailpit (MP_WEBROOT), so no prefix strip —
+ # this is how a headless/preview stack reads its passwordless login codes.
+ handle /mailpit/* {
+ reverse_proxy mailpit:8025
+ }
+ redir /mailpit /mailpit/ 308
"#;
@@ -133,6 +150,20 @@ const STATIC_FILE_DEV: &str = r#" handle_path /static-file/* {
}
"#;
+/// Headless mode: the proxy serves the built app bundle (mounted at
+/// `/srv/frontend` — see `gen_compose::add_proxy_service`). The bundle is built
+/// with `base: /app`, so URL space `/app/*` maps onto the dist root after the
+/// prefix strip; unknown paths fall back to `index.html` (SPA routing). Caddy
+/// sorts `redir` before `handle_path`, so the exact-path redirects win first.
+const FRONTEND_STATIC: &str = r#" redir / /app/ 302
+ redir /app /app/ 308
+ handle_path /app/* {
+ root * /srv/frontend
+ try_files {path} /index.html
+ file_server
+ }
+"#;
+
const CADDY_TAIL: &str = r#"
respond "macro local proxy" 200
}
diff --git a/tooling/xtask/crates/xtask_local/src/local/proxy/test.rs b/tooling/xtask/crates/xtask_local/src/local/proxy/test.rs
index ed615a7a4d..ac4a2e9e55 100644
--- a/tooling/xtask/crates/xtask_local/src/local/proxy/test.rs
+++ b/tooling/xtask/crates/xtask_local/src/local/proxy/test.rs
@@ -6,7 +6,7 @@ use crate::local::{inventory, repo_root};
/// the guarantee that replaces the old hand-maintained route list.
#[test]
fn generates_a_route_for_every_prefixed_service() {
- let caddy = caddyfile(Mode::Local);
+ let caddy = caddyfile(Mode::Local, false);
for svc in inventory::RUST_SERVICES {
let Some(prefix) = svc.path_prefix else {
continue;
@@ -27,7 +27,7 @@ fn generates_a_route_for_every_prefixed_service() {
/// services use `handle_path`.
#[test]
fn websocket_services_use_a_matcher() {
- let caddy = caddyfile(Mode::Local);
+ let caddy = caddyfile(Mode::Local, false);
// connection-gateway is the inventoried WebSocket service.
assert!(caddy.contains("@connection_gateway path /connection-gateway /connection-gateway/*"));
assert!(caddy.contains("uri strip_prefix /connection-gateway"));
@@ -39,9 +39,9 @@ fn websocket_services_use_a_matcher() {
/// fan-out locally, the dev-pointed service in dev.
#[test]
fn static_file_block_is_mode_specific() {
- assert!(caddyfile(Mode::Local).contains("/static-file-storage"));
- assert!(caddyfile(Mode::Dev).contains("handle_path /static-file/*"));
- assert!(!caddyfile(Mode::Dev).contains("/static-file-storage"));
+ assert!(caddyfile(Mode::Local, false).contains("/static-file-storage"));
+ assert!(caddyfile(Mode::Dev, false).contains("handle_path /static-file/*"));
+ assert!(!caddyfile(Mode::Dev, false).contains("/static-file-storage"));
}
/// Drift gate across the Rust↔TypeScript seam: every proxied service's prefix
@@ -67,3 +67,24 @@ fn frontend_wires_every_inventory_prefix() {
);
}
}
+
+/// The static-frontend block only appears in headless mode, and serves the
+/// mounted bundle under `/app` with an SPA fallback. Attached `run_local` keeps
+/// the dev server as the frontend origin and must not grow the block.
+#[test]
+fn static_frontend_block_is_opt_in() {
+ let headless = caddyfile(Mode::Local, true);
+ assert!(headless.contains("handle_path /app/* {"));
+ assert!(headless.contains("root * /srv/frontend"));
+ assert!(headless.contains("try_files {path} /index.html"));
+ assert!(headless.contains("redir / /app/ 302"));
+ assert!(headless.contains("handle /mailpit/*"));
+
+ let attached = caddyfile(Mode::Local, false);
+ assert!(!attached.contains("/srv/frontend"));
+ assert!(!attached.contains("redir / /app/ 302"));
+ assert!(!attached.contains("handle /mailpit/*"));
+
+ let headless_dev = caddyfile(Mode::Dev, true);
+ assert!(!headless_dev.contains("handle /mailpit/*"));
+}
diff --git a/tooling/xtask/crates/xtask_local/src/local/snapshot.rs b/tooling/xtask/crates/xtask_local/src/local/snapshot.rs
new file mode 100644
index 0000000000..c25b362c46
--- /dev/null
+++ b/tooling/xtask/crates/xtask_local/src/local/snapshot.rs
@@ -0,0 +1,321 @@
+//! Content-addressed snapshots of the stack's *initialized* infra state.
+//!
+//! A cold bring-up pays for the expensive one-time init — DB migrations, the
+//! FusionAuth kickstart (~a minute), OpenSearch index creation. That state is
+//! fully determined by a small set of inputs (the migrations, the generated
+//! kickstart, the index mappings, the infra image pins, the container
+//! platform), so instead of re-running the init on every `stack up`, we key
+//! it: hash the inputs, and if
+//! a snapshot of the stateful volumes exists under that key, restore it
+//! and skip the init entirely. The full-delete/full-create idempotency
+//! guarantee survives because the key *is* the definition of "clean" — any
+//! change to an input misses the cache and falls back to a real init (which
+//! then saves the new snapshot).
+//!
+//! Snapshots are tarballs of the Docker volumes, written by a throwaway helper
+//! container, stored under `infra/local/generated/.snapshots/<key>/` (override
+//! with `MACRO_STACK_SNAPSHOT_DIR` — CI bakes this dir into preview images).
+
+use std::path::PathBuf;
+use std::process::Command;
+
+use anyhow::{Context, Result, bail};
+use serde::{Deserialize, Serialize};
+use sha2::{Digest, Sha256};
+
+use super::instance::Instance;
+use super::stage::Stage;
+use super::{env_layer, fusionauth, gen_compose, repo_root, workspace_root};
+
+/// Bump when the snapshot mechanism itself changes shape (archive layout,
+/// volume set) so old snapshots can't be restored into a new scheme.
+/// 2: Kafka's volume joined the set (topics live in the broker data dir, so a
+/// restore carries them and the rdkafka-backed provisioning is skipped).
+/// 3: invalidates snapshots saved while the FusionAuth kickstart could still
+/// be mid-flight (the readiness gate only checked `/api/status`, so a save
+/// could freeze a tenant-less FusionAuth DB — and the key never changed, so
+/// the bad snapshot was sticky).
+const FORMAT: u32 = 3;
+
+/// The throwaway container image used to tar/untar volumes. Alpine for its
+/// size; only needs `tar` + `sh`.
+const HELPER_IMAGE: &str = "alpine:3";
+
+/// The stateful volumes a snapshot captures, as `(archive name, volume)`
+/// pairs. Redis is deliberately absent — it's a cache, and an empty cache is
+/// valid clean state.
+fn archives(instance: &Instance) -> [(&'static str, String); 5] {
+ [
+ ("postgres.tar.gz", instance.volume_postgres()),
+ ("opensearch.tar.gz", instance.volume_opensearch()),
+ ("kafka.tar.gz", instance.volume_kafka()),
+ ("fusionauth-db.tar.gz", instance.volume_fusionauth_db()),
+ (
+ "fusionauth-config.tar.gz",
+ instance.volume_fusionauth_config(),
+ ),
+ ]
+}
+
+/// The compose services that own those volumes — quiesced (stopped) while the
+/// archives are written so the files are consistent.
+const STATEFUL_SERVICES: &[&str] = &["postgres", "search", "kafka", "fusionauth", "db"];
+
+/// Where snapshots live. `MACRO_STACK_SNAPSHOT_DIR` overrides for CI bakes and
+/// preview images; the default sits inside the gitignored generated dir.
+// xtask is host tooling, not a service reading APP_SECRETS_JSON, so reading the
+// process environment directly is correct here.
+#[allow(clippy::disallowed_methods)]
+pub fn root_dir() -> PathBuf {
+ std::env::var_os("MACRO_STACK_SNAPSHOT_DIR")
+ .map(PathBuf::from)
+ .unwrap_or_else(|| repo_root().join("infra/local/generated/.snapshots"))
+}
+
+/// What a snapshot directory contains, recorded alongside the archives.
+#[derive(Serialize, Deserialize)]
+struct Manifest {
+ format: u32,
+ key: String,
+ created_unix: u64,
+ archives: Vec<String>,
+}
+
+/// A resolved snapshot decision for one instance: the content key and where
+/// that key's snapshot would live. `exists()` says which way `up` goes.
+pub struct Plan {
+ pub key: String,
+ pub dir: PathBuf,
+}
+
+impl Plan {
+ /// Hash the init-defining inputs. Must be called after `prepare` (the
+ /// generated kickstart is one of the inputs — it encodes the instance's
+ /// ports, so a snapshot can never be restored into an instance whose
+ /// FusionAuth config wouldn't match).
+ pub fn compute(instance: &Instance) -> Result<Plan> {
+ let mut h = Sha256::new();
+ h.update(FORMAT.to_le_bytes());
+
+ // The container platform. Volume bytes are written by arch-specific
+ // images (an Apple Silicon Postgres data dir is arm64-born), and while
+ // Postgres/Lucene data happens to be portable across amd64/arm64 in
+ // practice, that's unsupported territory — keying on the platform makes
+ // cross-arch restore a structural cache miss instead of a latent bug
+ // if snapshots are ever shared between machines (e.g. an S3 cache).
+ // Today each snapshot store is machine-local or CI-baked for
+ // same-arch Fly machines, so this is insurance, not a behavior change.
+ h.update(super::arch::detect()?.docker_platform.as_bytes());
+
+ // Infra image pins + topology.
+ for rel in [
+ "docker/docker-compose.yml",
+ "docker/docker-compose-databases.yml",
+ "infra/stacks/fusionauth-instance/docker-compose.yml",
+ ] {
+ hash_file(&mut h, &repo_root().join(rel))?;
+ }
+ // The custom local OpenSearch image inputs.
+ hash_dir(&mut h, &repo_root().join("infra/local/opensearch"), &[])?;
+ // Database schema.
+ hash_dir(
+ &mut h,
+ &workspace_root().join("crates/macro_db_client/migrations"),
+ &[],
+ )?;
+ // Search index mappings (canonical TS helpers; node_modules excluded).
+ hash_dir(
+ &mut h,
+ &repo_root().join("infra/stacks/opensearch/helpers"),
+ &["node_modules"],
+ )?;
+ // The generated FusionAuth kickstart (identity + this instance's ports).
+ hash_file(
+ &mut h,
+ &gen_compose::kickstart_dir(instance).join("kickstart.json"),
+ )?;
+
+ let key = hex(&h.finalize());
+ let dir = root_dir().join(&key);
+ Ok(Plan { key, dir })
+ }
+
+ /// Whether a completed snapshot exists for this key (manifest written last,
+ /// so its presence implies the archives are whole).
+ pub fn exists(&self) -> bool {
+ std::fs::read_to_string(self.dir.join("manifest.json"))
+ .ok()
+ .and_then(|raw| serde_json::from_str::<Manifest>(&raw).ok())
+ .is_some_and(|m| m.format == FORMAT && m.key == self.key)
+ }
+
+ /// The first 12 hex chars — enough to name it in output.
+ pub fn short_key(&self) -> &str {
+ &self.key[..12]
+ }
+}
+
+/// Unpack the snapshot's archives into the instance's (freshly created, empty)
+/// volumes. Runs before the infra containers start.
+pub fn restore(stage: &Stage, instance: &Instance, plan: &Plan) -> Result<()> {
+ let dir = plan.dir.clone();
+ let pairs = archives(instance);
+ stage.run_step(
+ &format!("Restoring init snapshot ({})", plan.short_key()),
+ || {
+ for (archive, volume) in &pairs {
+ if !dir.join(archive).exists() {
+ bail!("snapshot {} is missing {archive}", dir.display());
+ }
+ helper_tar(
+ &format!("{volume}:/vol"),
+ &format!("{}:/snap:ro", dir.display()),
+ &format!("tar xzf /snap/{archive} -C /vol"),
+ )?;
+ }
+ Ok(())
+ },
+ )
+}
+
+/// Archive the just-initialized volumes under the plan's key. Called after the
+/// infra init and BEFORE the app services start (nothing is connected yet, so
+/// the stateful containers can be stopped for a consistent copy and started
+/// again). Written to a temp dir and renamed in, so a crash never leaves a
+/// half-snapshot that `exists()` would trust.
+pub fn save(
+ stage: &Stage,
+ instance: &Instance,
+ env: &env_layer::ResolvedEnv,
+ plan: &Plan,
+) -> Result<()> {
+ let mut stop = super::compose_cmd(instance, env);
+ stop.args(["stop", "-t", "30"]).args(STATEFUL_SERVICES);
+ stage.run("Quiescing infra for snapshot", &mut stop)?;
+
+ let tmp = root_dir().join(format!(".tmp-{}", plan.key));
+ let pairs = archives(instance);
+ let write = || -> Result<()> {
+ if tmp.exists() {
+ std::fs::remove_dir_all(&tmp)?;
+ }
+ std::fs::create_dir_all(&tmp)?;
+ for (archive, volume) in &pairs {
+ helper_tar(
+ &format!("{volume}:/vol:ro"),
+ &format!("{}:/snap", tmp.display()),
+ &format!("tar czf /snap/{archive} -C /vol ."),
+ )?;
+ }
+ let manifest = Manifest {
+ format: FORMAT,
+ key: plan.key.clone(),
+ created_unix: std::time::SystemTime::now()
+ .duration_since(std::time::UNIX_EPOCH)
+ .map(|d| d.as_secs())
+ .unwrap_or(0),
+ archives: pairs.iter().map(|(a, _)| a.to_string()).collect(),
+ };
+ std::fs::write(
+ tmp.join("manifest.json"),
+ serde_json::to_string_pretty(&manifest)?,
+ )?;
+ if plan.dir.exists() {
+ std::fs::remove_dir_all(&plan.dir)?;
+ }
+ std::fs::rename(&tmp, &plan.dir).context("moving snapshot into place")?;
+ Ok(())
+ };
+ stage.run_step(
+ &format!("Saving init snapshot ({})", plan.short_key()),
+ write,
+ )?;
+
+ // Bring the quiesced infra back to ready before the app services start.
+ let mut up = super::compose_cmd(instance, env);
+ up.args(["up", "-d", "--wait", "postgres", "search", "kafka"]);
+ stage.run("Restarting infra after snapshot", &mut up)?;
+ let mut fa = super::compose_cmd(instance, env);
+ fa.args(["up", "-d", "fusionauth"]);
+ stage.run("Restarting FusionAuth", &mut fa)?;
+ fusionauth::wait_ready(stage, instance)
+}
+
+/// Run `tar` in the helper container with two mounts. The stack's own stage
+/// plumbing is bypassed on purpose — these run inside `run_step` parents.
+fn helper_tar(vol_mount: &str, snap_mount: &str, script: &str) -> Result<()> {
+ let out = Command::new("docker")
+ .args(["run", "--rm", "-v", vol_mount, "-v", snap_mount])
+ .arg(HELPER_IMAGE)
+ .args(["sh", "-ceu", script])
+ .output()
+ .context("running snapshot helper container")?;
+ if !out.status.success() {
+ bail!(
+ "snapshot helper failed ({script}):\n{}",
+ String::from_utf8_lossy(&out.stderr)
+ );
+ }
+ Ok(())
+}
+
+fn hash_file(h: &mut Sha256, path: &std::path::Path) -> Result<()> {
+ let bytes = std::fs::read(path)
+ .with_context(|| format!("reading snapshot input {}", path.display()))?;
+ h.update(path.file_name().unwrap_or_default().as_encoded_bytes());
+ h.update((bytes.len() as u64).to_le_bytes());
+ h.update(&bytes);
+ Ok(())
+}
+
+/// Hash a directory tree deterministically: sorted relative paths + contents.
+fn hash_dir(h: &mut Sha256, dir: &std::path::Path, exclude: &[&str]) -> Result<()> {
+ let mut files = Vec::new();
+ collect_files(dir, dir, exclude, &mut files)?;
+ files.sort();
+ for rel in files {
+ let path = dir.join(&rel);
+ let bytes = std::fs::read(&path)
+ .with_context(|| format!("reading snapshot input {}", path.display()))?;
+ h.update(rel.as_bytes());
+ h.update((bytes.len() as u64).to_le_bytes());
+ h.update(&bytes);
+ }
+ Ok(())
+}
+
+fn collect_files(
+ root: &std::path::Path,
+ dir: &std::path::Path,
+ exclude: &[&str],
+ out: &mut Vec<String>,
+) -> Result<()> {
+ let entries = std::fs::read_dir(dir)
+ .with_context(|| format!("reading snapshot input dir {}", dir.display()))?;
+ for entry in entries {
+ let entry = entry?;
+ let name = entry.file_name().to_string_lossy().to_string();
+ if exclude.contains(&name.as_str()) {
+ continue;
+ }
+ let path = entry.path();
+ if path.is_dir() {
+ collect_files(root, &path, exclude, out)?;
+ } else {
+ out.push(
+ path.strip_prefix(root)
+ .expect("child path is under its root")
+ .to_string_lossy()
+ .to_string(),
+ );
+ }
+ }
+ Ok(())
+}
+
+fn hex(bytes: &[u8]) -> String {
+ bytes.iter().map(|b| format!("{b:02x}")).collect()
+}
+
+#[cfg(test)]
+mod test;
diff --git a/tooling/xtask/crates/xtask_local/src/local/snapshot/test.rs b/tooling/xtask/crates/xtask_local/src/local/snapshot/test.rs
new file mode 100644
index 0000000000..38775c42dc
--- /dev/null
+++ b/tooling/xtask/crates/xtask_local/src/local/snapshot/test.rs
@@ -0,0 +1,73 @@
+use super::*;
+use crate::local::instance::Instance;
+
+/// The key must change when any init-defining input changes, and only then —
+/// two computes over identical inputs agree, and the kickstart (which encodes
+/// the instance's ports) is part of the key, so snapshots can't cross
+/// instances with different port windows.
+#[test]
+fn key_is_deterministic_and_kickstart_sensitive() {
+ let instance = Instance::derive(None, None).unwrap();
+ crate::local::fusionauth::write_kickstart(&instance).unwrap();
+
+ let a = Plan::compute(&instance).unwrap();
+ let b = Plan::compute(&instance).unwrap();
+ assert_eq!(a.key, b.key, "same inputs must produce the same key");
+ assert_eq!(a.key.len(), 64, "sha256 hex");
+
+ // A named instance generates a kickstart with different ports → new key.
+ let other = Instance::derive(Some("snapshot-key-test"), None).unwrap();
+ crate::local::fusionauth::write_kickstart(&other).unwrap();
+ let c = Plan::compute(&other).unwrap();
+ assert_ne!(
+ a.key, c.key,
+ "instances with different ports must not share snapshots"
+ );
+
+ // Cleanup the test instance's generated dir.
+ let _ = std::fs::remove_dir_all(other.artifact_dir());
+}
+
+/// `exists()` trusts only a manifest that matches the key and format — a
+/// half-written or foreign directory is a cache miss, not a restore.
+#[test]
+fn exists_requires_a_matching_manifest() {
+ let dir = std::env::temp_dir().join("macro-snapshot-exists-test");
+ let _ = std::fs::remove_dir_all(&dir);
+ std::fs::create_dir_all(&dir).unwrap();
+
+ let plan = Plan {
+ key: "k".repeat(64),
+ dir: dir.clone(),
+ };
+ assert!(!plan.exists(), "no manifest → miss");
+
+ std::fs::write(dir.join("manifest.json"), "not json").unwrap();
+ assert!(!plan.exists(), "garbage manifest → miss");
+
+ let manifest = Manifest {
+ format: FORMAT,
+ key: plan.key.clone(),
+ created_unix: 0,
+ archives: vec![],
+ };
+ std::fs::write(
+ dir.join("manifest.json"),
+ serde_json::to_string(&manifest).unwrap(),
+ )
+ .unwrap();
+ assert!(plan.exists(), "matching manifest → hit");
+
+ let mismatched = Manifest {
+ key: "x".repeat(64),
+ ..manifest
+ };
+ std::fs::write(
+ dir.join("manifest.json"),
+ serde_json::to_string(&mismatched).unwrap(),
+ )
+ .unwrap();
+ assert!(!plan.exists(), "key mismatch → miss");
+
+ let _ = std::fs::remove_dir_all(&dir);
+}
diff --git a/tooling/xtask/crates/xtask_local/src/local/stack.rs b/tooling/xtask/crates/xtask_local/src/local/stack.rs
new file mode 100644
index 0000000000..5ad2cadf7d
--- /dev/null
+++ b/tooling/xtask/crates/xtask_local/src/local/stack.rs
@@ -0,0 +1,690 @@
+//! Headless stack orchestration: `cargo x stack up|update|status|down`.
+//!
+//! The preview/agent/CI surface. Same bring-up as `run_local`, but no TTY
+//! hotkey loop and no attached dev server: the frontend is a static bundle
+//! served by the instance's Caddy, so a finished `up` leaves only Docker
+//! containers running — nothing to babysit, and the whole product lives behind
+//! the ONE proxy origin that an agent drives with a browser and a preview
+//! deploy fronts with a real hostname.
+//!
+//! `up` is full-delete/full-create like `run_local` (unconditionally
+//! idempotent); `update` is the `r`-hotkey as a one-shot verb (rebuild, restart
+//! only what changed); `status` is machine-readable state; `down` reclaims
+//! everything.
+
+use std::io::{BufReader, Read};
+use std::path::{Path, PathBuf};
+use std::process::Command;
+
+use anyhow::{Context, Result, bail};
+use clap::Args;
+use serde::{Deserialize, Serialize};
+use serde_json::json;
+
+use super::cli::{EnvArgs, InstanceArgs, RunArgs};
+use super::instance::{Instance, Port};
+use super::stage::Stage;
+use super::{Mode, arch, env_layer, frontend, mailpit, proxy, snapshot, summary};
+
+#[derive(Args, Clone, Default)]
+pub struct UpArgs {
+ #[command(flatten)]
+ pub run: RunArgs,
+ /// Stage this prebuilt frontend dist instead of building (CI artifact reuse).
+ #[arg(long)]
+ pub frontend_dist: Option<PathBuf>,
+ /// Neither restore from nor save an init snapshot — always run the full
+ /// migrate/kickstart/index init.
+ #[arg(long)]
+ pub no_snapshot: bool,
+ /// Stop after the infra bring-up + init (and the snapshot save/restore):
+ /// no app services, proxy, or frontend. This is the CI bake mode — the
+ /// app services need the Doppler-sourced env (AWS endpoints, shared
+ /// secrets) to even boot, which a bake environment deliberately lacks,
+ /// and the init snapshot only captures the infra volumes anyway.
+ #[arg(long)]
+ pub infra_only: bool,
+ /// Print a machine-readable JSON summary as the final line.
+ #[arg(long)]
+ pub json: bool,
+}
+
+#[derive(Args, Clone, Default)]
+pub struct SnapshotArgs {
+ #[command(flatten)]
+ pub instance: InstanceArgs,
+ /// Print machine-readable JSON (key, dir, present).
+ #[arg(long)]
+ pub json: bool,
+}
+
+#[derive(Args, Clone, Default)]
+pub struct UpdateArgs {
+ #[command(flatten)]
+ pub instance: InstanceArgs,
+ #[command(flatten)]
+ pub env: EnvArgs,
+ /// Also rebuild + restage the static frontend and recreate the proxy.
+ #[arg(long)]
+ pub frontend: bool,
+ /// Stage this prebuilt frontend dist instead of building (implies --frontend).
+ #[arg(long)]
+ pub frontend_dist: Option<PathBuf>,
+ /// Apply this complete prebuilt binary set instead of invoking Cargo.
+ #[arg(long, conflicts_with = "build_aux_services")]
+ pub binaries_dir: Option<PathBuf>,
+ /// Rebuild the Docker-built auxiliary services (sync, websocket, lexical).
+ #[arg(long)]
+ pub build_aux_services: bool,
+ /// Recreate registry-refreshed preview auxiliary services without building.
+ #[arg(long, hide = true, requires = "binaries_dir")]
+ pub recreate_aux_services: bool,
+ /// Stream subprocess output and show per-step timings.
+ #[arg(long, short)]
+ pub verbose: bool,
+ /// Print a machine-readable summary as the final line.
+ #[arg(long, hide = true, requires = "binaries_dir")]
+ pub json: bool,
+}
+
+#[derive(Args, Clone, Default)]
+pub struct StatusArgs {
+ #[command(flatten)]
+ pub instance: InstanceArgs,
+ /// Print machine-readable JSON instead of the human summary.
+ #[arg(long)]
+ pub json: bool,
+}
+
+#[derive(Args, Clone, Default)]
+pub struct DownArgs {
+ #[command(flatten)]
+ pub instance: InstanceArgs,
+ /// Stop containers but keep volumes (data survives the next `stack up`...
+ /// which wipes them anyway — use `just run_local`/`stop_local` semantics).
+ #[arg(long)]
+ pub keep_data: bool,
+}
+
+/// Durable per-instance record of what `up` brought up, so `update`/`status`
+/// don't need the flags repeated. Lives in the instance artifact dir.
+#[derive(Serialize, Deserialize)]
+struct StackState {
+ /// `local` / `dev` — mirrors [`Mode::label`].
+ mode: String,
+ /// `static` (Caddy serves the bundle) or `none` (`--no-frontend`).
+ frontend: String,
+ /// Stable directory bind-mounted into every Rust service. Prebuilt updates
+ /// replace files here atomically so existing container mounts see them.
+ #[serde(default)]
+ binaries_dir: Option<PathBuf>,
+}
+
+fn state_path(instance: &Instance) -> PathBuf {
+ instance.artifact_dir().join("stack.json")
+}
+
+fn write_state(instance: &Instance, state: &StackState) -> Result<()> {
+ instance.ensure_artifact_dir()?;
+ let path = state_path(instance);
+ std::fs::write(&path, serde_json::to_string_pretty(state)?)
+ .with_context(|| format!("writing {}", path.display()))
+}
+
+fn read_state(instance: &Instance) -> Option<StackState> {
+ let raw = std::fs::read_to_string(state_path(instance)).ok()?;
+ serde_json::from_str(&raw).ok()
+}
+
+/// Invalidate headless ownership before another flow replaces this instance.
+/// State is written again only after a full headless stack passes its health
+/// gate, so failed, infra-only, and interactive replacements cannot inherit it.
+pub(super) fn clear_state(instance: &Instance) -> Result<()> {
+ let path = state_path(instance);
+ match std::fs::remove_file(&path) {
+ Ok(()) => Ok(()),
+ Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(()),
+ Err(e) => Err(e).with_context(|| format!("removing {}", path.display())),
+ }
+}
+
+/// `cargo x stack up` — bring the whole stack up and return, leaving only
+/// Docker containers running.
+pub fn up(mode: Mode, args: &UpArgs) -> Result<()> {
+ let stage = Stage::from_env_cli(args.run.verbose);
+ let instance = Instance::derive(
+ args.run.instance.instance.as_deref(),
+ args.run.instance.port_base,
+ )?;
+ stage.section(&format!(
+ "macro {} stack (headless) — instance {}",
+ mode.label(),
+ instance.name()
+ ));
+ if !stage.is_dry_run() {
+ clear_state(&instance)?;
+ }
+
+ // Same full-delete/full-create overlap as `run_stack`: tear the previous
+ // stack down in the background while the host-side build runs.
+ let teardown = (!stage.is_dry_run()).then(|| {
+ let instance = instance.clone();
+ std::thread::spawn(move || super::teardown_commands(&instance))
+ });
+
+ let static_frontend = !args.run.no_frontend && !args.infra_only;
+ let (env, target) = super::prepare(&stage, mode, &instance, &args.run, static_frontend)?;
+ let configured_binaries = args
+ .run
+ .build
+ .binaries_dir
+ .clone()
+ .unwrap_or_else(|| super::workspace_root().join(target.debug_dir()));
+ let active_binaries = super::build::BinariesDir::classify(&configured_binaries)?
+ .host_dir()
+ .to_path_buf();
+
+ // Build + stage the frontend bundle in the background: it's pure host-side
+ // work, independent of Docker until the proxy container mounts the staged
+ // dir — so it overlaps the teardown join and the infra bring-up, and is
+ // joined just before `bring_up_app` creates that container. (Dry run:
+ // synchronously, so the command preview prints in order.)
+ if static_frontend && stage.is_dry_run() {
+ frontend::build_static(&stage, &instance, args.frontend_dist.as_deref())?;
+ }
+ let fe_build = (static_frontend && !stage.is_dry_run()).then(|| {
+ let instance = instance.clone();
+ let prebuilt = args.frontend_dist.clone();
+ std::thread::spawn(move || {
+ frontend::build_static(&Stage::from_env().quiet(), &instance, prebuilt.as_deref())
+ })
+ });
+
+ // The init snapshot decision: hash the init-defining inputs (possible only
+ // after `prepare` wrote the kickstart) and check for a stored snapshot.
+ // Restores skip migrate/kickstart/index-init; a cold init saves one for
+ // next time — that's how the cache seeds itself.
+ let snapshot_plan = (!args.no_snapshot && !stage.is_dry_run())
+ .then(|| snapshot::Plan::compute(&instance))
+ .transpose()?;
+
+ if let Some(handle) = teardown {
+ stage.run_step("Tearing down previous stack", move || {
+ let _ = handle.join();
+ Ok(())
+ })?;
+ }
+ super::ensure_external_resources(&stage, &instance)?;
+ match &snapshot_plan {
+ Some(plan) if plan.exists() => {
+ snapshot::restore(&stage, &instance, plan)?;
+ super::bring_up_infra(
+ &stage,
+ mode,
+ &instance,
+ &env,
+ super::InfraInit::FromSnapshot,
+ )?;
+ }
+ Some(plan) => {
+ super::bring_up_infra(&stage, mode, &instance, &env, super::InfraInit::Full)?;
+ snapshot::save(&stage, &instance, &env, plan)?;
+ }
+ None => {
+ super::bring_up_infra(&stage, mode, &instance, &env, super::InfraInit::Full)?;
+ }
+ }
+ if let Some(handle) = fe_build {
+ stage.run_step("Building frontend (static bundle)", move || {
+ handle
+ .join()
+ .map_err(|_| anyhow::anyhow!("frontend build panicked"))?
+ })?;
+ }
+ if args.infra_only {
+ // No app stack, no proxy: nothing to summarize or gate on, and no
+ // stack.json — `update`/`status` on an infra-only bake should say
+ // "bring the stack up first" rather than assume a running app.
+ stage.note(" infra-only: app services, proxy, and frontend skipped");
+ if args.json {
+ println!(
+ "{}",
+ serde_json::to_string(&summary_json(mode, &instance, false))?
+ );
+ }
+ return Ok(());
+ }
+ super::bring_up_app(&stage, mode, &instance, &env)?;
+
+ // Headless "ready" means the backend answers through the proxy — the caller
+ // (a CI step, an agent) acts on the URL the moment we return.
+ if mode.spec().wait_backend_before_frontend {
+ frontend::wait_backend_ready(&stage, &instance)?;
+ }
+
+ if !stage.is_dry_run() {
+ write_state(
+ &instance,
+ &StackState {
+ mode: mode.label().to_string(),
+ frontend: if static_frontend { "static" } else { "none" }.to_string(),
+ binaries_dir: Some(active_binaries),
+ },
+ )?;
+ }
+
+ let frontend_url = if static_frontend {
+ frontend::static_url(&instance)
+ } else {
+ "(disabled — --no-frontend)".to_string()
+ };
+ let mailpit_url = if static_frontend {
+ mailpit::proxy_ui_url(&instance)
+ } else {
+ mailpit::direct_ui_url(&instance)
+ };
+ summary::print(mode, &instance, &env, &frontend_url, &mailpit_url);
+ stage.note(&format!(
+ " headless: `just stack status`, `just stack update`, `just stack down`{}",
+ instance_suffix(&instance)
+ ));
+ if args.json {
+ println!(
+ "{}",
+ serde_json::to_string(&summary_json(mode, &instance, static_frontend))?
+ );
+ }
+ Ok(())
+}
+
+/// `cargo x stack update` — the `r` hotkey as a one-shot verb: rebuild the
+/// binaries (and optionally the frontend bundle) and restart only what changed.
+pub fn update(args: &UpdateArgs) -> Result<()> {
+ let stage = Stage::from_env_cli(args.verbose);
+ let instance = Instance::derive(args.instance.instance.as_deref(), args.instance.port_base)?;
+ let Some(state) = read_state(&instance) else {
+ bail!(
+ "no stack state at {} — bring the stack up first (`just stack up{}`)",
+ state_path(&instance).display(),
+ instance_suffix(&instance)
+ );
+ };
+ let mode = mode_from_label(&state.mode)?;
+ stage.section(&format!(
+ "macro {} stack update — instance {}",
+ mode.label(),
+ instance.name()
+ ));
+
+ let env = env_layer::resolve(
+ mode,
+ &instance,
+ args.env.no_doppler,
+ args.env.env_file.as_deref(),
+ )?;
+ let changed = if let Some(source) = args.binaries_dir.as_deref() {
+ let changed = apply_prebuilt_binaries(mode, &state, source)?;
+ if args.recreate_aux_services {
+ super::recreate_preview_aux_service_containers(&stage, &instance, &env)?;
+ }
+ if !changed.is_empty() {
+ super::reload_services(&stage, &instance, &changed)?;
+ }
+ changed
+ } else {
+ let target = arch::detect()?;
+ super::rebuild_and_reload(
+ &stage,
+ mode,
+ &instance,
+ &env,
+ target,
+ args.build_aux_services,
+ )?;
+ Vec::new()
+ };
+
+ if args.frontend || args.frontend_dist.is_some() {
+ reload_static_frontend(
+ &stage,
+ &instance,
+ &env,
+ &state,
+ args.frontend_dist.as_deref(),
+ )?;
+ }
+ if mode.spec().wait_backend_before_frontend {
+ frontend::wait_backend_ready(&stage, &instance)?;
+ }
+ if args.json {
+ println!(
+ "{}",
+ serde_json::to_string(&json!({
+ "changed_services": changed.iter().map(|s| s.compose_name).collect::<Vec<_>>(),
+ "frontend_updated": args.frontend || args.frontend_dist.is_some(),
+ "aux_services_recreated": args.recreate_aux_services,
+ }))?
+ );
+ }
+ Ok(())
+}
+
+fn apply_prebuilt_binaries(
+ mode: Mode,
+ state: &StackState,
+ binaries_dir: &Path,
+) -> Result<Vec<&'static super::inventory::RustService>> {
+ let destination = state.binaries_dir.as_ref().ok_or_else(|| {
+ anyhow::anyhow!(
+ "stack state predates prebuilt updates — perform a full stack up before updating"
+ )
+ })?;
+ let source = super::build::BinariesDir::classify(binaries_dir)?;
+ source.validate(&super::inventory::local_binaries())?;
+ let mut changed = Vec::new();
+ for service in super::inventory::services_for_mode(mode) {
+ if replace_binary_if_changed(source.host_dir(), destination, service.cargo_bin)? {
+ changed.push(service);
+ }
+ }
+
+ Ok(changed)
+}
+
+fn reload_static_frontend(
+ stage: &Stage,
+ instance: &Instance,
+ env: &env_layer::ResolvedEnv,
+ state: &StackState,
+ frontend_dist: Option<&Path>,
+) -> Result<()> {
+ if state.frontend != "static" {
+ bail!(
+ "this stack was brought up without a static frontend (--no-frontend); \
+ re-run `just stack up` to serve one"
+ );
+ }
+ frontend::build_static(stage, instance, frontend_dist)?;
+ // build_static replaces the staged directory, so the container must be
+ // recreated to establish a bind mount to the new inode.
+ let mut up = super::compose_cmd(instance, env);
+ up.args(["up", "-d", "--force-recreate", "--no-deps", "proxy"]);
+ stage.run("Recreating proxy (frontend bundle)", &mut up)
+}
+
+fn replace_binary_if_changed(
+ source_dir: &Path,
+ destination_dir: &Path,
+ name: &str,
+) -> Result<bool> {
+ let source = source_dir.join(name);
+ let destination = destination_dir.join(name);
+ if files_equal(&source, &destination)? {
+ return Ok(false);
+ }
+ std::fs::create_dir_all(destination_dir)
+ .with_context(|| format!("creating {}", destination_dir.display()))?;
+ let temporary = destination_dir.join(format!(".{name}.hot-update-{}", std::process::id()));
+ let _ = std::fs::remove_file(&temporary);
+ std::fs::copy(&source, &temporary).with_context(|| {
+ format!(
+ "copying prebuilt binary {} to {}",
+ source.display(),
+ temporary.display()
+ )
+ })?;
+ std::fs::rename(&temporary, &destination).with_context(|| {
+ format!(
+ "atomically replacing {} with {}",
+ destination.display(),
+ source.display()
+ )
+ })?;
+ Ok(true)
+}
+
+fn files_equal(left: &Path, right: &Path) -> Result<bool> {
+ let left_meta = std::fs::metadata(left)
+ .with_context(|| format!("reading binary metadata for {}", left.display()))?;
+ let Ok(right_meta) = std::fs::metadata(right) else {
+ return Ok(false);
+ };
+ if left_meta.len() != right_meta.len() {
+ return Ok(false);
+ }
+ let mut left = BufReader::new(
+ std::fs::File::open(left).with_context(|| format!("opening {}", left.display()))?,
+ );
+ let mut right = BufReader::new(
+ std::fs::File::open(right).with_context(|| format!("opening {}", right.display()))?,
+ );
+ let mut left_buf = [0_u8; 64 * 1024];
+ let mut right_buf = [0_u8; 64 * 1024];
+ loop {
+ let left_read = left.read(&mut left_buf)?;
+ let right_read = right.read(&mut right_buf)?;
+ if left_read != right_read || left_buf[..left_read] != right_buf[..right_read] {
+ return Ok(false);
+ }
+ if left_read == 0 {
+ return Ok(true);
+ }
+ }
+}
+
+/// `cargo x stack status` — container states + health through the proxy, as a
+/// human summary or `--json` for machines.
+pub fn status(args: &StatusArgs) -> Result<()> {
+ let instance = Instance::derive(args.instance.instance.as_deref(), args.instance.port_base)?;
+ let state = read_state(&instance);
+ let services = compose_ps(&instance)?;
+ let running = services.iter().any(|s| s.state == "running");
+ let backend_healthy = running && probe(&format!("{}/auth/health", proxy::url(&instance)));
+ let static_frontend = state.as_ref().is_some_and(|s| s.frontend == "static");
+ let frontend_url = static_frontend.then(|| frontend::static_url(&instance));
+ if args.json {
+ let mut out = summary_json(
+ state.as_ref().map_or(Mode::Local, |s| {
+ mode_from_label(&s.mode).unwrap_or(Mode::Local)
+ }),
+ &instance,
+ static_frontend,
+ );
+ out["running"] = json!(running);
+ out["backend_healthy"] = json!(backend_healthy);
+ out["services"] = json!(services);
+ println!("{}", serde_json::to_string(&out)?);
+ return Ok(());
+ }
+
+ println!(
+ "instance {} (project {}): {}",
+ instance.name(),
+ instance.project_name(),
+ if running { "running" } else { "not running" }
+ );
+ if running {
+ println!(
+ " backend {} ({})",
+ proxy::url(&instance),
+ if backend_healthy {
+ "healthy"
+ } else {
+ "UNHEALTHY"
+ }
+ );
+ if let Some(url) = frontend_url {
+ println!(" frontend {url}");
+ }
+ }
+ for svc in &services {
+ let health = svc
+ .health
+ .as_deref()
+ .filter(|h| !h.is_empty())
+ .map(|h| format!(" ({h})"))
+ .unwrap_or_default();
+ println!(" {:<32} {}{health}", svc.service, svc.state);
+ }
+ if services.is_empty() {
+ println!(" (no containers)");
+ }
+ Ok(())
+}
+
+/// `cargo x stack down` — reclaim the instance: containers, volumes, networks,
+/// and state. `--keep-data` only stops containers.
+pub fn down(args: &DownArgs) -> Result<()> {
+ let instance = Instance::derive(args.instance.instance.as_deref(), args.instance.port_base)?;
+ if args.keep_data {
+ return super::stop(&args.instance);
+ }
+ super::destroy(&args.instance)?;
+ clear_state(&instance)?;
+ Ok(())
+}
+
+/// `cargo x stack snapshot` — report the instance's init-snapshot key and
+/// whether a snapshot exists for it. CI uses `--json` to find the directory to
+/// bake into preview images.
+pub fn snapshot_status(args: &SnapshotArgs) -> Result<()> {
+ let instance = Instance::derive(args.instance.instance.as_deref(), args.instance.port_base)?;
+ // The key hashes the generated kickstart; (re)write it so the verb works
+ // before any `up` has run. Deterministic, so this never changes a key.
+ super::fusionauth::write_kickstart(&instance)?;
+ let plan = snapshot::Plan::compute(&instance)?;
+ if args.json {
+ println!(
+ "{}",
+ serde_json::to_string(&json!({
+ "key": plan.key,
+ "dir": plan.dir,
+ "present": plan.exists(),
+ "root": snapshot::root_dir(),
+ }))?
+ );
+ return Ok(());
+ }
+ println!("key {}", plan.key);
+ println!("dir {}", plan.dir.display());
+ println!(
+ "present {}",
+ if plan.exists() {
+ "yes"
+ } else {
+ "no (next cold `stack up` will save it)"
+ }
+ );
+ Ok(())
+}
+
+/// The machine-readable endpoint block `up --json` and `status --json` share.
+fn summary_json(mode: Mode, instance: &Instance, static_frontend: bool) -> serde_json::Value {
+ let mailpit_url = mode.spec().runs_local_infra.then(|| {
+ if static_frontend {
+ mailpit::proxy_ui_url(instance)
+ } else {
+ mailpit::direct_ui_url(instance)
+ }
+ });
+ json!({
+ "instance": instance.name(),
+ "project": instance.project_name(),
+ "mode": mode.label(),
+ "proxy_url": proxy::url(instance),
+ "frontend_url": static_frontend.then(|| frontend::static_url(instance)),
+ "fusionauth_url": format!("http://localhost:{}", instance.port(Port::FusionAuth)),
+ "mailpit_url": mailpit_url,
+ "localstack_url": format!("http://localhost:{}", instance.port(Port::LocalStack)),
+ "postgres_url": format!(
+ "postgres://user:password@localhost:{}/macrodb",
+ instance.port(Port::Postgres)
+ ),
+ "logs_cmd": format!("docker compose -p {} logs -f", instance.project_name()),
+ })
+}
+
+/// One row of `docker compose ps`.
+#[derive(Serialize)]
+struct ServiceStatus {
+ service: String,
+ state: String,
+ health: Option<String>,
+}
+
+/// `docker compose -p <project> ps --all --format json`, tolerant of both the
+/// NDJSON (v2.21+) and JSON-array output shapes.
+fn compose_ps(instance: &Instance) -> Result<Vec<ServiceStatus>> {
+ let out = Command::new("docker")
+ .args([
+ "compose",
+ "-p",
+ instance.project_name(),
+ "ps",
+ "--all",
+ "--format",
+ "json",
+ ])
+ .output()
+ .context("running `docker compose ps`")?;
+ if !out.status.success() {
+ bail!(
+ "`docker compose ps` failed:\n{}",
+ String::from_utf8_lossy(&out.stderr)
+ );
+ }
+ let text = String::from_utf8_lossy(&out.stdout);
+ let values: Vec<serde_json::Value> = if text.trim_start().starts_with('[') {
+ serde_json::from_str(text.trim()).context("parsing compose ps json array")?
+ } else {
+ text.lines()
+ .filter(|l| !l.trim().is_empty())
+ .map(serde_json::from_str)
+ .collect::<std::result::Result<_, _>>()
+ .context("parsing compose ps ndjson")?
+ };
+ Ok(values
+ .into_iter()
+ .map(|v| ServiceStatus {
+ service: str_field(&v, "Service"),
+ state: str_field(&v, "State"),
+ health: v.get("Health").and_then(|h| h.as_str()).map(str::to_string),
+ })
+ .collect())
+}
+
+fn str_field(v: &serde_json::Value, key: &str) -> String {
+ v.get(key)
+ .and_then(|s| s.as_str())
+ .unwrap_or_default()
+ .to_string()
+}
+
+/// One quick `curl` probe (the tooling already leans on curl for readiness).
+fn probe(url: &str) -> bool {
+ Command::new("curl")
+ .args(["-fsS", "-m", "3", "-o", "/dev/null", url])
+ .status()
+ .map(|s| s.success())
+ .unwrap_or(false)
+}
+
+fn mode_from_label(label: &str) -> Result<Mode> {
+ match label {
+ "local" => Ok(Mode::Local),
+ "dev" => Ok(Mode::Dev),
+ other => bail!("unknown mode '{other}' in stack.json"),
+ }
+}
+
+/// The `--instance <name>` suffix for copy-pasteable hints ("" for the default).
+fn instance_suffix(instance: &Instance) -> String {
+ if instance.is_default() {
+ String::new()
+ } else {
+ format!(" --instance {}", instance.name())
+ }
+}
+
+#[cfg(test)]
+mod test;
diff --git a/tooling/xtask/crates/xtask_local/src/local/stack/test.rs b/tooling/xtask/crates/xtask_local/src/local/stack/test.rs
new file mode 100644
index 0000000000..0d948adee0
--- /dev/null
+++ b/tooling/xtask/crates/xtask_local/src/local/stack/test.rs
@@ -0,0 +1,61 @@
+use super::*;
+
+/// `up` writes stack.json and `update`/`status` read it back — the record and
+/// the mode labels must stay in agreement.
+#[test]
+fn stack_state_roundtrips() {
+ let state = StackState {
+ mode: "local".to_string(),
+ frontend: "static".to_string(),
+ binaries_dir: Some(PathBuf::from("/tmp/binaries")),
+ };
+ let json = serde_json::to_string(&state).unwrap();
+ let back: StackState = serde_json::from_str(&json).unwrap();
+ assert_eq!(back.mode, "local");
+ assert_eq!(back.frontend, "static");
+ assert_eq!(back.binaries_dir, Some(PathBuf::from("/tmp/binaries")));
+ assert!(mode_from_label(&back.mode).is_ok());
+ assert!(mode_from_label("nonsense").is_err());
+}
+
+#[test]
+fn legacy_stack_state_has_no_binaries_dir() {
+ let state: StackState =
+ serde_json::from_str(r#"{"mode":"local","frontend":"static"}"#).unwrap();
+ assert_eq!(state.binaries_dir, None);
+}
+
+#[test]
+fn clearing_state_invalidates_a_previous_headless_stack() {
+ let instance =
+ Instance::derive(Some(&format!("state-clear-{}", std::process::id())), None).unwrap();
+ write_state(
+ &instance,
+ &StackState {
+ mode: "local".to_string(),
+ frontend: "static".to_string(),
+ binaries_dir: None,
+ },
+ )
+ .unwrap();
+ assert!(read_state(&instance).is_some());
+ clear_state(&instance).unwrap();
+ assert!(read_state(&instance).is_none());
+ clear_state(&instance).unwrap();
+ let _ = std::fs::remove_dir_all(instance.artifact_dir());
+}
+
+#[test]
+fn compares_binary_contents_not_just_size() {
+ let root = std::env::temp_dir().join(format!("macro-stack-test-{}", std::process::id()));
+ let _ = std::fs::remove_dir_all(&root);
+ std::fs::create_dir_all(&root).unwrap();
+ let left = root.join("left");
+ let right = root.join("right");
+ std::fs::write(&left, b"same").unwrap();
+ std::fs::write(&right, b"same").unwrap();
+ assert!(files_equal(&left, &right).unwrap());
+ std::fs::write(&right, b"diff").unwrap();
+ assert!(!files_equal(&left, &right).unwrap());
+ std::fs::remove_dir_all(root).unwrap();
+}
diff --git a/tooling/xtask/crates/xtask_local/src/local/summary.rs b/tooling/xtask/crates/xtask_local/src/local/summary.rs
index 84c2c963eb..9751f21c94 100644
--- a/tooling/xtask/crates/xtask_local/src/local/summary.rs
+++ b/tooling/xtask/crates/xtask_local/src/local/summary.rs
@@ -4,10 +4,19 @@ use console::Style;
use super::env_layer::ResolvedEnv;
use super::instance::{Instance, Port};
-use super::{Mode, frontend, mailpit, proxy};
+use super::{Mode, proxy};
/// Print the mode/instance/endpoints block after a successful startup.
-pub fn print(mode: Mode, instance: &Instance, env: &ResolvedEnv) {
+/// `frontend_url` differs by flow: the dev-server origin for `run_local`, the
+/// proxy-served bundle for headless `stack up`; `mailpit_url` follows the same
+/// direct-versus-single-origin distinction.
+pub fn print(
+ mode: Mode,
+ instance: &Instance,
+ env: &ResolvedEnv,
+ frontend_url: &str,
+ mailpit_url: &str,
+) {
let key = Style::new().dim();
let link = Style::new().cyan();
let row = |k: &str, v: String| {
@@ -51,7 +60,7 @@ pub fn print(mode: Mode, instance: &Instance, env: &ResolvedEnv) {
.unwrap_or_else(|| "(none)".into()),
);
row("generated env", env.generated_path.display().to_string());
- row("frontend", frontend::url(instance));
+ row("frontend", frontend_url.to_string());
if mode.spec().show_infra_in_summary {
row("proxy", proxy::url(instance));
@@ -59,7 +68,7 @@ pub fn print(mode: Mode, instance: &Instance, env: &ResolvedEnv) {
"fusionauth",
format!("http://localhost:{}", instance.port(Port::FusionAuth)),
);
- row("mailpit", mailpit::ui_url(instance));
+ row("mailpit", mailpit_url.to_string());
row(
"localstack",
format!("http://localhost:{}", instance.port(Port::LocalStack)),
diff --git a/tooling/xtask/crates/xtask_local/src/local/validate.rs b/tooling/xtask/crates/xtask_local/src/local/validate.rs
index 81e72b9afe..eb38524562 100644
--- a/tooling/xtask/crates/xtask_local/src/local/validate.rs
+++ b/tooling/xtask/crates/xtask_local/src/local/validate.rs
@@ -30,7 +30,7 @@ pub fn local_compose(instance: &Instance, mode: Mode) -> Result<()> {
// target dir — config rendering does not need the binaries to exist.
let target = arch::detect()?;
let binaries = BinariesDir::TargetDir(workspace_root().join(target.debug_dir()));
- gen_compose::generate(mode, instance, &binaries)?;
+ gen_compose::generate(mode, instance, &binaries, false)?;
let resolved = env_layer::resolve(mode, instance, true, None)?;
let files = gen_compose::compose_files(instance);
diff --git a/tooling/xtask/crates/xtask_paths/src/lib.rs b/tooling/xtask/crates/xtask_paths/src/lib.rs
index 572c4a01bd..c2452fd7b5 100644
--- a/tooling/xtask/crates/xtask_paths/src/lib.rs
+++ b/tooling/xtask/crates/xtask_paths/src/lib.rs
@@ -1,7 +1,7 @@
#![deny(missing_docs)]
//! Shared path discovery for the xtask command crates.
//!
-//! Every xtask command needs to locate the repository Rust workspace root (to
+//! Every xtask command needs to locate the repository Cargo workspace root (to
//! anchor `cargo metadata`, read member manifests, spawn `wasm-pack`, …) and
//! the repository root (where `docker/`, `infra/`, `apps/`, and the root
//! `justfile` live). Both are derived from this crate's own
@@ -19,8 +19,14 @@ use std::path::{Path, PathBuf};
/// The repository Cargo workspace root.
///
/// `env!("CARGO_MANIFEST_DIR")` expands to this crate's directory, which is a
-/// fixed location, so the depth here does not depend on the caller.
+/// fixed location, so the depth here does not depend on the caller. Packaged
+/// xtask binaries can set `MACRO_REPO_ROOT` when they run against a staged copy
+/// of the repository rather than the checkout where they were compiled.
pub fn workspace_root() -> PathBuf {
+ if let Some(root) = repo_root_override() {
+ return root;
+ }
+
// <workspace>/tooling/xtask/crates/xtask_paths -> nth(4) == <workspace>.
Path::new(env!("CARGO_MANIFEST_DIR"))
.ancestors()
@@ -34,6 +40,15 @@ pub fn repo_root() -> PathBuf {
workspace_root()
}
+/// Read the runtime repository root used by packaged xtask binaries.
+///
+/// Environment variables are appropriate here because this tooling crate runs
+/// both inside and outside the service runtime, including in the preview VM.
+#[allow(clippy::disallowed_methods)]
+fn repo_root_override() -> Option<PathBuf> {
+ std::env::var_os("MACRO_REPO_ROOT").map(PathBuf::from)
+}
+
/// A UTF-8, repository-relative path that must resolve to a regular file.
#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)]
pub struct RepoFile<'a>(&'a str);
diff --git a/tooling/xtask/crates/xtask_workflows/src/workflows/mod.rs b/tooling/xtask/crates/xtask_workflows/src/workflows/mod.rs
index 12f3fc4bf9..f37a0694ce 100644
--- a/tooling/xtask/crates/xtask_workflows/src/workflows/mod.rs
+++ b/tooling/xtask/crates/xtask_workflows/src/workflows/mod.rs
@@ -28,6 +28,7 @@ mod deploy_preview;
mod deploy_web_app;
mod deploy_web_app_dev_push;
mod path_validation;
+mod preview_fly;
mod pulumi_preview_pr;
mod reusable_preview_service;
mod runners;
@@ -202,6 +203,16 @@ const WORKFLOWS: &[WorkflowFile] = &[
)
},
},
+ WorkflowFile {
+ slug: "preview_fly",
+ file_name: "preview-fly.yml",
+ render_yaml: || render_gh_workflow(preview_fly::preview_fly)(),
+ },
+ WorkflowFile {
+ slug: "preview_fly",
+ file_name: "preview-fly-cleanup.yml",
+ render_yaml: || render_gh_workflow(preview_fly::preview_fly_cleanup)(),
+ },
WorkflowFile {
slug: "check_node_modules_nix",
file_name: "check_node_modules_nix.yml",
diff --git a/tooling/xtask/crates/xtask_workflows/src/workflows/preview_fly.rs b/tooling/xtask/crates/xtask_workflows/src/workflows/preview_fly.rs
new file mode 100644
index 0000000000..086f8be3cb
--- /dev/null
+++ b/tooling/xtask/crates/xtask_workflows/src/workflows/preview_fly.rs
@@ -0,0 +1,635 @@
+//! `Fly Preview` — full-stack preview deploys on Fly Machines, plus the
+//! cleanup workflow that destroys the app when the PR closes.
+//!
+//! Opt-in via the `preview` label. The deploy job builds every artifact on the
+//! runner (service binaries, same-origin frontend bundle, init snapshot),
+//! mirrors every stack image into the per-PR app's Fly registry repo, stages
+//! the rest into a slim VM image (`infra/preview/`), and deploys it as the
+//! per-PR Fly app `macro-pr-<N>` — which suspends when idle and wakes on
+//! request. The machine pulls the stack images at boot (layer-level dedup
+//! against its persistent /var/lib/docker volume) instead of shipping them
+//! inside the VM image. Runtime secrets reach the machine as a config-scoped
+//! Doppler service token (`DOPPLER_PREVIEW_TOKEN` → Fly secret
+//! `DOPPLER_TOKEN`), which the stack's env layer pulls at boot.
+//!
+//! NOTE: never a required status check — previews are opt-in by design.
+
+use gh_workflow::{
+ Concurrency, Event, Expression, Job, Level, Permissions, PullRequest, PullRequestType, Run,
+ Step, Use, Workflow,
+};
+
+use crate::workflows::{runners, steps, vars};
+
+/// The per-PR Fly app name; also the preview hostname (`<app>.fly.dev`).
+const APP_NAME: &str = "macro-pr-${{ github.event.pull_request.number }}";
+
+/// Build the deploy workflow.
+pub fn preview_fly() -> Workflow {
+ Workflow::new("Fly Preview")
+ .on(Event::default().pull_request(
+ PullRequest::default()
+ .add_type(PullRequestType::Opened)
+ .add_type(PullRequestType::Reopened)
+ .add_type(PullRequestType::Synchronize)
+ .add_type(PullRequestType::Labeled),
+ ))
+ .concurrency(
+ Concurrency::new(Expression::new(
+ "fly-preview-${{ github.event.pull_request.number }}",
+ ))
+ .cancel_in_progress(true),
+ )
+ .add_job("deploy", deploy())
+}
+
+/// Build the cleanup workflow (destroys the app on close / label removal).
+pub fn preview_fly_cleanup() -> Workflow {
+ Workflow::new("Fly Preview Cleanup")
+ .on(Event::default().pull_request(
+ PullRequest::default()
+ .add_type(PullRequestType::Closed)
+ .add_type(PullRequestType::Unlabeled),
+ ))
+ .add_job("destroy", destroy())
+}
+
+fn deploy() -> Job {
+ Job::default()
+ // Same-repo branches only: the preview image embeds CI-built artifacts
+ // and the app serves under our Fly org — never build it from fork code.
+ .cond(Expression::new(
+ "github.event.pull_request.head.repo.full_name == github.repository && \
+ contains(github.event.pull_request.labels.*.name, 'preview')",
+ ))
+ // Dedicated cache volume pool (nix store + cargo target + snapshots):
+ // see PREVIEW_CACHE_TAG for why sharing the check/test jobs' pool was
+ // measured cold on both layers (different --target = disjoint sccache
+ // keys, and their volumes never carry a cargo target dir).
+ .runs_on(runners::Runner::RustCi.with_cache_tag(vars::PREVIEW_CACHE_TAG))
+ .permissions(
+ Permissions::default()
+ .contents(Level::Read)
+ .pull_requests(Level::Write),
+ )
+ .add_env(("FLY_API_TOKEN", vars::FLY_API_TOKEN))
+ .add_env(("APP_NAME", APP_NAME))
+ // Init snapshots live on the cache volume: an unchanged key skips the
+ // whole infra bake and keeps snapshot bytes (→ VM image layer) stable.
+ .add_env((
+ "MACRO_STACK_SNAPSHOT_DIR",
+ vars::PREVIEW_SNAPSHOT_VOLUME_DIR,
+ ))
+ // No incremental in CI: incremental caches are what bloated the cache
+ // volume to 41G (making it the workspace's eviction victim — observed
+ // volume loss between runs), and sccache refuses to cache incremental
+ // compiles, so they also locked workspace crates out of sccache.
+ // Cargo's fingerprint reuse (what makes warm runs fast) is unaffected.
+ // Local run_local keeps incremental for the edit-rebuild loop.
+ .add_env(("CARGO_INCREMENTAL", "0"))
+ .add_step(steps::checkout(false, true))
+ .add_step(steps::mount_cache_volume_with_cargo_target())
+ // CARGO_INCREMENTAL=0 stops incremental caches being written, but the
+ // ~25G already on older volumes never gets cleaned by cargo — it's
+ // exactly the weight that made this pool the workspace quota's
+ // eviction victim. Deleting is idempotent and instant when clean.
+ .add_step(
+ Step::new("Prune stale incremental caches")
+ .run("rm -rf target/*/debug/incremental target/debug/incremental"),
+ )
+ // Namespace remote builder: persistent BuildKit layer cache across
+ // runs, same as the deploy workflows use. The aux-image builds and the
+ // preview-image build both go through it.
+ .add_step(
+ Step::new("Set up Namespace Docker builder").uses(
+ "namespacelabs",
+ "nscloud-setup-buildx-action",
+ "d059ed7184f0bc7c8b27e8810cea153d02bcc6dd",
+ ), // v0.0.23
+ )
+ // Fail fast: the Docker-built images are the most fragile part of a
+ // fresh bring-up (stale local images mask their rot) and need nothing
+ // from nix/cargo — build them before the expensive toolchain setup so
+ // a broken Dockerfile fails in ~2 minutes, not ~12. This is also the
+ // ONLY thing that builds them now: the infra-only bake never starts
+ // the app layer, so the registry mirror ships exactly what's built here.
+ .add_step(Step::new("Build compose service images (fail fast)").run(
+ "docker compose --project-directory . -p macro -f docker/docker-compose.yml build \
+ search sync_service websocket_service lexical_service ai_editing_worker",
+ ))
+ .add_step(steps::setup_nix())
+ .add_step(steps::setup_reqs_web("Setup dev shell + web deps", false))
+ .add_step(steps::configure_namespace_sccache(
+ vars::PREVIEW_SCCACHE_NAME,
+ ))
+ .add_step(Step::new("Build service binaries").run(indoc::indoc! {r#"
+ cargo x zigbuild
+ sccache --show-stats
+ "#}))
+ .add_step(
+ // xtask_local, not the dependency-free `xtask` launcher (which
+ // just re-invokes cargo — useless in the VM). Deliberately WITHOUT
+ // the `local-stack` feature: the VM restores the baked snapshot,
+ // whose Kafka volume already carries the topics, so it never
+ // provisions Kafka — and skipping rdkafka means no
+ // dynamically-linked librdkafka to ship into the VM. zigbuild,
+ // not a host build: the nix dev shell links the /nix/store ELF
+ // interpreter, which doesn't exist in the VM image (exit 127,
+ // "required file not found").
+ Step::new("Build xtask_local (runs inside the preview VM)").run(
+ "cargo zigbuild --release --target x86_64-unknown-linux-gnu.2.36 -p xtask_local",
+ ),
+ )
+ .add_step(
+ Step::new("Build frontend bundle (same-origin)")
+ .run("bun run --bun build")
+ .working_directory(xtask_paths::repo_dir!("apps/web"))
+ .add_env(("MODE", "development"))
+ .add_env(("NODE_ENV", "production"))
+ .add_env(("VITE_LOCAL_SERVERS", "ALL"))
+ .add_env(("VITE_LOCAL_BACKEND_ORIGIN", "same-origin")),
+ )
+ .add_step(bake_snapshot())
+ .add_step(dump_stack_diagnostics())
+ .add_step(stage_context())
+ .add_step(setup_flyctl())
+ .add_step(deploy_to_fly())
+ .add_step(dump_fly_diagnostics())
+ .add_step(dump_boot_timings())
+ .add_step(comment_preview_url())
+}
+
+/// When the machine never passes its health check, the answer is inside the
+/// VM (dockerd, image load, `stack up`) — dump its logs and state so a failed
+/// deploy is diagnosable from CI output alone.
+fn dump_fly_diagnostics() -> Step<Run> {
+ Step::new("Dump Fly diagnostics")
+ .run(indoc::indoc! {r#"
+ flyctl machine list --app "$APP_NAME" || true
+ flyctl logs --app "$APP_NAME" --no-tail || true
+ "#})
+ .if_condition(Expression::new("failure()"))
+}
+
+/// The entrypoint logs how long each boot phase took (image pulls, stack up),
+/// but Fly's log retention is short and nobody looks at a healthy machine —
+/// surface the `[preview]` lines in CI so every deploy records its boot
+/// breakdown next to the deploy timings.
+fn dump_boot_timings() -> Step<Run> {
+ Step::new("Dump boot timings")
+ .run(r#"flyctl logs --app "$APP_NAME" --no-tail | grep -E '\[preview\]|✓' || true"#)
+}
+
+/// A cold `stack up --infra-only` on the runner runs the real init (migrate,
+/// kickstart, Kafka topics, indices) and saves the content-addressed snapshot
+/// the VM restores from. Infra only: the app services need the Doppler-sourced
+/// env to boot, which this runner deliberately lacks — the snapshot captures
+/// only the infra volumes anyway. Via `just` (not the bare `cargo x` alias)
+/// because the justfile enables the `local-stack` feature the Kafka
+/// provisioning needs.
+///
+/// The store lives on the cache volume (MACRO_STACK_SNAPSHOT_DIR) for a
+/// zero-copy hit. Every content-addressed snapshot is also stored as one tar in
+/// Namespace artifact storage, keyed by the same hash. Cache-volume onboarding
+/// misses therefore download the exact snapshot instead of paying for a full
+/// init again. Artifact access is an optimization: download/upload failures
+/// fall back to the existing volume/cold-bake behavior.
+///
+/// The final `snapshot --json` line lands in $RUNNER_TEMP for the stage step,
+/// which ships exactly that key's directory.
+fn bake_snapshot() -> Step<Run> {
+ Step::new("Bake init snapshot").run(indoc::indoc! {r#"
+ set -euo pipefail
+ mkdir -p "$MACRO_STACK_SNAPSHOT_DIR"
+ status=$(just stack snapshot --json | tail -n 1)
+ key=$(echo "$status" | jq -r '.key')
+ root=$(echo "$status" | jq -r '.root')
+ artifact="macro-preview/init-snapshots/${key}.tar"
+ archive="$RUNNER_TEMP/init-snapshot-${key}.tar"
+ artifact_hit=
+
+ # The Namespace cache-volume pool has an onboarding period where a
+ # runner can receive an empty fork. Artifact storage is the durable
+ # fallback: restore into a temporary directory and only publish it to
+ # the live store after validating the embedded manifest through xtask.
+ if ! echo "$status" | jq -e '.present' >/dev/null; then
+ restored="$RUNNER_TEMP/restored-init-snapshot"
+ rm -rf "$restored"
+ mkdir -p "$restored"
+ if nsc artifact download "$artifact" "$archive" \
+ && tar -xf "$archive" -C "$restored" \
+ && [ -f "$restored/$key/manifest.json" ]; then
+ rm -rf "$root/$key"
+ cp -a "$restored/$key" "$root/$key"
+ status=$(just stack snapshot --json | tail -n 1)
+ if echo "$status" | jq -e '.present' >/dev/null; then
+ artifact_hit=1
+ echo "init snapshot artifact hit: $status"
+ else
+ rm -rf "$root/$key"
+ echo "downloaded init snapshot failed validation; baking cold" >&2
+ fi
+ else
+ echo "init snapshot artifact miss; baking cold" >&2
+ fi
+ fi
+
+ if echo "$status" | jq -e '.present' >/dev/null; then
+ echo "init snapshot cache hit: $status"
+ cargo run --quiet --manifest-path Cargo.toml \
+ -p xtask_local --features local-stack -- gen-compose
+ # The cold path builds the runtime image inside `stack up`; on a
+ # hit nothing else does, and the registry mirror needs it in the
+ # local daemon (its pull fallback only covers public images —
+ # macro-local-runtime:dev is not on Docker Hub).
+ cargo run --quiet --manifest-path Cargo.toml \
+ -p xtask_local --features local-stack -- runtime-image
+ else
+ just stack up --infra-only --no-doppler --no-build
+ status=$(just stack snapshot --json | tail -n 1)
+ echo "$status" | jq -e '.present' >/dev/null
+ fi
+
+ # Seed artifact storage from either a volume hit or a cold bake. Avoid
+ # uploading another version when the artifact already exists; a race
+ # between concurrent PRs is harmless and must not fail the preview.
+ if [ -z "$artifact_hit" ] && ! nsc artifact describe "$artifact" >/dev/null 2>&1; then
+ snap_dir=$(echo "$status" | jq -r '.dir')
+ tar -cf "$archive" -C "$root" "$(basename "$snap_dir")"
+ nsc artifact upload "$archive" "$artifact" --expires_in 720h \
+ || echo "warning: failed to upload init snapshot artifact" >&2
+ fi
+ echo "$status" > "$RUNNER_TEMP/preview-snapshot.json"
+ "#})
+}
+
+/// When the bake dies (typically the backend health gate), the answer is in
+/// the service logs — dump container states and recent logs so a failed run is
+/// diagnosable from CI output alone.
+fn dump_stack_diagnostics() -> Step<Run> {
+ Step::new("Dump stack diagnostics")
+ .run(indoc::indoc! {r#"
+ docker compose -p macro ps --all || true
+ for svc in authentication-service proxy fusionauth postgres kafka \
+ connection_gateway document_storage_service email_service \
+ notification_service contacts_service; do
+ echo "==================== $svc ===================="
+ docker compose -p macro logs --no-color --tail 80 "$svc" 2>&1 || true
+ done
+ "#})
+ .if_condition(Expression::new("failure()"))
+}
+
+fn stage_context() -> Step<Run> {
+ Step::new("Stage preview build context").run(indoc::indoc! {r#"
+ set -euo pipefail
+ # The dev-shell env (BASH_ENV) exports a Nix LD_LIBRARY_PATH; host
+ # binaries like rsync then resolve Nix-store libs whose deps drag in
+ # Nix glibc mid-process and crash on glibc symbol versions. This step
+ # is pure file shuffling — drop it.
+ unset LD_LIBRARY_PATH
+ ctx=preview-ctx
+ rm -rf "$ctx"
+ mkdir -p "$ctx/repo" "$ctx/artifacts/binaries" "$ctx/bin"
+ # The minimal repo layout xtask reads at runtime — including every
+ # snapshot-key input, byte-identical to this checkout, so the VM's
+ # key matches the snapshot baked above.
+ rsync -a --relative --exclude node_modules \
+ docker/docker-compose.yml \
+ docker/docker-compose-databases.yml \
+ infra/stacks/fusionauth-instance/docker-compose.yml \
+ infra/local/nginx \
+ infra/local/opensearch \
+ infra/stacks/opensearch/helpers \
+ crates/macro_db_client/migrations \
+ "$ctx/repo/"
+ # Service binaries only — the target dir also holds gigabytes of
+ # build intermediates. `-p` everywhere: preserved mtimes mean files
+ # cargo didn't relink produce byte-identical Docker layers, so the
+ # registry push/pull skips them (the Dockerfile orders its COPYs
+ # stable → volatile for the same reason).
+ find target/x86_64-unknown-linux-gnu/debug \
+ -maxdepth 1 -type f -executable ! -name '*.d' ! -name '*.so' \
+ -exec cp -p {} "$ctx/artifacts/binaries/" \;
+ cp -a apps/web/dist "$ctx/artifacts/frontend-dist"
+ # Only the current key's snapshot — the volume store accumulates keys.
+ snap_dir=$(jq -r '.dir' "$RUNNER_TEMP/preview-snapshot.json")
+ mkdir -p "$ctx/artifacts/snapshots"
+ cp -a "$snap_dir" "$ctx/artifacts/snapshots/"
+ snapshot_key=$(jq -r '.key' "$RUNNER_TEMP/preview-snapshot.json")
+ runtime_key=$(cd "$ctx/repo" \
+ && find . -type f -print0 \
+ | sort -z \
+ | xargs -0 sha256sum \
+ | sha256sum \
+ | cut -d ' ' -f 1)
+ frontend_key=$(cd "$ctx/artifacts/frontend-dist" \
+ && find . -type f -print0 \
+ | sort -z \
+ | xargs -0 sha256sum \
+ | sha256sum \
+ | cut -d ' ' -f 1)
+ jq -n \
+ --arg snapshot_key "$snapshot_key" \
+ --arg runtime_key "$runtime_key" \
+ --arg frontend_key "$frontend_key" \
+ --arg commit "${{ github.sha }}" \
+ '{format: 1, snapshot_key: $snapshot_key, runtime_key: $runtime_key, frontend_key: $frontend_key, commit: $commit}' \
+ > "$ctx/deployment.json"
+ cp -p target/x86_64-unknown-linux-gnu/release/xtask_local "$ctx/bin/xtask"
+ cp -p infra/preview/hot-update.sh "$ctx/bin/hot-update"
+ cp -p infra/preview/entrypoint.sh "$ctx/entrypoint.sh"
+ cp -p infra/preview/Dockerfile "$ctx/Dockerfile"
+ cp -p infra/preview/update.Dockerfile "$ctx/update.Dockerfile"
+ "#})
+}
+
+fn setup_flyctl() -> Step<Use> {
+ Step::new("Setup flyctl").uses(
+ "superfly",
+ "flyctl-actions/setup-flyctl",
+ "ed8efb33836e8b2096c7fd3ba1c8afe303ebbff1",
+ ) // 1.6
+}
+
+/// Select bootstrap / hot update / rehydrate from the running machine's durable
+/// compatibility marker. All modes mirror content-addressed stack images. A
+/// compatible machine pulls a scratch artifact carrier and applies it in place;
+/// an absent, incompatible, or failed hot update falls through to the immutable
+/// full-image deploy and snapshot restore.
+fn deploy_to_fly() -> Step<Run> {
+ Step::new("Deploy to Fly")
+ .run(indoc::indoc! {r#"
+ set -euo pipefail
+ if [ -z "$DOPPLER_PREVIEW_TOKEN" ]; then
+ echo "DOPPLER_PREVIEW_TOKEN secret is not set (a Doppler service token scoped to the preview config)" >&2
+ exit 1
+ fi
+ if [ -z "$FLY_ORG" ]; then
+ echo "FLY_ORG is not set (repo variable or secret with the Fly org slug)" >&2
+ exit 1
+ fi
+
+ app_existed=
+ if flyctl status --app "$APP_NAME" >/dev/null 2>&1; then
+ app_existed=1
+ else
+ # A transient status error must not turn an existing app into a
+ # hard create failure: create, then prove the app is reachable.
+ flyctl apps create "$APP_NAME" --org "$FLY_ORG" \
+ || flyctl status --app "$APP_NAME" >/dev/null
+ fi
+
+ # A read-only, app-scoped, time-boxed pull token so the machine's
+ # inner dockerd can pull the mirrored stack images from this app's
+ # registry repo — and nothing else (PR code can read machine
+ # secrets, so scope matters). The org deploy token can't mint new
+ # tokens (createLimitedAccessToken is not authorized), but macaroon
+ # attenuation is pure client-side crypto: append an Apps caveat
+ # (numeric app id, mask "r") and a validity window to our own token.
+ app_id=$(curl -sf https://api.fly.io/graphql \
+ -H "Authorization: Bearer $FLY_API_TOKEN" \
+ -H "content-type: application/json" \
+ -d "{\"query\":\"{ app(name: \\\"$APP_NAME\\\") { internalNumericId } }\"}" \
+ | jq -re '.data.app.internalNumericId')
+ now=$(date +%s)
+ pull_token=$(printf '[{"type":"Apps","body":{"apps":{"%s":"r"}}},{"type":"ValidityWindow","body":{"not_before":%d,"not_after":%d}}]' \
+ "$app_id" $((now - 60)) $((now + 604800)) | flyctl tokens attenuate)
+
+ snapshot_key=$(jq -r '.key' "$RUNNER_TEMP/preview-snapshot.json")
+ runtime_key=$(jq -r '.runtime_key' preview-ctx/deployment.json)
+ mode=bootstrap
+ machine_id=
+ if [ -n "$app_existed" ]; then
+ mode=rehydrate
+ machine_id=$(flyctl machine list --app "$APP_NAME" --json \
+ | jq -r '[.[] | select((.config.mounts // []) | length > 0)][0].id // empty')
+ if [ -n "$machine_id" ]; then
+ # A suspended preview needs demand before SSH is reachable.
+ curl -s -o /dev/null --max-time 10 "https://$APP_NAME.fly.dev/" || true
+ for _ in $(seq 1 12); do
+ marker=$(flyctl ssh console --app "$APP_NAME" --machine "$machine_id" \
+ --quiet --command 'if [ -f /var/lib/docker/.macro-preview/deployment.json ]; then cat /var/lib/docker/.macro-preview/deployment.json; else echo __NO_MARKER__; fi' \
+ 2>/dev/null || true)
+ if echo "$marker" | jq -e \
+ --arg key "$snapshot_key" \
+ --arg runtime "$runtime_key" \
+ '.format == 1 and .snapshot_key == $key and .runtime_key == $runtime and (.frontend_key | type == "string")' \
+ >/dev/null 2>&1; then
+ mode=hot
+ break
+ fi
+ # SSH succeeded and returned a marker (or proved it absent):
+ # incompatibility is real, not a wake-up race.
+ if [ "$marker" = __NO_MARKER__ ] \
+ || echo "$marker" | jq -e '.format != null' >/dev/null 2>&1; then
+ break
+ fi
+ sleep 5
+ done
+ fi
+ fi
+ echo "preview deployment mode: $mode (snapshot $snapshot_key)"
+
+ flyctl auth docker
+ registry="registry.fly.io/$APP_NAME"
+ # Mirror every stack image into the app's registry repo instead of
+ # baking docker-save tars into the VM image (which made every
+ # deploy re-ship ~5GB the machine's volume already had). Pushes
+ # dedup at the layer level against the repo, so a redeploy only
+ # uploads layers that actually changed; the entrypoint pulls with
+ # the same dedup against the machine's persistent layer store. The
+ # manifest maps image ID -> local tag -> registry ref so boots can
+ # skip images already present. alpine:3 rides along for the
+ # snapshot-restore helper container.
+ # On a snapshot cache hit only gen-compose ran, which doesn't write
+ # the env file; compose hard-fails on a missing --env-file, and the
+ # image names don't interpolate env vars anyway.
+ envfile=infra/local/generated/macro/local.generated.env
+ [ -f "$envfile" ] || : > "$envfile"
+ images=$(docker compose --project-directory . -p macro \
+ -f docker/docker-compose.yml \
+ -f infra/local/generated/macro/docker-compose.override.yml \
+ --env-file "$envfile" \
+ config --images | sort -u)
+ echo "mirroring images:" $images
+ docker pull alpine:3
+ mkdir -p preview-ctx/preload
+ : > preview-ctx/preload/manifest.txt
+ : > "$RUNNER_TEMP/push-refs.txt"
+ for img in $images alpine:3; do
+ # Images the bake didn't leave in the daemon (snapshot cache hit,
+ # or app-layer-only images like proxy/mailpit) get pulled here.
+ docker image inspect "$img" >/dev/null 2>&1 || docker pull "$img"
+ id=$(docker image inspect -f '{{.Id}}' "$img")
+ # Content-addressed tag: same image content = same ref, so a
+ # redeploy's push is skippable by a manifest existence check and
+ # concurrent PRs' deploys can never clobber each other's tags.
+ ref="$registry:img-$(echo "$img" | tr '/:' '__')-$(echo "$id" | cut -c8-19)"
+ docker tag "$img" "$ref"
+ echo "$id $img $ref" >> preview-ctx/preload/manifest.txt
+ if ! docker manifest inspect "$ref" >/dev/null 2>&1; then
+ echo "$ref" >> "$RUNNER_TEMP/push-refs.txt"
+ fi
+ done
+ echo "pushing $(wc -l < "$RUNNER_TEMP/push-refs.txt") of $(wc -l < preview-ctx/preload/manifest.txt) images"
+ # The Fly registry occasionally aborts large uploads ("s3aws:
+ # append to zero-size path unsupported") — retry; layers that made
+ # it are reused. Four pushes at a time keeps the first (cold) push
+ # moving without saturating the registry.
+ if [ -s "$RUNNER_TEMP/push-refs.txt" ]; then
+ xargs -P 4 -I {} sh -c '
+ for _ in 1 2 3; do
+ docker push "{}" && exit 0
+ echo "push of {} failed, retrying" >&2
+ sleep 15
+ done
+ exit 1' < "$RUNNER_TEMP/push-refs.txt"
+ fi
+
+ push_image() {
+ image_to_push=$1
+ for _ in 1 2 3; do
+ if docker push "$image_to_push"; then return 0; fi
+ echo "docker push failed for $image_to_push, retrying" >&2
+ sleep 15
+ done
+ return 1
+ }
+
+ if [ "$mode" = hot ]; then
+ update_image="$registry:update-${{ github.sha }}"
+ docker build -f preview-ctx/update.Dockerfile -t "$update_image" preview-ctx
+ push_image "$update_image"
+ token_file="$RUNNER_TEMP/registry-pull-token"
+ printf '%s' "$pull_token" > "$token_file"
+ chmod 600 "$token_file"
+ hot_rc=0
+ flyctl ssh sftp put "$token_file" /tmp/macro-registry-token \
+ --app "$APP_NAME" --machine "$machine_id" --mode 0600 \
+ || hot_rc=$?
+ if [ "$hot_rc" = 0 ]; then
+ hot_command="/srv/macro/bin/hot-update '$update_image' /tmp/macro-registry-token"
+ flyctl ssh console --app "$APP_NAME" --machine "$machine_id" \
+ --command "$hot_command" || hot_rc=$?
+ fi
+ if [ "$hot_rc" = 0 ]; then
+ echo "hot update completed without restarting the Fly machine"
+ exit 0
+ fi
+ echo "hot update failed (exit $hot_rc); falling back to full rehydrate" >&2
+ mode=rehydrate
+ fi
+
+ # Bootstrap/rehydrate refreshes app secrets and reconciles the
+ # persistent Docker volume before replacing the machine image.
+ flyctl secrets set --app "$APP_NAME" --stage \
+ "DOPPLER_TOKEN=$DOPPLER_PREVIEW_TOKEN" \
+ "REGISTRY_PULL_TOKEN=$pull_token"
+ if ! flyctl volumes list --app "$APP_NAME" --json \
+ | jq -e 'map(select(.name == "docker_data")) | length > 0' >/dev/null; then
+ flyctl volumes create docker_data --app "$APP_NAME" --region ewr --size 40 --yes
+ fi
+ # Machines created before the volume existed can't take the mounts
+ # config update. A destroyed/partial machine is recreated by deploy.
+ flyctl machine list --app "$APP_NAME" --json \
+ | jq -r '.[] | select((.config.mounts // []) | length == 0) | .id' \
+ | while read -r id; do
+ [ -z "$id" ] || flyctl machine destroy "$id" --app "$APP_NAME" --force || true
+ done
+
+ image="$registry:${{ github.sha }}"
+ docker build -t "$image" preview-ctx
+ push_image "$image"
+ # Keep the machine in demand while it boots: with zero inbound
+ # traffic, fly-proxy counts a health-failing machine as excess
+ # capacity and suspends it mid-boot (observed mid-docker-load).
+ # Any response status counts as traffic, and a request also
+ # resumes an already-suspended machine.
+ (while :; do
+ curl -s -o /dev/null --max-time 10 "https://$APP_NAME.fly.dev/" || true
+ sleep 15
+ done) &
+ keepalive=$!
+ # First boot does real work before 8090 opens (pulling the stack
+ # images, snapshot restore, compose up, FusionAuth's JVM) —
+ # give the health check more runway than flyctl's default wait.
+ rc=0
+ flyctl deploy --app "$APP_NAME" \
+ --config infra/preview/fly.toml \
+ --image "$image" \
+ --wait-timeout 1800 \
+ --yes || rc=$?
+ kill "$keepalive" 2>/dev/null || true
+ exit "$rc"
+ "#})
+ // Accept the org slug from either a repo variable or a repo secret —
+ // it's not sensitive, but people reasonably reach for secrets first.
+ .add_env(("FLY_ORG", "${{ vars.FLY_ORG || secrets.FLY_ORG }}"))
+ .add_env(("DOPPLER_PREVIEW_TOKEN", vars::DOPPLER_PREVIEW_TOKEN))
+}
+
+fn comment_preview_url() -> Step<Use> {
+ Step::new("Comment preview URL")
+ .uses(
+ "actions",
+ "github-script",
+ "f28e40c7f34bde8b3046d885e986cb6290c5673b",
+ )
+ .add_with(("github-token", "${{ secrets.GITHUB_TOKEN }}"))
+ .add_with((
+ "script",
+ indoc::indoc! {r#"
+ const app = process.env.APP_NAME;
+ const url = `https://${app}.fly.dev`;
+ const marker = '<!-- fly-preview -->';
+ const body = [
+ marker,
+ `🚀 **Full-stack preview**: ${url}`,
+ '',
+ `- Suspended when idle — the first request wakes it (a few seconds).`,
+ `- Log in with any email; the passwordless code lands in [Mailpit](${url}/mailpit/).`,
+ `- Deployed from ${context.payload.pull_request.head.sha.slice(0, 7)}.`,
+ ].join('\n');
+ const { data: comments } = await github.rest.issues.listComments({
+ owner: context.repo.owner,
+ repo: context.repo.repo,
+ issue_number: context.issue.number,
+ per_page: 100,
+ });
+ const existing = comments.find(c => c.body && c.body.startsWith(marker));
+ if (existing) {
+ await github.rest.issues.updateComment({
+ owner: context.repo.owner,
+ repo: context.repo.repo,
+ comment_id: existing.id,
+ body,
+ });
+ } else {
+ await github.rest.issues.createComment({
+ owner: context.repo.owner,
+ repo: context.repo.repo,
+ issue_number: context.issue.number,
+ body,
+ });
+ }
+ "#},
+ ))
+}
+
+fn destroy() -> Job {
+ Job::default()
+ // Closed PRs always attempt cleanup (destroy is a no-op if no app);
+ // unlabeled only cleans up when the removed label is `preview`.
+ .cond(Expression::new(
+ "github.event.action == 'closed' || github.event.label.name == 'preview'",
+ ))
+ .runs_on(runners::Runner::TinyNoCache.to_string())
+ .permissions(Permissions::default().contents(Level::Read))
+ .add_env(("FLY_API_TOKEN", vars::FLY_API_TOKEN))
+ .add_step(setup_flyctl())
+ .add_step(
+ Step::new("Destroy app")
+ .run(format!("flyctl apps destroy \"{APP_NAME}\" --yes || true")),
+ )
+}
diff --git a/tooling/xtask/crates/xtask_workflows/src/workflows/steps.rs b/tooling/xtask/crates/xtask_workflows/src/workflows/steps.rs
index 1f53a2319a..02f8a1f61e 100644
--- a/tooling/xtask/crates/xtask_workflows/src/workflows/steps.rs
+++ b/tooling/xtask/crates/xtask_workflows/src/workflows/steps.rs
@@ -104,6 +104,51 @@ pub fn mount_cache_volume() -> Step<Use> {
.continue_on_error(true)
}
+/// [`mount_cache_volume`] plus the checkout's cargo target dir and the init
+/// snapshot store. Persisting `target/` is what makes the preview job's
+/// zigbuild incremental — cargo's own fingerprints carry across runs, where
+/// remote sccache alone leaves build scripts, native (cmake/zig) compiles, and
+/// linking cold every time. Persisting the snapshot store gives the bake step
+/// a zero-copy fast path; Namespace artifact storage is its durable fallback.
+/// The volume is a block-device mount, so multi-GB trees cost nothing to save
+/// or restore when it hits.
+pub fn mount_cache_volume_with_cargo_target() -> Step<Use> {
+ Step::new("Mount Namespace cache volume")
+ .uses(
+ "namespacelabs",
+ "nscloud-cache-action",
+ "15799a6b54e5765f85b2aac25b3f0df43ed571c0", // v1.4.3
+ )
+ .add_with(("cache", "rust"))
+ .add_with((
+ "path",
+ format!(
+ "/nix\n${{{{ github.workspace }}}}/target\n{}",
+ vars::PREVIEW_SNAPSHOT_VOLUME_DIR,
+ ),
+ ))
+ .continue_on_error(true)
+}
+
+/// Configure Namespace's artifact-backed remote sccache. `setup-reqs-web`
+/// already installs sccache and exports `RUSTC_WRAPPER=sccache`; this replaces
+/// the local-disk backend with short-lived WebDAV credentials that work across
+/// runners and cache-volume misses.
+pub fn configure_namespace_sccache(cache_name: &str) -> Step<Run> {
+ Step::new("Configure Namespace remote sccache").run(format!(
+ r#"set -euo pipefail
+env_file="$RUNNER_TEMP/namespace-sccache.env"
+if nsc cache sccache setup --cache_name {cache_name} > "$env_file"; then
+ cat "$env_file" >> "$GITHUB_ENV"
+ # Force the next compiler invocation to start a server with the new remote
+ # backend even if a setup hook happened to launch one already.
+ sccache --stop-server >/dev/null 2>&1 || true
+else
+ echo "::warning::Namespace remote sccache setup failed; using local cache fallback"
+fi"#
+ ))
+}
+
/// Repoint sccache at the persisted volume. Runs AFTER `setup-cachix`; this keeps
/// the compiled artifact cache directory aligned with the Namespace mounted path.
pub fn pin_sccache_dir() -> Step<Run> {
diff --git a/tooling/xtask/crates/xtask_workflows/src/workflows/vars.rs b/tooling/xtask/crates/xtask_workflows/src/workflows/vars.rs
index b7ab1c8aaf..bbf55745c9 100644
--- a/tooling/xtask/crates/xtask_workflows/src/workflows/vars.rs
+++ b/tooling/xtask/crates/xtask_workflows/src/workflows/vars.rs
@@ -18,7 +18,9 @@ secret!(CLOUDFLARE_API_TOKEN);
secret!(DD_API_KEY);
secret!(DD_APP_KEY);
secret!(DD_WEB_APP_TOKEN);
+secret!(DOPPLER_PREVIEW_TOKEN);
secret!(DOPPLER_TOKEN);
+secret!(FLY_API_TOKEN);
secret!(MACOS_DEVELOPER_ID_CERTIFICATE_BASE64);
secret!(MACOS_DEVELOPER_ID_CERTIFICATE_PASSWORD);
secret!(POSTHOG_API_KEY);
@@ -40,6 +42,23 @@ pub const NEXTEST_TEST_THREADS: u32 = 32;
/// global across all branches — see [`crate::workflows::runners::Runner::with_cache_tag`].
pub const CI_CACHE_TAG: &str = "sccache-ci";
+/// Namespace cache tag for the Fly preview deploy job. Its own pool, NOT
+/// [`CI_CACHE_TAG`]: sharing looked economical but was measured cold both ways
+/// (run 28968155599: sccache 2.65% hits, cargo target dir absent). The
+/// check/test jobs compile for the host while this job zigbuilds with
+/// `--target x86_64-unknown-linux-gnu.2.36`, so their sccache entries hash
+/// differently and never serve this job — and they don't persist the cargo
+/// target dir at all, so a volume from the shared pool almost never carries
+/// one. A dedicated low-concurrency pool gives the job's target dir and init
+/// snapshots the best chance of a zero-copy hit; Rust objects use the durable
+/// remote sccache below instead of depending on this volume's placement.
+pub const PREVIEW_CACHE_TAG: &str = "fly-preview";
+
+/// Durable Namespace remote sccache shared by Fly preview jobs. Unlike the
+/// local cache volume, this has an artifact-backed cold tier, so a job placed
+/// on a runner that has never seen the volume can still reuse Rust objects.
+pub const PREVIEW_SCCACHE_NAME: &str = "fly-preview";
+
/// Namespace cache tag for the web-app jobs (PR checks + preview deploys).
/// Cache volumes are keyed workspace-wide by tag alone, so a dedicated tag
/// gives the frontend its own volume — isolated both from the Rust CI volume
@@ -51,6 +70,12 @@ pub const WEB_CI_CACHE_TAG: &str = "web-ci";
/// volume so it persists across runs — this is what replaces the S3 bucket.
pub const SCCACHE_VOLUME_DIR: &str = "/home/runner/.cache/sccache";
+/// Init-snapshot store for the preview job (`MACRO_STACK_SNAPSHOT_DIR`). Lives
+/// on the preview cache volume for the zero-copy fast path; the workflow also
+/// backs each content-addressed snapshot up to Namespace artifact storage so a
+/// cache-volume miss does not force another infra bake.
+pub const PREVIEW_SNAPSHOT_VOLUME_DIR: &str = "/home/runner/.cache/macro-preview-snapshots";
+
/// Max on-disk size for the sccache cache. Larger than the setup default since
/// the persisted volume can hold a full-workspace cache.
pub const SCCACHE_CACHE_SIZE: &str = "20G";