Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.
TypeScript agentaiai-agentsall-in-onecrmcrm-systememaillinearmcpmessagingnotesnotionofficerustslackslack-alternativestartupstartup-toolstypescriptworkspace
feat: fullstack preview (#4547)
teo7b254ab62dc2
diff --git a/.github/workflows/preview-fly-cleanup.yml b/.github/workflows/preview-fly-cleanup.ymlnew file mode 100644index 0000000000..ebe911e0bb--- /dev/null+++ b/.github/workflows/preview-fly-cleanup.yml@@ -0,0 +1,21 @@+# DO NOT EDIT — regenerate with `cargo x workflows` (from the repository root).+# Source: tooling/xtask/crates/xtask_workflows/src/workflows/preview_fly.rs+name: Fly Preview Cleanup+'on':+ pull_request:+ types:+ - closed+ - unlabeled+jobs:+ destroy:+ if: github.event.action == 'closed' || github.event.label.name == 'preview'+ runs-on: namespace-profile-linux-tiny-no-cache+ permissions:+ contents: read+ env:+ FLY_API_TOKEN: ${{ secrets.FLY_API_TOKEN }}+ steps:+ - name: Setup flyctl+ uses: superfly/flyctl-actions/setup-flyctl@ed8efb33836e8b2096c7fd3ba1c8afe303ebbff1+ - name: Destroy app+ run: flyctl apps destroy "macro-pr-${{ github.event.pull_request.number }}" --yes || truediff --git a/.github/workflows/preview-fly.yml b/.github/workflows/preview-fly.ymlnew file mode 100644index 0000000000..35b24d53e3--- /dev/null+++ b/.github/workflows/preview-fly.yml@@ -0,0 +1,465 @@+# DO NOT EDIT — regenerate with `cargo x workflows` (from the repository root).+# Source: tooling/xtask/crates/xtask_workflows/src/workflows/preview_fly.rs+name: Fly Preview+'on':+ pull_request:+ types:+ - opened+ - reopened+ - synchronize+ - labeled+jobs:+ deploy:+ if: github.event.pull_request.head.repo.full_name == github.repository && contains(github.event.pull_request.labels.*.name, 'preview')+ runs-on: namespace-profile-linux-rust-ci;overrides.cache-tag=fly-preview+ permissions:+ contents: read+ pull-requests: write+ env:+ FLY_API_TOKEN: ${{ secrets.FLY_API_TOKEN }}+ APP_NAME: macro-pr-${{ github.event.pull_request.number }}+ MACRO_STACK_SNAPSHOT_DIR: /home/runner/.cache/macro-preview-snapshots+ CARGO_INCREMENTAL: '0'+ steps:+ - name: Checkout+ uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd+ with:+ clean: 'false'+ - name: Mount Namespace cache volume+ uses: namespacelabs/nscloud-cache-action@15799a6b54e5765f85b2aac25b3f0df43ed571c0+ with:+ cache: rust+ path: |-+ /nix+ ${{ github.workspace }}/target+ /home/runner/.cache/macro-preview-snapshots+ continue-on-error: true+ - name: Prune stale incremental caches+ run: rm -rf target/*/debug/incremental target/debug/incremental+ - name: Set up Namespace Docker builder+ uses: namespacelabs/nscloud-setup-buildx-action@d059ed7184f0bc7c8b27e8810cea153d02bcc6dd+ - name: Build compose service images (fail fast)+ run: docker compose --project-directory . -p macro -f docker/docker-compose.yml build search sync_service websocket_service lexical_service ai_editing_worker+ - name: Setup Nix+ uses: './.github/actions/setup-nix'+ - name: Setup dev shell + web deps+ uses: './.github/actions/setup-reqs-web'+ with:+ cachix-auth-token: ${{ secrets.CACHIX_AUTH_TOKEN }}+ - name: Configure Namespace remote sccache+ run: |-+ set -euo pipefail+ env_file="$RUNNER_TEMP/namespace-sccache.env"+ if nsc cache sccache setup --cache_name fly-preview > "$env_file"; then+ cat "$env_file" >> "$GITHUB_ENV"+ # Force the next compiler invocation to start a server with the new remote+ # backend even if a setup hook happened to launch one already.+ sccache --stop-server >/dev/null 2>&1 || true+ else+ echo "::warning::Namespace remote sccache setup failed; using local cache fallback"+ fi+ - name: Build service binaries+ run: |+ cargo x zigbuild+ sccache --show-stats+ - name: Build xtask_local (runs inside the preview VM)+ run: cargo zigbuild --release --target x86_64-unknown-linux-gnu.2.36 -p xtask_local+ - name: Build frontend bundle (same-origin)+ run: bun run --bun build+ env:+ MODE: development+ NODE_ENV: production+ VITE_LOCAL_SERVERS: ALL+ VITE_LOCAL_BACKEND_ORIGIN: same-origin+ working-directory: apps/web+ - name: Bake init snapshot+ run: |+ set -euo pipefail+ mkdir -p "$MACRO_STACK_SNAPSHOT_DIR"+ status=$(just stack snapshot --json | tail -n 1)+ key=$(echo "$status" | jq -r '.key')+ root=$(echo "$status" | jq -r '.root')+ artifact="macro-preview/init-snapshots/${key}.tar"+ archive="$RUNNER_TEMP/init-snapshot-${key}.tar"+ artifact_hit=++ # The Namespace cache-volume pool has an onboarding period where a+ # runner can receive an empty fork. Artifact storage is the durable+ # fallback: restore into a temporary directory and only publish it to+ # the live store after validating the embedded manifest through xtask.+ if ! echo "$status" | jq -e '.present' >/dev/null; then+ restored="$RUNNER_TEMP/restored-init-snapshot"+ rm -rf "$restored"+ mkdir -p "$restored"+ if nsc artifact download "$artifact" "$archive" \+ && tar -xf "$archive" -C "$restored" \+ && [ -f "$restored/$key/manifest.json" ]; then+ rm -rf "$root/$key"+ cp -a "$restored/$key" "$root/$key"+ status=$(just stack snapshot --json | tail -n 1)+ if echo "$status" | jq -e '.present' >/dev/null; then+ artifact_hit=1+ echo "init snapshot artifact hit: $status"+ else+ rm -rf "$root/$key"+ echo "downloaded init snapshot failed validation; baking cold" >&2+ fi+ else+ echo "init snapshot artifact miss; baking cold" >&2+ fi+ fi++ if echo "$status" | jq -e '.present' >/dev/null; then+ echo "init snapshot cache hit: $status"+ cargo run --quiet --manifest-path Cargo.toml \+ -p xtask_local --features local-stack -- gen-compose+ # The cold path builds the runtime image inside `stack up`; on a+ # hit nothing else does, and the registry mirror needs it in the+ # local daemon (its pull fallback only covers public images —+ # macro-local-runtime:dev is not on Docker Hub).+ cargo run --quiet --manifest-path Cargo.toml \+ -p xtask_local --features local-stack -- runtime-image+ else+ just stack up --infra-only --no-doppler --no-build+ status=$(just stack snapshot --json | tail -n 1)+ echo "$status" | jq -e '.present' >/dev/null+ fi++ # Seed artifact storage from either a volume hit or a cold bake. Avoid+ # uploading another version when the artifact already exists; a race+ # between concurrent PRs is harmless and must not fail the preview.+ if [ -z "$artifact_hit" ] && ! nsc artifact describe "$artifact" >/dev/null 2>&1; then+ snap_dir=$(echo "$status" | jq -r '.dir')+ tar -cf "$archive" -C "$root" "$(basename "$snap_dir")"+ nsc artifact upload "$archive" "$artifact" --expires_in 720h \+ || echo "warning: failed to upload init snapshot artifact" >&2+ fi+ echo "$status" > "$RUNNER_TEMP/preview-snapshot.json"+ - name: Dump stack diagnostics+ if: failure()+ run: |+ docker compose -p macro ps --all || true+ for svc in authentication-service proxy fusionauth postgres kafka \+ connection_gateway document_storage_service email_service \+ notification_service contacts_service; do+ echo "==================== $svc ===================="+ docker compose -p macro logs --no-color --tail 80 "$svc" 2>&1 || true+ done+ - name: Stage preview build context+ run: |+ set -euo pipefail+ # The dev-shell env (BASH_ENV) exports a Nix LD_LIBRARY_PATH; host+ # binaries like rsync then resolve Nix-store libs whose deps drag in+ # Nix glibc mid-process and crash on glibc symbol versions. This step+ # is pure file shuffling — drop it.+ unset LD_LIBRARY_PATH+ ctx=preview-ctx+ rm -rf "$ctx"+ mkdir -p "$ctx/repo" "$ctx/artifacts/binaries" "$ctx/bin"+ # The minimal repo layout xtask reads at runtime — including every+ # snapshot-key input, byte-identical to this checkout, so the VM's+ # key matches the snapshot baked above.+ rsync -a --relative --exclude node_modules \+ docker/docker-compose.yml \+ docker/docker-compose-databases.yml \+ infra/stacks/fusionauth-instance/docker-compose.yml \+ infra/local/nginx \+ infra/local/opensearch \+ infra/stacks/opensearch/helpers \+ crates/macro_db_client/migrations \+ "$ctx/repo/"+ # Service binaries only — the target dir also holds gigabytes of+ # build intermediates. `-p` everywhere: preserved mtimes mean files+ # cargo didn't relink produce byte-identical Docker layers, so the+ # registry push/pull skips them (the Dockerfile orders its COPYs+ # stable → volatile for the same reason).+ find target/x86_64-unknown-linux-gnu/debug \+ -maxdepth 1 -type f -executable ! -name '*.d' ! -name '*.so' \+ -exec cp -p {} "$ctx/artifacts/binaries/" \;+ cp -a apps/web/dist "$ctx/artifacts/frontend-dist"+ # Only the current key's snapshot — the volume store accumulates keys.+ snap_dir=$(jq -r '.dir' "$RUNNER_TEMP/preview-snapshot.json")+ mkdir -p "$ctx/artifacts/snapshots"+ cp -a "$snap_dir" "$ctx/artifacts/snapshots/"+ snapshot_key=$(jq -r '.key' "$RUNNER_TEMP/preview-snapshot.json")+ runtime_key=$(cd "$ctx/repo" \+ && find . -type f -print0 \+ | sort -z \+ | xargs -0 sha256sum \+ | sha256sum \+ | cut -d ' ' -f 1)+ frontend_key=$(cd "$ctx/artifacts/frontend-dist" \+ && find . -type f -print0 \+ | sort -z \+ | xargs -0 sha256sum \+ | sha256sum \+ | cut -d ' ' -f 1)+ jq -n \+ --arg snapshot_key "$snapshot_key" \+ --arg runtime_key "$runtime_key" \+ --arg frontend_key "$frontend_key" \+ --arg commit "${{ github.sha }}" \+ '{format: 1, snapshot_key: $snapshot_key, runtime_key: $runtime_key, frontend_key: $frontend_key, commit: $commit}' \+ > "$ctx/deployment.json"+ cp -p target/x86_64-unknown-linux-gnu/release/xtask_local "$ctx/bin/xtask"+ cp -p infra/preview/hot-update.sh "$ctx/bin/hot-update"+ cp -p infra/preview/entrypoint.sh "$ctx/entrypoint.sh"+ cp -p infra/preview/Dockerfile "$ctx/Dockerfile"+ cp -p infra/preview/update.Dockerfile "$ctx/update.Dockerfile"+ - name: Setup flyctl+ uses: superfly/flyctl-actions/setup-flyctl@ed8efb33836e8b2096c7fd3ba1c8afe303ebbff1+ - name: Deploy to Fly+ run: |+ set -euo pipefail+ if [ -z "$DOPPLER_PREVIEW_TOKEN" ]; then+ echo "DOPPLER_PREVIEW_TOKEN secret is not set (a Doppler service token scoped to the preview config)" >&2+ exit 1+ fi+ if [ -z "$FLY_ORG" ]; then+ echo "FLY_ORG is not set (repo variable or secret with the Fly org slug)" >&2+ exit 1+ fi++ app_existed=+ if flyctl status --app "$APP_NAME" >/dev/null 2>&1; then+ app_existed=1+ else+ # A transient status error must not turn an existing app into a+ # hard create failure: create, then prove the app is reachable.+ flyctl apps create "$APP_NAME" --org "$FLY_ORG" \+ || flyctl status --app "$APP_NAME" >/dev/null+ fi++ # A read-only, app-scoped, time-boxed pull token so the machine's+ # inner dockerd can pull the mirrored stack images from this app's+ # registry repo — and nothing else (PR code can read machine+ # secrets, so scope matters). The org deploy token can't mint new+ # tokens (createLimitedAccessToken is not authorized), but macaroon+ # attenuation is pure client-side crypto: append an Apps caveat+ # (numeric app id, mask "r") and a validity window to our own token.+ app_id=$(curl -sf https://api.fly.io/graphql \+ -H "Authorization: Bearer $FLY_API_TOKEN" \+ -H "content-type: application/json" \+ -d "{\"query\":\"{ app(name: \\\"$APP_NAME\\\") { internalNumericId } }\"}" \+ | jq -re '.data.app.internalNumericId')+ now=$(date +%s)+ pull_token=$(printf '[{"type":"Apps","body":{"apps":{"%s":"r"}}},{"type":"ValidityWindow","body":{"not_before":%d,"not_after":%d}}]' \+ "$app_id" $((now - 60)) $((now + 604800)) | flyctl tokens attenuate)++ snapshot_key=$(jq -r '.key' "$RUNNER_TEMP/preview-snapshot.json")+ runtime_key=$(jq -r '.runtime_key' preview-ctx/deployment.json)+ mode=bootstrap+ machine_id=+ if [ -n "$app_existed" ]; then+ mode=rehydrate+ machine_id=$(flyctl machine list --app "$APP_NAME" --json \+ | jq -r '[.[] | select((.config.mounts // []) | length > 0)][0].id // empty')+ if [ -n "$machine_id" ]; then+ # A suspended preview needs demand before SSH is reachable.+ curl -s -o /dev/null --max-time 10 "https://$APP_NAME.fly.dev/" || true+ for _ in $(seq 1 12); do+ marker=$(flyctl ssh console --app "$APP_NAME" --machine "$machine_id" \+ --quiet --command 'if [ -f /var/lib/docker/.macro-preview/deployment.json ]; then cat /var/lib/docker/.macro-preview/deployment.json; else echo __NO_MARKER__; fi' \+ 2>/dev/null || true)+ if echo "$marker" | jq -e \+ --arg key "$snapshot_key" \+ --arg runtime "$runtime_key" \+ '.format == 1 and .snapshot_key == $key and .runtime_key == $runtime and (.frontend_key | type == "string")' \+ >/dev/null 2>&1; then+ mode=hot+ break+ fi+ # SSH succeeded and returned a marker (or proved it absent):+ # incompatibility is real, not a wake-up race.+ if [ "$marker" = __NO_MARKER__ ] \+ || echo "$marker" | jq -e '.format != null' >/dev/null 2>&1; then+ break+ fi+ sleep 5+ done+ fi+ fi+ echo "preview deployment mode: $mode (snapshot $snapshot_key)"++ flyctl auth docker+ registry="registry.fly.io/$APP_NAME"+ # Mirror every stack image into the app's registry repo instead of+ # baking docker-save tars into the VM image (which made every+ # deploy re-ship ~5GB the machine's volume already had). Pushes+ # dedup at the layer level against the repo, so a redeploy only+ # uploads layers that actually changed; the entrypoint pulls with+ # the same dedup against the machine's persistent layer store. The+ # manifest maps image ID -> local tag -> registry ref so boots can+ # skip images already present. alpine:3 rides along for the+ # snapshot-restore helper container.+ # On a snapshot cache hit only gen-compose ran, which doesn't write+ # the env file; compose hard-fails on a missing --env-file, and the+ # image names don't interpolate env vars anyway.+ envfile=infra/local/generated/macro/local.generated.env+ [ -f "$envfile" ] || : > "$envfile"+ images=$(docker compose --project-directory . -p macro \+ -f docker/docker-compose.yml \+ -f infra/local/generated/macro/docker-compose.override.yml \+ --env-file "$envfile" \+ config --images | sort -u)+ echo "mirroring images:" $images+ docker pull alpine:3+ mkdir -p preview-ctx/preload+ : > preview-ctx/preload/manifest.txt+ : > "$RUNNER_TEMP/push-refs.txt"+ for img in $images alpine:3; do+ # Images the bake didn't leave in the daemon (snapshot cache hit,+ # or app-layer-only images like proxy/mailpit) get pulled here.+ docker image inspect "$img" >/dev/null 2>&1 || docker pull "$img"+ id=$(docker image inspect -f '{{.Id}}' "$img")+ # Content-addressed tag: same image content = same ref, so a+ # redeploy's push is skippable by a manifest existence check and+ # concurrent PRs' deploys can never clobber each other's tags.+ ref="$registry:img-$(echo "$img" | tr '/:' '__')-$(echo "$id" | cut -c8-19)"+ docker tag "$img" "$ref"+ echo "$id $img $ref" >> preview-ctx/preload/manifest.txt+ if ! docker manifest inspect "$ref" >/dev/null 2>&1; then+ echo "$ref" >> "$RUNNER_TEMP/push-refs.txt"+ fi+ done+ echo "pushing $(wc -l < "$RUNNER_TEMP/push-refs.txt") of $(wc -l < preview-ctx/preload/manifest.txt) images"+ # The Fly registry occasionally aborts large uploads ("s3aws:+ # append to zero-size path unsupported") — retry; layers that made+ # it are reused. Four pushes at a time keeps the first (cold) push+ # moving without saturating the registry.+ if [ -s "$RUNNER_TEMP/push-refs.txt" ]; then+ xargs -P 4 -I {} sh -c '+ for _ in 1 2 3; do+ docker push "{}" && exit 0+ echo "push of {} failed, retrying" >&2+ sleep 15+ done+ exit 1' < "$RUNNER_TEMP/push-refs.txt"+ fi++ push_image() {+ image_to_push=$1+ for _ in 1 2 3; do+ if docker push "$image_to_push"; then return 0; fi+ echo "docker push failed for $image_to_push, retrying" >&2+ sleep 15+ done+ return 1+ }++ if [ "$mode" = hot ]; then+ update_image="$registry:update-${{ github.sha }}"+ docker build -f preview-ctx/update.Dockerfile -t "$update_image" preview-ctx+ push_image "$update_image"+ token_file="$RUNNER_TEMP/registry-pull-token"+ printf '%s' "$pull_token" > "$token_file"+ chmod 600 "$token_file"+ hot_rc=0+ flyctl ssh sftp put "$token_file" /tmp/macro-registry-token \+ --app "$APP_NAME" --machine "$machine_id" --mode 0600 \+ || hot_rc=$?+ if [ "$hot_rc" = 0 ]; then+ hot_command="/srv/macro/bin/hot-update '$update_image' /tmp/macro-registry-token"+ flyctl ssh console --app "$APP_NAME" --machine "$machine_id" \+ --command "$hot_command" || hot_rc=$?+ fi+ if [ "$hot_rc" = 0 ]; then+ echo "hot update completed without restarting the Fly machine"+ exit 0+ fi+ echo "hot update failed (exit $hot_rc); falling back to full rehydrate" >&2+ mode=rehydrate+ fi++ # Bootstrap/rehydrate refreshes app secrets and reconciles the+ # persistent Docker volume before replacing the machine image.+ flyctl secrets set --app "$APP_NAME" --stage \+ "DOPPLER_TOKEN=$DOPPLER_PREVIEW_TOKEN" \+ "REGISTRY_PULL_TOKEN=$pull_token"+ if ! flyctl volumes list --app "$APP_NAME" --json \+ | jq -e 'map(select(.name == "docker_data")) | length > 0' >/dev/null; then+ flyctl volumes create docker_data --app "$APP_NAME" --region ewr --size 40 --yes+ fi+ # Machines created before the volume existed can't take the mounts+ # config update. A destroyed/partial machine is recreated by deploy.+ flyctl machine list --app "$APP_NAME" --json \+ | jq -r '.[] | select((.config.mounts // []) | length == 0) | .id' \+ | while read -r id; do+ [ -z "$id" ] || flyctl machine destroy "$id" --app "$APP_NAME" --force || true+ done++ image="$registry:${{ github.sha }}"+ docker build -t "$image" preview-ctx+ push_image "$image"+ # Keep the machine in demand while it boots: with zero inbound+ # traffic, fly-proxy counts a health-failing machine as excess+ # capacity and suspends it mid-boot (observed mid-docker-load).+ # Any response status counts as traffic, and a request also+ # resumes an already-suspended machine.+ (while :; do+ curl -s -o /dev/null --max-time 10 "https://$APP_NAME.fly.dev/" || true+ sleep 15+ done) &+ keepalive=$!+ # First boot does real work before 8090 opens (pulling the stack+ # images, snapshot restore, compose up, FusionAuth's JVM) —+ # give the health check more runway than flyctl's default wait.+ rc=0+ flyctl deploy --app "$APP_NAME" \+ --config infra/preview/fly.toml \+ --image "$image" \+ --wait-timeout 1800 \+ --yes || rc=$?+ kill "$keepalive" 2>/dev/null || true+ exit "$rc"+ env:+ FLY_ORG: ${{ vars.FLY_ORG || secrets.FLY_ORG }}+ DOPPLER_PREVIEW_TOKEN: ${{ secrets.DOPPLER_PREVIEW_TOKEN }}+ - name: Dump Fly diagnostics+ if: failure()+ run: |+ flyctl machine list --app "$APP_NAME" || true+ flyctl logs --app "$APP_NAME" --no-tail || true+ - name: Dump boot timings+ run: flyctl logs --app "$APP_NAME" --no-tail | grep -E '\[preview\]|✓' || true+ - name: Comment preview URL+ uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b+ with:+ github-token: ${{ secrets.GITHUB_TOKEN }}+ script: |+ const app = process.env.APP_NAME;+ const url = `https://${app}.fly.dev`;+ const marker = '<!-- fly-preview -->';+ const body = [+ marker,+ `🚀 **Full-stack preview**: ${url}`,+ '',+ `- Suspended when idle — the first request wakes it (a few seconds).`,+ `- Log in with any email; the passwordless code lands in [Mailpit](${url}/mailpit/).`,+ `- Deployed from ${context.payload.pull_request.head.sha.slice(0, 7)}.`,+ ].join('\n');+ const { data: comments } = await github.rest.issues.listComments({+ owner: context.repo.owner,+ repo: context.repo.repo,+ issue_number: context.issue.number,+ per_page: 100,+ });+ const existing = comments.find(c => c.body && c.body.startsWith(marker));+ if (existing) {+ await github.rest.issues.updateComment({+ owner: context.repo.owner,+ repo: context.repo.repo,+ comment_id: existing.id,+ body,+ });+ } else {+ await github.rest.issues.createComment({+ owner: context.repo.owner,+ repo: context.repo.repo,+ issue_number: context.issue.number,+ body,+ });+ }+concurrency:+ group: fly-preview-${{ github.event.pull_request.number }}+ cancel-in-progress: truediff --git a/Cargo.lock b/Cargo.lockindex 157d96cbf8..ce558ccacd 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -16281,6 +16281,7 @@ dependencies = ["serde","serde_json","serde_yaml",+ "sha2 0.10.9","strum","tokio","xtask_paths",diff --git a/apps/web/src/lib/core/constant/servers.ts b/apps/web/src/lib/core/constant/servers.tsindex 4788dc81bd..62e2e29ed5 100644--- a/apps/web/src/lib/core/constant/servers.ts+++ b/apps/web/src/lib/core/constant/servers.ts@@ -47,8 +47,18 @@ type Servers = Record<keyof typeof serverHostRemote, string>;// direct-port behavior (unchanged). Declared BEFORE SERVER_HOSTS so it is// initialized before SERVER_HOSTS evaluates selectLocalServers() at module load// (these are consts in a temporal dead zone; the functions below are hoisted).-const proxyOrigin: string | undefined = import.meta.env+//+// The special value 'same-origin' resolves to the origin the bundle is served+// from, at runtime. The headless stack (`cargo x stack up`) builds with it so+// the static bundle Caddy serves works unchanged on any host that reaches the+// proxy — localhost, a tunnel URL, a preview domain. (globalThis.location+// exists in both windows and workers.)+const rawLocalBackendOrigin: string | undefined = import.meta.env.VITE_LOCAL_BACKEND_ORIGIN;+const proxyOrigin: string | undefined =+ rawLocalBackendOrigin === 'same-origin'+ ? globalThis.location?.origin+ : rawLocalBackendOrigin;const wsProxyOrigin = proxyOrigin?.replace(/^http/, 'ws');export const SERVER_HOSTS: Servers =diff --git a/docs/RUNNING_LOCALLY.md b/docs/RUNNING_LOCALLY.mdindex a4d8b7dabe..7ad2680bdc 100644--- a/docs/RUNNING_LOCALLY.md+++ b/docs/RUNNING_LOCALLY.md@@ -120,6 +120,40 @@ pick up a change there.If you already started without `--build-aux-services` and suspect a stalesync/lexical image, press `q` and restart with the flag.+## Headless Mode (previews, agents, CI)++`just stack` is the same stack without a terminal attached: no hotkey loop, no+dev server. The frontend is built once (a dev-mode bundle with production+optimizations) and served statically by the instance's Caddy proxy, so the whole+product lives behind **one origin** and a finished `up` leaves only Docker+containers running — nothing to babysit.++```bash+just stack up # bring everything up, print URLs, return+just stack status --json # machine-readable state (containers, health, URLs)+just stack update # rebuild + reload only changed services (the `r` hotkey)+just stack update --frontend # also rebuild the frontend bundle+just stack down # containers + volumes + state+```++All the `run_local` flags apply (`--instance`, `--no-doppler --env-file`,+`--no-build`, `--binaries-dir`); CI can hand in a prebuilt bundle with+`--frontend-dist`, and `--infra-only` stops after the infra bring-up + init+(the CI bake mode — without Doppler the app services have no env to boot+with, and the snapshot only captures infra volumes). The app is served at `<proxy>/app/` — the bundle resolves its+backend from the origin it is served on, so the same stack works on localhost+or behind a preview hostname without a rebuild.++`stack up` also caches the expensive infra init. The first cold run migrates+the DB, waits out the FusionAuth kickstart, and creates the search indices,+then saves those volumes as a content-addressed **init snapshot** (keyed by the+migrations, kickstart, index mappings, image pins, and container platform —+stored under `infra/local/generated/.snapshots`). Later runs whose inputs match restore the+snapshot and skip the init entirely; any input change is a cache miss and a+normal full init. `just stack snapshot` shows the current key; `--no-snapshot`+opts out. This is also what makes Fly previews boot fast — CI bakes the+snapshot into the preview image (see `infra/preview/README.md`).+## Common CommandsRun local binaries against shared dev resources instead of a fully local stack:diff --git a/infra/preview/Dockerfile b/infra/preview/Dockerfilenew file mode 100644index 0000000000..d131641c65--- /dev/null+++ b/infra/preview/Dockerfile@@ -0,0 +1,59 @@+# The Fly preview machine image: the entire local stack inside one VM.+#+# Built by .github/workflows/preview-fly.yml from a STAGED context (not the+# repo root) — CI lays out everything the machine needs and this file just+# copies it in:+#+# repo/ minimal repo layout xtask reads (compose files, migrations,+# index helpers) — MACRO_REPO_ROOT points here+# artifacts/ binaries/ (zigbuild output), frontend-dist/ (same-origin+# bundle), snapshots/ (baked init snapshot)+# bin/ prebuilt orchestrator + the in-place hot-update driver+# preload/ manifest of every image the stack runs (image ID -> local tag+# -> registry ref); the entrypoint pulls them from the app's Fly+# registry repo at boot, skipping whatever the machine's+# persistent layer store already has+#+# Fly machines are full microVMs, so an inner Docker daemon runs the same+# compose topology as `just run_local` — the preview IS the local stack.+FROM ubuntu:24.04++RUN apt-get update && apt-get install -y --no-install-recommends \+ docker.io \+ docker-compose-v2 \+ docker-buildx \+ ca-certificates \+ curl \+ gnupg \+ jq \+ && rm -rf /var/lib/apt/lists/*++# Doppler CLI: the machine boots with a config-scoped service token+# (DOPPLER_TOKEN, set as a Fly secret by the workflow) and the stack's env+# layer pulls the preview secrets with it.+RUN curl -Ls --tlsv1.2 --proto "=https" https://cli.doppler.com/install.sh | sh++# One COPY per artifact, ordered stable → volatile: the staging step preserves+# mtimes, so a layer whose inputs didn't change this deploy is byte-identical+# to the previous image's — the registry push and the Fly host's pull both+# skip it. Everything after the first changed layer re-ships, so the pieces+# that churn every deploy (frontend bundle, service binaries) go last.+COPY entrypoint.sh /srv/macro/entrypoint.sh+COPY deployment.json /srv/macro/deployment.json+COPY repo/ /srv/macro/repo/+COPY bin/ /srv/macro/bin/+COPY artifacts/snapshots/ /srv/macro/artifacts/snapshots/+COPY preload/ /srv/macro/preload/+COPY artifacts/frontend-dist/ /srv/macro/artifacts/frontend-dist/+COPY artifacts/binaries/ /srv/macro/artifacts/binaries/+RUN chmod +x /srv/macro/entrypoint.sh /srv/macro/bin/xtask /srv/macro/bin/hot-update++# Point the prebuilt xtask at the staged layout (its compiled-in paths refer to+# the CI checkout) and at the baked snapshot store.+ENV MACRO_REPO_ROOT=/srv/macro/repo \+ MACRO_STACK_SNAPSHOT_DIR=/srv/macro/artifacts/snapshots++# The instance's single-origin proxy (fly.toml's internal_port).+EXPOSE 8090++ENTRYPOINT ["/srv/macro/entrypoint.sh"]diff --git a/infra/preview/README.md b/infra/preview/README.mdnew file mode 100644index 0000000000..6b60a17d90--- /dev/null+++ b/infra/preview/README.md@@ -0,0 +1,93 @@+# Full-stack PR previews on Fly++Label a PR `preview` and CI deploys the **entire stack** — every service,+Postgres, OpenSearch, FusionAuth, LocalStack, the frontend — as one Fly app at+`https://macro-pr-<N>.fly.dev`. The app suspends when idle (≈zero cost) and the+fly-proxy wakes it on the next request. Closing the PR (or removing the label)+destroys it.++## How it works++The preview is not a parallel deployment system: it is the local stack+(`just stack up`, see `RUNNING_LOCALLY.md`) running inside one Fly machine.+A Fly machine is a full microVM, so an inner Docker daemon runs the exact+compose topology `run_local` uses. What differs is that **everything expensive+is baked in CI** and the machine only restores:++| Baked in CI | Restored on the machine |+|---|---|+| Service binaries (`cargo x zigbuild`) | bind-mounted into the runtime image |+| Frontend bundle (`same-origin` build) | served statically by the instance Caddy |+| Init snapshot (migrations + FusionAuth kickstart + search indices) | volumes restored, init skipped |+| Every Docker image, mirrored to the app's Fly registry repo | `docker pull` at boot — layer-dedup'd against the machine's persistent layer store, nothing built |++Boot on a fresh machine ≈ image pulls + snapshot restore + JVM startup+(a couple of minutes); wake from suspend ≈ seconds. Pushes to the PR choose one+of three paths from the durable deployment marker on the Docker volume:++- **bootstrap** — no app/machine yet: create everything and restore the snapshot;+- **hot update** — the snapshot key, staged-runtime hash, and update protocol+ are unchanged: pull a layer-deduplicated artifact carrier, atomically replace+ changed binaries/frontend, and restart only affected containers without+ restarting the Fly machine or infra;+- **rehydrate** — compatibility changed, or a hot update fails: fall back to+ the immutable full-image deploy and snapshot restore.++The URL remains stable across all three paths.++The whole product sits behind the stack's single-origin Caddy proxy+(`:8090`), so one `internal_port` covers the frontend, APIs, WebSockets, and+Mailpit.++## Logging in++Login is passwordless (email codes) and preview email is captured, not sent:+enter any address on the login screen, then open+`https://macro-pr-<N>.fly.dev/mailpit/` to read the code. (Routing login codes+through scoped SES so teammates get real emails is a planned follow-up.)++## Secrets (Doppler)++The stack's local plumbing (DBs, queues, auth identity) is code-owned and+dummy, but integration secrets (AI keys etc.) come from Doppler, exactly like+`run_local`. The workflow stages a **config-scoped service token** onto each+preview app as the Fly secret `DOPPLER_TOKEN`; at boot the env layer detects+the token and pulls that config (no `doppler login` involved), then overlays+the code-owned `LocalEnv` on top — so Doppler supplies integration keys while+local plumbing stays local. A preview machine can only ever read the one+config its token is scoped to.++## One-time setup++1. Create a Fly org deploy token: `fly tokens create org -o <org>` →+ repo secret `FLY_API_TOKEN`.+2. Set the repo variable `FLY_ORG` to the org slug.+3. In Doppler, create a preview config in the `local` project (e.g.+ `lcl_preview` — start from `lcl_personal`, keep only what previews should+ have) and a service token for it → repo secret `DOPPLER_PREVIEW_TOKEN`.+4. Create the `preview` label.++## Security posture++- Previews only build from same-repo branches (the workflow refuses forks).+- No real AWS: S3/SQS/DynamoDB are LocalStack, email is Mailpit. The+ cloud-reachable credentials on a machine are the preview Doppler config's+ contents — scope that config accordingly (assume PR code can read it) — and+ `REGISTRY_PULL_TOKEN`, the CI deploy token attenuated (client-side macaroon+ caveats) to read-only on this one preview app with a 7-day validity window:+ it can pull the app's registry repo, nothing else.+- URLs are public. Anyone with the link can use the preview (and read its+ Mailpit). Don't put sensitive data in one. Edge auth (oauth2-proxy in front+ of Caddy) is a planned hardening step.++## Known costs & future optimizations++- The stack images are mirrored into the per-PR app's registry repo, so the+ first deploy of a PR pushes (and its first boot pulls) the full set; after+ that both sides move only changed layers. There is no cross-PR sharing —+ a shared registry (GHCR/Namespace) would fix that at the cost of new+ org-level credentials.+- The aux images (sync/websocket/lexical) rebuild in CI on every preview+ deploy. Runner-level Docker layer caching covers most of it.+- `cpus = 8, memory = 16gb` (shared) is a deliberate over-provision; measure+ and shrink once a few previews have run.diff --git a/infra/preview/entrypoint.sh b/infra/preview/entrypoint.shnew file mode 100644index 0000000000..9186fcaae9--- /dev/null+++ b/infra/preview/entrypoint.sh@@ -0,0 +1,94 @@+#!/usr/bin/env bash+# Boot for the Fly preview machine: start the inner Docker daemon, pull the+# stack images from the app's registry repo, bring the stack up from the baked+# artifacts, then idle while the containers serve. Everything expensive+# (compile, bundle, init) happened in CI — this script only restores and+# starts.+set -euo pipefail++log() { echo "[preview] $*"; }++t_boot=$(date +%s)+# A hard loss during a hot update cannot run its EXIT trap. The full immutable+# boot is the recovery boundary, so clear any abandoned volume-backed lock.+state_dir=/var/lib/docker/.macro-preview+mkdir -p "$state_dir"+rmdir "$state_dir/update.lock" 2>/dev/null || true+log "starting inner dockerd"+dockerd >/var/log/dockerd.log 2>&1 &++for _ in $(seq 1 120); do+ docker info >/dev/null 2>&1 && break+ sleep 1+done+if ! docker info >/dev/null 2>&1; then+ log "dockerd failed to start:"+ tail -n 50 /var/log/dockerd.log+ exit 1+fi+log "dockerd ready in $(($(date +%s) - t_boot))s"++# Pull the stack images from the app's Fly registry repo (CI mirrored them+# there at deploy time). /var/lib/docker is a persistent volume, so the+# manifest (image ID per tag) lets redeploys skip every image already in the+# store, and pulls dedup at the layer level — only layers that actually+# changed cross the network. Suspend/resume never re-runs this; only real+# boots do. REGISTRY_PULL_TOKEN is an app-scoped deploy token staged as a Fly+# secret by the workflow.+if [ -f /srv/macro/preload/manifest.txt ]; then+ if [ -n "${REGISTRY_PULL_TOKEN:-}" ]; then+ docker login registry.fly.io -u x --password-stdin <<<"$REGISTRY_PULL_TOKEN"+ fi+ log "pulling stack images (parallel, warm store aware)"+ t0=$(date +%s)+ pids=""+ while read -r id tag ref; do+ have=$(docker image inspect -f '{{.Id}}' "$tag" 2>/dev/null || true)+ if [ "$have" = "$id" ]; then+ log "already present: $tag"+ continue+ fi+ (docker pull -q "$ref" && docker tag "$ref" "$tag" && docker rmi "$ref" >/dev/null) &+ pids="$pids $!"+ done < /srv/macro/preload/manifest.txt+ rc=0+ for pid in $pids; do wait "$pid" || rc=1; done+ [ "$rc" = 0 ] || { log "image pull failed"; exit 1; }+ log "images pulled in $(($(date +%s) - t0))s"+fi++# With a DOPPLER_TOKEN (a config-scoped service token, injected as a Fly+# secret) the env layer pulls the preview secrets; without one the stack runs+# on the code-owned local dummies only.+doppler_args=(--no-doppler)+if [ -n "${DOPPLER_TOKEN:-}" ]; then+ log "DOPPLER_TOKEN present — pulling preview secrets"+ doppler_args=()+fi++log "bringing the stack up (snapshot restore)"+t0=$(date +%s)+/srv/macro/bin/xtask stack up \+ "${doppler_args[@]}" \+ --no-build \+ --binaries-dir /srv/macro/artifacts/binaries \+ --frontend-dist /srv/macro/artifacts/frontend-dist \+ --json++log "stack up took $(($(date +%s) - t0))s (boot total $(($(date +%s) - t_boot))s)"+# The backend health gate is the longest stack-up phase; surface the auth+# service's step timings plus its timestamped tail so the wait is attributable+# from the CI boot-timings step (which greps [preview] out of `flyctl logs`).+docker logs -t macro-authentication-service-1 2>&1 \+ | grep 'authentication startup step' \+ | sed 's/^/[preview][auth-startup] /' || true+docker logs -t --tail 40 macro-authentication-service-1 2>&1 \+ | sed 's/^/[preview][auth] /' || true++# Commit the compatibility marker only after the restored stack is healthy.+# It lives beside (not inside) Docker's own data on the persistent volume, so+# CI can distinguish a hot-updatable machine from an absent/partial/old deploy.+install -m 0644 /srv/macro/deployment.json "$state_dir/deployment.json.next"+mv -f "$state_dir/deployment.json.next" "$state_dir/deployment.json"+log "stack ready — proxy serving on :8090"+exec sleep infinitydiff --git a/infra/preview/fly.toml b/infra/preview/fly.tomlnew file mode 100644index 0000000000..e49362343c--- /dev/null+++ b/infra/preview/fly.toml@@ -0,0 +1,43 @@+# Fly config for per-PR preview apps. The workflow overrides the app name+# (`--app macro-pr-<N>`) and supplies the image (`--image`); this file carries+# everything else. See infra/preview/README.md.+app = "macro-preview-template"+primary_region = "ewr"++# Everything (frontend, APIs, WebSockets, Mailpit) is behind the stack's+# single-origin Caddy proxy.+[http_service]+ internal_port = 8090+ force_https = true+ # The whole point: an idle preview suspends to ~zero cost and the fly-proxy+ # wakes it on the next request.+ auto_stop_machines = "suspend"+ auto_start_machines = true+ min_machines_running = 0++ [[http_service.checks]]+ interval = "15s"+ timeout = "5s"+ # First boot loads the baked images and restores the init snapshot before+ # the backend answers — give it room.+ grace_period = "5m"+ method = "GET"+ path = "/auth/health"++# The inner Docker store must NOT live on the machine rootfs: Fly's rootfs is+# an overlay (nested overlayfs forbids the whiteout device nodes layer+# unpacking needs — every Debian-based image failed with EPERM) and has only+# a few GB of headroom (ENOSPC mid-load). A volume is real ext4, properly+# sized, and persists across redeploys — so later pushes reuse the layer+# store. The deploy step creates the volume if it doesn't exist.+[[mounts]]+ source = "docker_data"+ destination = "/var/lib/docker"++[[vm]]+ cpu_kind = "shared"+ cpus = 8+ memory = "16gb"++[[restart]]+ policy = "on-failure"diff --git a/infra/preview/hot-update.sh b/infra/preview/hot-update.shnew file mode 100644index 0000000000..c31a1c79b6--- /dev/null+++ b/infra/preview/hot-update.sh@@ -0,0 +1,122 @@+#!/usr/bin/env bash+# Apply a registry-carried artifact update to an already-running preview VM.+# Exit 42 asks CI to fall back to a full image deploy/stack rehydrate.+set -euo pipefail++readonly REHYDRATE_EXIT=42+readonly STATE_DIR=/var/lib/docker/.macro-preview+readonly STATE_FILE="$STATE_DIR/deployment.json"++log() { echo "[preview][hot-update] $*"; }+rehydrate() { log "$*; requesting full rehydrate" >&2; exit "$REHYDRATE_EXIT"; }++if [ "$#" -ne 2 ]; then+ echo "usage: hot-update <registry-image> <registry-token-file>" >&2+ exit 2+fi+update_image=$1+token_file=$2++mkdir -p "$STATE_DIR"+if ! mkdir "$STATE_DIR/update.lock" 2>/dev/null; then+ echo "another preview update is already running" >&2+ exit 1+fi++staging=+container=+cleanup() {+ rm -f "$token_file"+ [ -z "$container" ] || docker rm -f "$container" >/dev/null 2>&1 || true+ [ -z "$staging" ] || rm -rf "$staging"+ rmdir "$STATE_DIR/update.lock" 2>/dev/null || true+}+trap cleanup EXIT++[ -s "$token_file" ] || rehydrate "registry pull token is missing"+[ -f "$STATE_FILE" ] || rehydrate "deployment state is missing"++log "pulling artifact image $update_image"+docker login registry.fly.io -u x --password-stdin < "$token_file" >/dev/null+docker pull -q "$update_image"+container=$(docker create "$update_image")+staging=$(mktemp -d /srv/macro/.hot-update.XXXXXX)+docker cp "$container:/update/." "$staging/"+docker rm "$container" >/dev/null+container=++new_format=$(jq -er '.format' "$staging/deployment.json")+new_snapshot=$(jq -er '.snapshot_key' "$staging/deployment.json")+new_runtime=$(jq -er '.runtime_key' "$staging/deployment.json")+new_frontend=$(jq -er '.frontend_key' "$staging/deployment.json")+current_format=$(jq -er '.format' "$STATE_FILE")+current_snapshot=$(jq -er '.snapshot_key' "$STATE_FILE")+current_runtime=$(jq -er '.runtime_key' "$STATE_FILE")+current_frontend=$(jq -er '.frontend_key' "$STATE_FILE")+[ "$new_format" = "1" ] || rehydrate "unsupported update format $new_format"+[ "$current_format" = "$new_format" ] || rehydrate "deployment format changed"+[ "$current_snapshot" = "$new_snapshot" ] || rehydrate "snapshot key changed"+[ "$current_runtime" = "$new_runtime" ] || rehydrate "runtime configuration changed"++# CI mirrors every compose image and puts the desired IDs in the carrier.+# Docker-built leaf services are safe to recreate in place. Any other image+# change is infrastructure/runtime drift and must use the full rehydrate path.+aux_changed=+unsafe_images=+while read -r expected_id tag registry_ref; do+ current_id=$(docker image inspect -f '{{.Id}}' "$tag" 2>/dev/null || true)+ [ "$current_id" = "$expected_id" ] && continue+ case "$tag" in+ *-ai_editing_worker|*-lexical_service|*-sync_service|*-websocket_service)+ log "refreshing auxiliary image $tag"+ docker pull -q "$registry_ref"+ pulled_id=$(docker image inspect -f '{{.Id}}' "$registry_ref")+ [ "$pulled_id" = "$expected_id" ] \+ || rehydrate "registry image ID mismatch for $tag"+ docker tag "$registry_ref" "$tag"+ docker rmi "$registry_ref" >/dev/null+ aux_changed=1+ ;;+ *) unsafe_images="$unsafe_images $tag" ;;+ esac+done < "$staging/manifest.txt"+[ -z "$unsafe_images" ] || rehydrate "non-hot-updatable images changed:$unsafe_images"++# SSH sessions don't inherit app secrets. Recover only the config-scoped+# Doppler token from PID 1 without printing it, then let xtask regenerate the+# local env file before recreating/restarting containers.+doppler_token=$(tr '\0' '\n' < /proc/1/environ | sed -n 's/^DOPPLER_TOKEN=//p' | head -n 1)+[ -z "$doppler_token" ] || export DOPPLER_TOKEN="$doppler_token"+export MACRO_REPO_ROOT=/srv/macro/repo+export MACRO_STACK_SNAPSHOT_DIR=/srv/macro/artifacts/snapshots++apply_args=(+ stack update+ --binaries-dir "$staging/binaries"+ --json+)+[ "$current_frontend" = "$new_frontend" ] \+ || apply_args+=(--frontend-dist "$staging/frontend-dist")+[ -z "$aux_changed" ] || apply_args+=(--recreate-aux-services)++log "applying prebuilt artifacts"+"$staging/xtask" "${apply_args[@]}"++# Publish the new orchestrator and compatibility marker only after the stack is+# healthy. A failed apply leaves the old marker, so the next run rehydrates.+install -m 0755 "$staging/xtask" /srv/macro/bin/xtask.next+mv -f /srv/macro/bin/xtask.next /srv/macro/bin/xtask+install -m 0644 "$staging/deployment.json" "$STATE_FILE.next"+mv -f "$STATE_FILE.next" "$STATE_FILE"++# Retain the current carrier's layers for the next incremental pull, but remove+# older carrier tags and their now-unreferenced layers from the persistent store.+repository=${update_image%:*}+while read -r old; do+ case "$old" in+ "$repository":update-*) [ "$old" = "$update_image" ] || docker rmi "$old" >/dev/null || true ;;+ esac+done < <(docker images "$repository" --format '{{.Repository}}:{{.Tag}}')+docker image prune -f >/dev/null || true++log "update complete"diff --git a/infra/preview/update.Dockerfile b/infra/preview/update.Dockerfilenew file mode 100644index 0000000000..28084f0e25--- /dev/null+++ b/infra/preview/update.Dockerfile@@ -0,0 +1,13 @@+# Registry-carried artifact bundle for a running preview machine. The image is+# never started: hot-update.sh creates a stopped container and copies /update+# out of it. Scratch keeps every layer limited to actual application artifacts,+# and stable COPY ordering maximizes registry/inner-Docker dedup across pushes.+FROM scratch++COPY bin/xtask /update/xtask+COPY preload/manifest.txt /update/manifest.txt+COPY deployment.json /update/deployment.json+COPY artifacts/frontend-dist/ /update/frontend-dist/+COPY artifacts/binaries/ /update/binaries/++CMD ["/update/xtask"]diff --git a/services/authentication_service/src/main.rs b/services/authentication_service/src/main.rsindex d4e19b901e..6c907a211c 100644--- a/services/authentication_service/src/main.rs+++ b/services/authentication_service/src/main.rs@@ -66,8 +66,10 @@ async fn main() -> anyhow::Result<()> {MacroEntrypoint::default().init();let env = Environment::new_or_prod();+ // One SDK config is sufficient for every AWS client in this process.+ let aws_config = macro_aws_config::get_macro_aws_config().await;let secretsmanager_client = secretsmanager_client::SecretsManager::new(- aws_sdk_secretsmanager::Client::new(¯o_aws_config::get_macro_aws_config().await),+ aws_sdk_secretsmanager::Client::new(&aws_config),);// Parse our configuration from the environment.@@ -173,7 +175,7 @@ async fn main() -> anyhow::Result<()> {// `from_env` routes to local SMTP (Mailpit) when SMTP_HOST is set, else SES.let ses_client = ses_client::Ses::from_env(- aws_sdk_sesv2::Client::new(¯o_aws_config::get_macro_aws_config().await),+ aws_sdk_sesv2::Client::new(&aws_config),&config.environment.to_string(),);@@ -201,12 +203,10 @@ async fn main() -> anyhow::Result<()> {};tracing::trace!("initialized notification ingress service");- let sqs_client = sqs_client::SQS::new(aws_sdk_sqs::Client::new(- ¯o_aws_config::get_macro_aws_config().await,- ))- .search_event_queue(&search_event_queue)- .email_link_manager_queue(&link_manager_queue)- .email_backfill_queue(&email_backfill_queue);+ let sqs_client = sqs_client::SQS::new(aws_sdk_sqs::Client::new(&aws_config))+ .search_event_queue(&search_event_queue)+ .email_link_manager_queue(&link_manager_queue)+ .email_backfill_queue(&email_backfill_queue);tracing::trace!("initialized sqs client");// Initialize analytics client with configured providersdiff --git a/tooling/just/xtask.just b/tooling/just/xtask.justindex 8e942d0d8c..c1e8d91dea 100644--- a/tooling/just/xtask.just+++ b/tooling/just/xtask.just@@ -31,3 +31,11 @@ destroy_local *ARGS:# Stop an instance's containers, keeping volumes (xtask).stop_local *ARGS:{{ xtask }} stop-local "$@"++# Headless stack orchestration for previews, agents, and CI.+# just stack up # full stack; static frontend served by the proxy+# just stack update # rebuild + reload only what changed (the `r` hotkey)+# just stack status --json # machine-readable state+# just stack down # reclaim everything+stack *ARGS:+ {{ xtask }} stack "$@"diff --git a/tooling/xtask/crates/xtask_local/Cargo.toml b/tooling/xtask/crates/xtask_local/Cargo.tomlindex 979276ec5a..8bebb50fb3 100644--- a/tooling/xtask/crates/xtask_local/Cargo.toml+++ b/tooling/xtask/crates/xtask_local/Cargo.toml@@ -22,6 +22,8 @@ clap = { workspace = true }macro_aws_config = { path = "../../../../crates/macro_aws_config" }macro_queues = { path = "../../../../crates/macro_queues" }serde = { workspace = true }+# Content-address the init snapshots (`stack up` restore/save keys).+sha2 = { workspace = true }tokio = { workspace = true }# Source of truth for the declared Kafka topics (local-stack provisioning).macro_event_topics = { path = "../../../../crates/macro_event_topics" }diff --git a/tooling/xtask/crates/xtask_local/src/local/build/zigbuild.rs b/tooling/xtask/crates/xtask_local/src/local/build/zigbuild.rsindex 88d551b909..3e6bbb4680 100644--- a/tooling/xtask/crates/xtask_local/src/local/build/zigbuild.rs+++ b/tooling/xtask/crates/xtask_local/src/local/build/zigbuild.rs@@ -98,9 +98,37 @@ fn base_command(ws: &Path, zig_cache: &Path, jobs: &str, target: Target) -> Comm.env("ZIG_LOCAL_CACHE_DIR", zig_cache).env("RUSTC_WRAPPER", "sccache").env("XDG_CACHE_HOME", zig_cache);+ if let Some(flag) = curl_include_flag() {+ // First-found wins in cc's env chain (CFLAGS_<target> before CFLAGS),+ // so set the target-specific var too in case something else exports a+ // plain CFLAGS that would otherwise shadow ours — and vice versa.+ let var = format!("CFLAGS_{}", target.triple.replace('-', "_"));+ cmd.env(var, &flag).env("CFLAGS", &flag);+ }cmd}+/// librdkafka 2.12's bundled cmake build needs curl *headers* even with+/// `WITH_CURL=0` (its config header always defines `WITH_OAUTHBEARER_OIDC` via+/// `#cmakedefine01`, and rdkafka_conf.c guards the curl include with `#ifdef`+/// instead of `#if`). Host builds get them from the Nix cc-wrapper, which+/// reads `NIX_CFLAGS_COMPILE` — but `zig cc` is a raw clang that ignores it,+/// so the dev shell's `curl.dev` never reaches the cross cmake build. Mine+/// the wrapper flags for an include dir that actually carries `curl/curl.h`+/// and hand it to cc/cmake explicitly; `-idirafter` keeps it at the lowest+/// search priority so it cannot shadow zig's own libc headers.+// Reading the ambient Nix shell env is the point here; there is no config+// layer this could come from.+#[allow(clippy::disallowed_methods)]+fn curl_include_flag() -> Option<String> {+ let flags = std::env::var("NIX_CFLAGS_COMPILE").ok()?;+ flags+ .split_whitespace()+ .filter(|tok| tok.starts_with('/'))+ .find(|dir| Path::new(dir).join("curl/curl.h").is_file())+ .map(|dir| format!("-idirafter {dir}"))+}+/// Fail with an actionable hint if the rust-std for `target` is not installed./// The repo's `rustup` is a no-op shim, so the only fix is adding the triple to/// `rust-toolchain.toml` and re-entering `nix develop`.diff --git a/tooling/xtask/crates/xtask_local/src/local/cli.rs b/tooling/xtask/crates/xtask_local/src/local/cli.rsindex aa3e7dc966..947906293e 100644--- a/tooling/xtask/crates/xtask_local/src/local/cli.rs+++ b/tooling/xtask/crates/xtask_local/src/local/cli.rs@@ -42,6 +42,24 @@ enum Cmd {ResetLocal(InstanceArgs),/// Remove an instance's containers, networks, and volumes.DestroyLocal(InstanceArgs),+ /// Headless stack orchestration (previews, agents, CI) — no TTY, no+ /// attached dev server; the proxy serves a static frontend bundle.+ #[command(subcommand)]+ Stack(StackCmd),+}++#[derive(Subcommand)]+pub enum StackCmd {+ /// Bring a full local stack up and return (only containers keep running).+ Up(super::stack::UpArgs),+ /// Rebuild binaries (and optionally the frontend) and reload what changed.+ Update(super::stack::UpdateArgs),+ /// Report the instance's containers, health, and URLs (`--json` for machines).+ Status(super::stack::StatusArgs),+ /// Tear the instance down: containers, volumes, and state.+ Down(super::stack::DownArgs),+ /// Report the init-snapshot key/location for this instance.+ Snapshot(super::stack::SnapshotArgs),}#[derive(Args, Clone, Default)]@@ -85,11 +103,10 @@ pub struct RunArgs {pub env: EnvArgs,#[command(flatten)]pub build: BuildArgs,- /// Do not start the frontend dev server.+ /// Do not start or serve the frontend.#[arg(long)]pub no_frontend: bool,- /// Stream subprocess output and show per-step timings (e.g. on `r`, the- /// build vs reload split instead of one folded line).+ /// Stream subprocess output and show per-step timings.#[arg(long, short)]pub verbose: bool,}@@ -160,5 +177,12 @@ fn run(cli: Cli) -> Result<()> {Cmd::StopLocal(a) => super::stop(&a),Cmd::ResetLocal(a) => super::reset(&a),Cmd::DestroyLocal(a) => super::destroy(&a),+ Cmd::Stack(cmd) => match cmd {+ StackCmd::Up(a) => super::stack::up(Mode::Local, &a),+ StackCmd::Update(a) => super::stack::update(&a),+ StackCmd::Status(a) => super::stack::status(&a),+ StackCmd::Down(a) => super::stack::down(&a),+ StackCmd::Snapshot(a) => super::stack::snapshot_status(&a),+ },}}diff --git a/tooling/xtask/crates/xtask_local/src/local/env_layer.rs b/tooling/xtask/crates/xtask_local/src/local/env_layer.rsindex b691bbd83c..0d35e668da 100644--- a/tooling/xtask/crates/xtask_local/src/local/env_layer.rs+++ b/tooling/xtask/crates/xtask_local/src/local/env_layer.rs@@ -17,7 +17,7 @@ use std::collections::BTreeMap;use std::path::{Path, PathBuf};use std::process::Command;-use anyhow::{Context, Result};+use anyhow::{Context, Result, bail};use super::instance::Instance;use super::{Mode, local_env};@@ -171,23 +171,41 @@ fn pull_aws_credentials(env: &mut BTreeMap<String, String>) {}}-/// Pull the named Doppler `local`-project config as a flat env overlay (mirrors-/// the old `just get_environment`). Non-fatal: if Doppler is unavailable in local-/// mode we proceed on defaults + env-file. Returns whether secrets were loaded.+/// Pull Doppler secrets as a flat env overlay (mirrors the old+/// `just get_environment`). Two modes:+///+/// - `DOPPLER_TOKEN` in the process env (CI, Fly preview machines) is a+/// config-scoped service token: download WITHOUT `--project`/`--config` (the+/// token pins them), and treat failure as fatal — whoever provisioned the+/// token explicitly asked for these secrets, so proceeding without them+/// would just move the breakage into the services.+/// - Otherwise pull the named `local`-project config via the developer's+/// `doppler login`. Non-fatal: if Doppler is unavailable in local mode we+/// proceed on defaults + env-file.+///+/// Returns whether secrets were loaded.+// xtask is host tooling, not a service reading APP_SECRETS_JSON, so reading the+// process environment directly is correct here.+#[allow(clippy::disallowed_methods)]fn pull_doppler(config: &str, env: &mut BTreeMap<String, String>) -> Result<bool> {- let output = Command::new("doppler")- .args([- "secrets",- "download",- "--project",- "local",- "--config",- config,- ])- .args(["--format", "json", "--no-file"])- .output();+ let token_scoped = std::env::var("DOPPLER_TOKEN").is_ok_and(|v| !v.is_empty());+ let mut cmd = Command::new("doppler");+ cmd.args(["secrets", "download"]);+ if !token_scoped {+ cmd.args(["--project", "local", "--config", config]);+ }+ cmd.args(["--format", "json", "--no-file"]);+ let output = cmd.output();let output = match output {Ok(o) if o.status.success() => o,+ Ok(o) if token_scoped => bail!(+ "doppler secrets download failed with DOPPLER_TOKEN set:\n{}",+ String::from_utf8_lossy(&o.stderr).trim_end()+ ),+ Err(e) if token_scoped => {+ return Err(anyhow::Error::from(e))+ .context("running doppler with DOPPLER_TOKEN set (is the doppler CLI installed?)");+ }_ => return Ok(false),};let json: BTreeMap<String, serde_json::Value> =diff --git a/tooling/xtask/crates/xtask_local/src/local/frontend.rs b/tooling/xtask/crates/xtask_local/src/local/frontend.rsindex bc4046e5df..4e0bc5f954 100644--- a/tooling/xtask/crates/xtask_local/src/local/frontend.rs+++ b/tooling/xtask/crates/xtask_local/src/local/frontend.rs@@ -4,6 +4,7 @@use std::io::Read;use std::os::unix::process::CommandExt;+use std::path::Path;use std::process::{Child, Command, Stdio};use std::sync::{Arc, Mutex};use std::thread::JoinHandle;@@ -23,6 +24,69 @@ pub fn url(instance: &Instance) -> String {format!("http://localhost:{}/app", instance.port(Port::Frontend))}+/// Frontend URL when the proxy serves the static bundle (headless stacks): the+/// app lives on the single proxy origin, not a dev-server port.+pub fn static_url(instance: &Instance) -> String {+ format!("{}/app/", proxy::url(instance))+}++/// Where the instance's static frontend build is staged. Mounted read-only into+/// the proxy container at `/srv/frontend` (see `gen_compose::add_proxy_service`).+/// Staged per instance — not served from `dist/` directly — so concurrent+/// instances in one checkout can't clobber each other's running frontend, and a+/// later `bun run build` can't mutate a live stack out from under itself.+pub fn static_dir(instance: &Instance) -> std::path::PathBuf {+ instance.artifact_dir().join("frontend")+}++/// Build the app bundle for headless serving and stage it into the instance+/// dir. `prebuilt` skips the build and stages an existing dist (CI hands the+/// binaries-style artifact straight in). The build mirrors `just build-dev`+/// (dev-mode bundle, production optimizations), except the backend origin is+/// the `same-origin` sentinel — resolved from `location.origin` at runtime — so+/// the one bundle works on localhost and through any tunnel/preview hostname.+pub fn build_static(stage: &Stage, instance: &Instance, prebuilt: Option<&Path>) -> Result<()> {+ let dist = match prebuilt {+ Some(dir) => dir.to_owned(),+ None => {+ let mut cmd = Command::new("bun");+ cmd.current_dir(app_dir())+ .args(["run", "--bun", "build"])+ .env("MODE", "development")+ .env("NODE_ENV", "production")+ .env("VITE_LOCAL_SERVERS", "ALL")+ .env("VITE_LOCAL_BACKEND_ORIGIN", "same-origin");+ stage.run("Building frontend bundle", &mut cmd)?;+ app_dir().join("dist")+ }+ };+ if stage.is_dry_run() {+ return Ok(());+ }+ if !dist.join("index.html").exists() {+ anyhow::bail!(+ "no index.html in {} — frontend build did not produce a bundle",+ dist.display()+ );+ }+ stage.run_step("Staging frontend bundle", || {+ let staged = static_dir(instance);+ if staged.exists() {+ std::fs::remove_dir_all(&staged)+ .with_context(|| format!("clearing {}", staged.display()))?;+ }+ instance.ensure_artifact_dir()?;+ let status = Command::new("cp")+ .arg("-a")+ .arg(&dist)+ .arg(&staged)+ .status()+ .context("running cp -a")?;+ anyhow::ensure!(status.success(), "cp -a exited with {status}");+ Ok(())+ })+}+/// The env the dev server runs with. Both local and dev point the whole app at/// the local proxy origin (single backend origin); the modes differ only in/// what the *services* behind the proxy talk to (local infra vs dev resources).diff --git a/tooling/xtask/crates/xtask_local/src/local/fusionauth.rs b/tooling/xtask/crates/xtask_local/src/local/fusionauth.rsindex 80fa0cac3a..860898ad15 100644--- a/tooling/xtask/crates/xtask_local/src/local/fusionauth.rs+++ b/tooling/xtask/crates/xtask_local/src/local/fusionauth.rs@@ -6,7 +6,7 @@ use std::process::Command;use anyhow::{Context, Result};use super::instance::{Instance, Port};-use super::{gen_compose, kickstart, stage::Stage};+use super::{gen_compose, identity, kickstart, stage::Stage};/// The local populate-JWT lambda body, inlined into `kickstart.json`.///@@ -49,18 +49,30 @@ pub fn write_kickstart(instance: &Instance) -> Result<()> {Ok(())}-/// Block until FusionAuth reports status Ok (it applies the kickstart on first-/// boot against an empty DB, which is slow). Runs as a stage so it shows the-/// spinner.+/// Block until the kickstart has fully applied (first boot against an empty DB+/// is slow). Runs as a stage so it shows the spinner.+///+/// Polls the kickstart's OWN artifacts — a tenant fetch authorized by the+/// kickstart API key — NOT `/api/status`: FusionAuth reports status Ok while+/// the kickstart is still applying, and the snapshot save stops the containers+/// right after this wait. Gating on status alone once froze a mid-kickstart DB+/// into a snapshot (no tenant), and every stack restored from it 500'd at+/// login with `InvalidTenantIdException` — while the key never changed, so the+/// bad snapshot was sticky. The fetch below succeeds only once the API key,+/// the tenant, and the application all exist (the kickstart creates the+/// application after the tenant).pub fn wait_ready(stage: &Stage, instance: &Instance) -> Result<()> {let url = format!(- "http://localhost:{}/api/status",- instance.port(Port::FusionAuth)+ "http://localhost:{}/api/application/{}",+ instance.port(Port::FusionAuth),+ identity::APPLICATION_ID,);let script = format!(- "for i in $(seq 1 90); do curl -fsS {url} 2>/dev/null | grep -q '\"status\":\"Ok\"' && exit 0; sleep 2; done; echo 'timed out waiting for FusionAuth'; exit 1"+ "for i in $(seq 1 120); do curl -fsS -H 'Authorization: {key}' {url} >/dev/null 2>&1 && exit 0; sleep 2; done; \+ echo 'timed out waiting for the FusionAuth kickstart'; exit 1",+ key = identity::FUSIONAUTH_API_KEY,);let mut cmd = Command::new("bash");cmd.arg("-lc").arg(script);- stage.run("Waiting for FusionAuth", &mut cmd)+ stage.run("Waiting for FusionAuth (kickstart)", &mut cmd)}diff --git a/tooling/xtask/crates/xtask_local/src/local/gen_compose.rs b/tooling/xtask/crates/xtask_local/src/local/gen_compose.rsindex 455f38b7e4..b6c0e847cd 100644--- a/tooling/xtask/crates/xtask_local/src/local/gen_compose.rs+++ b/tooling/xtask/crates/xtask_local/src/local/gen_compose.rs@@ -53,7 +53,14 @@ pub fn caddyfile_path(instance: &Instance) -> PathBuf {}/// Build the override (typed model), apply the merge tags, and write it.-pub fn generate(mode: Mode, instance: &Instance, binaries: &BinariesDir) -> Result<PathBuf> {+/// `static_frontend` mounts the staged app bundle into the proxy (headless+/// stacks serve the frontend from Caddy instead of a dev server).+pub fn generate(+ mode: Mode,+ instance: &Instance,+ binaries: &BinariesDir,+ static_frontend: bool,+) -> Result<PathBuf> {let mut services: IndexMap<String, Option<dct::Service>> = IndexMap::new();let mounts = binaries.compose_mounts();@@ -80,12 +87,12 @@ pub fn generate(mode: Mode, instance: &Instance, binaries: &BinariesDir) -> Resu// The reverse proxy is the frontend's single origin in every mode, and// LocalStack runs in every mode (dev's `dev_personal` notification queue// lives there too).- add_proxy_service(&mut services, instance);+ add_proxy_service(&mut services, instance, static_frontend);add_localstack_service(&mut services, instance);// The rest of the local infra (FusionAuth, Mailpit, per-instance Postgres/// Redis/OpenSearch port remaps) only for the self-contained local stacks.if mode.spec().runs_local_infra {- add_local_infra(&mut services, instance);+ add_local_infra(&mut services, instance, static_frontend);}let compose = dct::Compose {@@ -138,19 +145,32 @@ fn add_localstack_service(}/// The single-origin reverse proxy (Caddy). Added in every mode — it's how the-/// frontend reaches the local service containers.-fn add_proxy_service(services: &mut IndexMap<String, Option<dct::Service>>, instance: &Instance) {+/// frontend reaches the local service containers. Headless stacks also mount+/// the staged app bundle so Caddy serves the frontend itself (see+/// `proxy::FRONTEND_STATIC`).+fn add_proxy_service(+ services: &mut IndexMap<String, Option<dct::Service>>,+ instance: &Instance,+ static_frontend: bool,+) {let proxy_port = instance.port(Port::Proxy);+ let mut volumes = vec![dct::Volumes::Simple(format!(+ "{}:/etc/caddy/Caddyfile:ro",+ caddyfile_path(instance).display()+ ))];+ if static_frontend {+ volumes.push(dct::Volumes::Simple(format!(+ "{}:/srv/frontend:ro",+ super::frontend::static_dir(instance).display()+ )));+ }services.insert("proxy".to_string(),Some(dct::Service {image: Some(CADDY_IMAGE.to_string()),environment: kv(&[("PROXY_PORT", &proxy_port.to_string())]),ports: dct::Ports::Short(vec![format!("{proxy_port}:{proxy_port}")]),- volumes: vec![dct::Volumes::Simple(format!(- "{}:/etc/caddy/Caddyfile:ro",- caddyfile_path(instance).display()- ))],+ volumes,networks: dct::Networks::Simple(vec!["services".to_string(), "databases".to_string()]),..Default::default()}),@@ -160,7 +180,11 @@ fn add_proxy_service(services: &mut IndexMap<String, Option<dct::Service>>, inst/// Add the local-only companion infra services (FusionAuth override + Mailpit)/// and, for named instances, the remapped infra ports. LocalStack is added/// separately (it runs in every mode), so it is not included here.-fn add_local_infra(services: &mut IndexMap<String, Option<dct::Service>>, instance: &Instance) {+fn add_local_infra(+ services: &mut IndexMap<String, Option<dct::Service>>,+ instance: &Instance,+ static_frontend: bool,+) {// FusionAuth: repoint at the generated kickstart (volumes replaced via tag).let mut fusionauth = dct::Service {environment: kv(&[(@@ -182,15 +206,21 @@ fn add_local_infra(services: &mut IndexMap<String, Option<dct::Service>>, instan}services.insert("fusionauth".to_string(), Some(fusionauth));- // Mailpit (also on `auth` so FusionAuth can deliver passwordless codes).+ // Mailpit is also on `auth` so FusionAuth can deliver passwordless codes.+ // Only headless mode moves its UI under the proxy-friendly /mailpit root;+ // attached run_local preserves the existing direct UI at port 8025.+ let mut mailpit_env = vec![+ ("MP_SMTP_AUTH_ACCEPT_ANY", "1"),+ ("MP_SMTP_AUTH_ALLOW_INSECURE", "1"),+ ];+ if static_frontend {+ mailpit_env.push(("MP_WEBROOT", "/mailpit"));+ }services.insert("mailpit".to_string(),Some(dct::Service {image: Some(MAILPIT_IMAGE.to_string()),- environment: kv(&[- ("MP_SMTP_AUTH_ACCEPT_ANY", "1"),- ("MP_SMTP_AUTH_ALLOW_INSECURE", "1"),- ]),+ environment: kv(&mailpit_env),ports: dct::Ports::Short(vec![format!("{}:1025", instance.port(Port::MailpitSmtp)),format!("{}:8025", instance.port(Port::MailpitUi)),diff --git a/tooling/xtask/crates/xtask_local/src/local/local_env.rs b/tooling/xtask/crates/xtask_local/src/local/local_env.rsindex 007d1d1b24..221d724c45 100644--- a/tooling/xtask/crates/xtask_local/src/local/local_env.rs+++ b/tooling/xtask/crates/xtask_local/src/local/local_env.rs@@ -235,16 +235,24 @@ impl ServiceAuthEnv {/// kickstart (see [`identity`]). The OAuth redirect is the only per-instance bit.struct FusionAuthEnv {oauth_redirect_uri: String,+ /// The auth service's own public origin (`BASE_URL`): OAuth callbacks and+ /// email verification links are built on it. Same host-port convention as+ /// [`identity::oauth_redirect_uri`]. Doppler supplies it for dev; the+ /// code-owned env must too, or a `--no-doppler` stack's auth service dies+ /// at startup on the missing required value.+ base_url: String,}impl FusionAuthEnv {fn for_instance(instance: &Instance) -> Self {FusionAuthEnv {oauth_redirect_uri: identity::oauth_redirect_uri(instance.port(Port::Auth)),+ base_url: format!("http://localhost:{}", instance.port(Port::Auth)),}}fn write(&self, env: &mut BTreeMap<String, String>) {+ env.insert("BASE_URL".into(), self.base_url.clone());env.insert("FUSIONAUTH_BASE_URL".into(),"http://fusionauth:9011".into(),diff --git a/tooling/xtask/crates/xtask_local/src/local/local_env/test.rs b/tooling/xtask/crates/xtask_local/src/local/local_env/test.rsindex 98d254afa6..d3aaa895e8 100644--- a/tooling/xtask/crates/xtask_local/src/local/local_env/test.rs+++ b/tooling/xtask/crates/xtask_local/src/local/local_env/test.rs@@ -15,6 +15,7 @@ fn emits_required_keys() {for key in ["ENVIRONMENT","PORT",+ "BASE_URL","DATABASE_URL","DATABASE_URL_READONLY","REDIS_URI",diff --git a/tooling/xtask/crates/xtask_local/src/local/mailpit.rs b/tooling/xtask/crates/xtask_local/src/local/mailpit.rsindex 5a86bf91b5..24915e3349 100644--- a/tooling/xtask/crates/xtask_local/src/local/mailpit.rs+++ b/tooling/xtask/crates/xtask_local/src/local/mailpit.rs@@ -4,7 +4,12 @@use super::instance::{Instance, Port};-/// The Mailpit web UI URL (host-facing).-pub fn ui_url(instance: &Instance) -> String {+/// The directly published Mailpit web UI used by attached `run_local`.+pub fn direct_ui_url(instance: &Instance) -> String {format!("http://localhost:{}", instance.port(Port::MailpitUi))}++/// The single-origin Mailpit route used by headless stacks and previews.+pub fn proxy_ui_url(instance: &Instance) -> String {+ format!("{}/mailpit/", super::proxy::url(instance))+}diff --git a/tooling/xtask/crates/xtask_local/src/local/mod.rs b/tooling/xtask/crates/xtask_local/src/local/mod.rsindex 39ae39cbe5..540af0be0b 100644--- a/tooling/xtask/crates/xtask_local/src/local/mod.rs+++ b/tooling/xtask/crates/xtask_local/src/local/mod.rs@@ -34,6 +34,8 @@ pub mod mailpit;pub mod opensearch;pub mod proxy;pub mod resources;+pub mod snapshot;+pub mod stack;pub mod stage;pub mod summary;pub mod validate;@@ -152,6 +154,16 @@ use stage::Stage;const AUX_SERVICE_IMAGES: &[&str] = &["websocket_service", "sync_service", "lexical_service"];+/// The artifact-driven preview updater can refresh every Docker-built app+/// service. Keep this separate so enabling local aux rebuilds does not make the+/// existing `run_local` edit loop rebuild the AI worker too.+const PREVIEW_AUX_SERVICE_IMAGES: &[&str] = &[+ "websocket_service",+ "sync_service",+ "lexical_service",+ "ai_editing_worker",+];+/// Bring up a Local or Dev stack and (unless `--no-frontend`) the frontend.pub fn run_stack(mode: Mode, args: &cli::RunArgs) -> Result<()> {if mode.spec().runs_local_infra {@@ -166,6 +178,9 @@ pub fn run_stack(mode: Mode, args: &cli::RunArgs) -> Result<()> {mode.label(),instance.name()));+ if !stage.is_dry_run() {+ stack::clear_state(&instance)?;+ }// `run_local`/`run_dev` are full delete + full create: tear the previous// stack and ALL its stateful volumes down so the bring-up is always from a@@ -183,7 +198,7 @@ pub fn run_stack(mode: Mode, args: &cli::RunArgs) -> Result<()> {// Foreground: resolve env, build binaries + runtime image, generate the// compose override / Caddyfile / kickstart. None of this touches the volumes// or containers the teardown is removing, so it's safe to overlap.- let (env, target) = prepare(&stage, mode, &instance, args)?;+ let (env, target) = prepare(&stage, mode, &instance, args, false)?;// Join the background teardown before we (re)create volumes + bring infra up,// surfaced as a live spinner so it's clear what we're blocked on. It@@ -206,7 +221,7 @@ pub fn run_stack(mode: Mode, args: &cli::RunArgs) -> Result<()> {// start. The teardown means everything is freshly created each run, so// otherwise the services race their backends on startup (no `macrodb`,// DynamoDB/OpenSearch connection refused).- bring_up_infra(&stage, mode, &instance, &env)?;+ bring_up_infra(&stage, mode, &instance, &env, InfraInit::Full)?;bring_up_app(&stage, mode, &instance, &env)?;// No restart-to-reload step: the teardown means `up` always creates fresh@@ -225,7 +240,13 @@ pub fn run_stack(mode: Mode, args: &cli::RunArgs) -> Result<()> {frontend::start(&stage, &instance, mode)?};- summary::print(mode, &instance, &env);+ summary::print(+ mode,+ &instance,+ &env,+ &frontend::url(&instance),+ &mailpit::direct_ui_url(&instance),+ );match frontend {// Interactive terminal: stay attached with a hotkey loop.@@ -430,12 +451,14 @@ fn interact(/// runtime image + service binaries, and generate the compose override //// Caddyfile / kickstart. Deliberately does NOT create the external/// networks/volumes — that's done after the background teardown joins, since-/// teardown removes them. Returns the resolved env + build target.+/// teardown removes them. `static_frontend` wires the proxy to serve the staged+/// app bundle (headless `stack up`). Returns the resolved env + build target.fn prepare(stage: &Stage,mode: Mode,instance: &Instance,args: &cli::RunArgs,+ static_frontend: bool,) -> Result<(env_layer::ResolvedEnv, arch::Target)> {let env = env_layer::resolve(mode,@@ -461,8 +484,8 @@ fn prepare(// through the proxy in every mode), and — for the self-contained local// stacks — the FusionAuth kickstart. (External networks/volumes are created// by the caller after the background teardown joins.)- gen_compose::generate(mode, instance, &binaries)?;- proxy::write_caddyfile(instance, mode)?;+ gen_compose::generate(mode, instance, &binaries, static_frontend)?;+ proxy::write_caddyfile(instance, mode, static_frontend)?;if mode.spec().runs_local_infra {fusionauth::write_kickstart(instance)?;}@@ -500,6 +523,29 @@ fn recreate_aux_service_containers(stage.run("Recreating auxiliary service containers", &mut up)}+fn recreate_preview_aux_service_containers(+ stage: &Stage,+ instance: &Instance,+ env: &env_layer::ResolvedEnv,+) -> Result<()> {+ let mut up = compose_cmd(instance, env);+ up.args(["up", "-d", "--force-recreate", "--no-deps"])+ .args(PREVIEW_AUX_SERVICE_IMAGES);+ stage.run("Recreating preview auxiliary services", &mut up)+}++/// How the local infra reaches its initialized state on bring-up.+#[derive(Clone, Copy, PartialEq, Eq)]+enum InfraInit {+ /// Run the real init: migrate the DB, let FusionAuth apply its kickstart,+ /// create the OpenSearch indices.+ Full,+ /// The volumes were restored from an init snapshot — the state already+ /// exists, so the init steps are skipped (LocalStack is provisioned either+ /// way; its state isn't volume-backed).+ FromSnapshot,+}+/// Bring up the backend infra the app services connect to at startup, and get it/// fully ready before any of them start. The clean-slate teardown means/// everything is freshly created each run, so this ordering is what stops the@@ -511,6 +557,7 @@ fn bring_up_infra(mode: Mode,instance: &Instance,env: &env_layer::ResolvedEnv,+ init: InfraInit,) -> Result<()> {if stage.is_dry_run() {return Ok(());@@ -545,7 +592,7 @@ fn bring_up_infra(localstack::provision(instance)})?;- if spec.migrates_db {+ if spec.migrates_db && init == InfraInit::Full {// Postgres is ready (`--wait`); create + migrate the freshly-wiped DB.db::migrate(stage, instance)?;}@@ -553,11 +600,18 @@ fn bring_up_infra(// Kafka is healthy (`--wait` gates on its broker healthcheck); create// the event topics declared in `macro_event_topics` — the local// equivalent of the MSK topic provisioning driven by the generated- // `.github/kafka-cluster-topics.json`.- stage.run_step("Creating Kafka topics", || kafka::provision(instance))?;+ // `.github/kafka-cluster-topics.json`. Restored volumes already carry+ // the topics (they live in the broker's data dir), so only a full init+ // provisions — which also means a snapshot-restoring `stack up` (e.g.+ // the preview VM) never needs the rdkafka-backed `local-stack` feature.+ if init == InfraInit::Full {+ stage.run_step("Creating Kafka topics", || kafka::provision(instance))?;+ }// Start FusionAuth on its own (impatient healthcheck → no `--wait`) and- // poll it patiently until the kickstart has applied.+ // poll it patiently until it's up. On a full init that wait covers the+ // ~minute kickstart; on a snapshot restore the kickstart is already in+ // the restored volumes, so this is just JVM boot.let mut fa = compose_cmd(instance, env);fa.arg("up").arg("-d").arg("fusionauth");stage.run("Starting FusionAuth", &mut fa)?;@@ -566,7 +620,10 @@ fn bring_up_infra(// OpenSearch is up (`--wait`) but empty. Create the search indices +// aliases (idempotent) so the unified search path works out of the box// instead of 404ing on the missing `documents`/`chats`/… indices.- opensearch::provision_indices(stage, instance, &env.merged)?;+ // Restored volumes already contain them.+ if init == InfraInit::Full {+ opensearch::provision_indices(stage, instance, &env.merged)?;+ }}Ok(())}@@ -743,7 +800,7 @@ pub fn gen_compose_only(args: &cli::InstanceArgs) -> Result<()> {let instance = Instance::derive(args.instance.as_deref(), args.port_base)?;let target = arch::detect()?;let binaries = build::BinariesDir::TargetDir(workspace_root().join(target.debug_dir()));- let path = gen_compose::generate(Mode::Local, &instance, &binaries)?;+ let path = gen_compose::generate(Mode::Local, &instance, &binaries, false)?;println!("{}", path.display());Ok(())}diff --git a/tooling/xtask/crates/xtask_local/src/local/proxy.rs b/tooling/xtask/crates/xtask_local/src/local/proxy.rsindex 6d28de5473..913acd0174 100644--- a/tooling/xtask/crates/xtask_local/src/local/proxy.rs+++ b/tooling/xtask/crates/xtask_local/src/local/proxy.rs@@ -22,29 +22,38 @@ pub fn url(instance: &Instance) -> String {/// Write the instance Caddyfile and return its path. Both local and dev route/// the frontend through this single origin to the local service containers; the/// only difference is the static-file block (dev has no local LocalStack).-pub fn write_caddyfile(instance: &Instance, mode: Mode) -> Result<PathBuf> {+/// With `static_frontend` the proxy also serves the built app bundle at `/app`,+/// making it the one origin for the whole product.+pub fn write_caddyfile(instance: &Instance, mode: Mode, static_frontend: bool) -> Result<PathBuf> {let path = caddyfile_path(instance);if let Some(dir) = path.parent() {std::fs::create_dir_all(dir).with_context(|| format!("creating proxy dir {}", dir.display()))?;}- std::fs::write(&path, caddyfile(mode))+ std::fs::write(&path, caddyfile(mode, static_frontend)).with_context(|| format!("writing {}", path.display()))?;Ok(path)}/// Assemble the Caddyfile: the listener head, the generated per-service routes/// (from the inventory), the special non-inventory routes, the mode's-/// static-file block, then the tail. Service routes are identical across modes-/// (they hit the local containers); only the static-file block differs.-fn caddyfile(mode: Mode) -> String {+/// static-file block, the optional static-frontend block, then the tail.+/// Service routes are identical across modes (they hit the local containers);+/// only the static-file and frontend blocks differ.+fn caddyfile(mode: Mode, static_frontend: bool) -> String {let static_block = if mode.spec().static_files_via_localstack {STATIC_FILE_LOCAL} else {STATIC_FILE_DEV};+ let mailpit_block = if static_frontend && mode.spec().runs_local_infra {+ MAILPIT_ROUTE+ } else {+ ""+ };+ let frontend_block = if static_frontend { FRONTEND_STATIC } else { "" };format!(- "{CADDY_HEAD}{routes}{SPECIAL_ROUTES}{static_block}{CADDY_TAIL}",+ "{CADDY_HEAD}{routes}{SPECIAL_ROUTES}{mailpit_block}{static_block}{frontend_block}{CADDY_TAIL}",routes = service_routes())}@@ -112,6 +121,14 @@ const SPECIAL_ROUTES: &str = r#" @websocket path /websocket /websocket/*uri strip_prefix /syncreverse_proxy sync-service:8787}+"#;++const MAILPIT_ROUTE: &str = r#" # Mailpit serves itself under /mailpit (MP_WEBROOT), so no prefix strip —+ # this is how a headless/preview stack reads its passwordless login codes.+ handle /mailpit/* {+ reverse_proxy mailpit:8025+ }+ redir /mailpit /mailpit/ 308"#;@@ -133,6 +150,20 @@ const STATIC_FILE_DEV: &str = r#" handle_path /static-file/* {}"#;+/// Headless mode: the proxy serves the built app bundle (mounted at+/// `/srv/frontend` — see `gen_compose::add_proxy_service`). The bundle is built+/// with `base: /app`, so URL space `/app/*` maps onto the dist root after the+/// prefix strip; unknown paths fall back to `index.html` (SPA routing). Caddy+/// sorts `redir` before `handle_path`, so the exact-path redirects win first.+const FRONTEND_STATIC: &str = r#" redir / /app/ 302+ redir /app /app/ 308+ handle_path /app/* {+ root * /srv/frontend+ try_files {path} /index.html+ file_server+ }+"#;+const CADDY_TAIL: &str = r#"respond "macro local proxy" 200}diff --git a/tooling/xtask/crates/xtask_local/src/local/proxy/test.rs b/tooling/xtask/crates/xtask_local/src/local/proxy/test.rsindex ed615a7a4d..ac4a2e9e55 100644--- a/tooling/xtask/crates/xtask_local/src/local/proxy/test.rs+++ b/tooling/xtask/crates/xtask_local/src/local/proxy/test.rs@@ -6,7 +6,7 @@ use crate::local::{inventory, repo_root};/// the guarantee that replaces the old hand-maintained route list.#[test]fn generates_a_route_for_every_prefixed_service() {- let caddy = caddyfile(Mode::Local);+ let caddy = caddyfile(Mode::Local, false);for svc in inventory::RUST_SERVICES {let Some(prefix) = svc.path_prefix else {continue;@@ -27,7 +27,7 @@ fn generates_a_route_for_every_prefixed_service() {/// services use `handle_path`.#[test]fn websocket_services_use_a_matcher() {- let caddy = caddyfile(Mode::Local);+ let caddy = caddyfile(Mode::Local, false);// connection-gateway is the inventoried WebSocket service.assert!(caddy.contains("@connection_gateway path /connection-gateway /connection-gateway/*"));assert!(caddy.contains("uri strip_prefix /connection-gateway"));@@ -39,9 +39,9 @@ fn websocket_services_use_a_matcher() {/// fan-out locally, the dev-pointed service in dev.#[test]fn static_file_block_is_mode_specific() {- assert!(caddyfile(Mode::Local).contains("/static-file-storage"));- assert!(caddyfile(Mode::Dev).contains("handle_path /static-file/*"));- assert!(!caddyfile(Mode::Dev).contains("/static-file-storage"));+ assert!(caddyfile(Mode::Local, false).contains("/static-file-storage"));+ assert!(caddyfile(Mode::Dev, false).contains("handle_path /static-file/*"));+ assert!(!caddyfile(Mode::Dev, false).contains("/static-file-storage"));}/// Drift gate across the Rust↔TypeScript seam: every proxied service's prefix@@ -67,3 +67,24 @@ fn frontend_wires_every_inventory_prefix() {);}}++/// The static-frontend block only appears in headless mode, and serves the+/// mounted bundle under `/app` with an SPA fallback. Attached `run_local` keeps+/// the dev server as the frontend origin and must not grow the block.+#[test]+fn static_frontend_block_is_opt_in() {+ let headless = caddyfile(Mode::Local, true);+ assert!(headless.contains("handle_path /app/* {"));+ assert!(headless.contains("root * /srv/frontend"));+ assert!(headless.contains("try_files {path} /index.html"));+ assert!(headless.contains("redir / /app/ 302"));+ assert!(headless.contains("handle /mailpit/*"));++ let attached = caddyfile(Mode::Local, false);+ assert!(!attached.contains("/srv/frontend"));+ assert!(!attached.contains("redir / /app/ 302"));+ assert!(!attached.contains("handle /mailpit/*"));++ let headless_dev = caddyfile(Mode::Dev, true);+ assert!(!headless_dev.contains("handle /mailpit/*"));+}diff --git a/tooling/xtask/crates/xtask_local/src/local/snapshot.rs b/tooling/xtask/crates/xtask_local/src/local/snapshot.rsnew file mode 100644index 0000000000..c25b362c46--- /dev/null+++ b/tooling/xtask/crates/xtask_local/src/local/snapshot.rs@@ -0,0 +1,321 @@+//! Content-addressed snapshots of the stack's *initialized* infra state.+//!+//! A cold bring-up pays for the expensive one-time init — DB migrations, the+//! FusionAuth kickstart (~a minute), OpenSearch index creation. That state is+//! fully determined by a small set of inputs (the migrations, the generated+//! kickstart, the index mappings, the infra image pins, the container+//! platform), so instead of re-running the init on every `stack up`, we key+//! it: hash the inputs, and if+//! a snapshot of the stateful volumes exists under that key, restore it+//! and skip the init entirely. The full-delete/full-create idempotency+//! guarantee survives because the key *is* the definition of "clean" — any+//! change to an input misses the cache and falls back to a real init (which+//! then saves the new snapshot).+//!+//! Snapshots are tarballs of the Docker volumes, written by a throwaway helper+//! container, stored under `infra/local/generated/.snapshots/<key>/` (override+//! with `MACRO_STACK_SNAPSHOT_DIR` — CI bakes this dir into preview images).++use std::path::PathBuf;+use std::process::Command;++use anyhow::{Context, Result, bail};+use serde::{Deserialize, Serialize};+use sha2::{Digest, Sha256};++use super::instance::Instance;+use super::stage::Stage;+use super::{env_layer, fusionauth, gen_compose, repo_root, workspace_root};++/// Bump when the snapshot mechanism itself changes shape (archive layout,+/// volume set) so old snapshots can't be restored into a new scheme.+/// 2: Kafka's volume joined the set (topics live in the broker data dir, so a+/// restore carries them and the rdkafka-backed provisioning is skipped).+/// 3: invalidates snapshots saved while the FusionAuth kickstart could still+/// be mid-flight (the readiness gate only checked `/api/status`, so a save+/// could freeze a tenant-less FusionAuth DB — and the key never changed, so+/// the bad snapshot was sticky).+const FORMAT: u32 = 3;++/// The throwaway container image used to tar/untar volumes. Alpine for its+/// size; only needs `tar` + `sh`.+const HELPER_IMAGE: &str = "alpine:3";++/// The stateful volumes a snapshot captures, as `(archive name, volume)`+/// pairs. Redis is deliberately absent — it's a cache, and an empty cache is+/// valid clean state.+fn archives(instance: &Instance) -> [(&'static str, String); 5] {+ [+ ("postgres.tar.gz", instance.volume_postgres()),+ ("opensearch.tar.gz", instance.volume_opensearch()),+ ("kafka.tar.gz", instance.volume_kafka()),+ ("fusionauth-db.tar.gz", instance.volume_fusionauth_db()),+ (+ "fusionauth-config.tar.gz",+ instance.volume_fusionauth_config(),+ ),+ ]+}++/// The compose services that own those volumes — quiesced (stopped) while the+/// archives are written so the files are consistent.+const STATEFUL_SERVICES: &[&str] = &["postgres", "search", "kafka", "fusionauth", "db"];++/// Where snapshots live. `MACRO_STACK_SNAPSHOT_DIR` overrides for CI bakes and+/// preview images; the default sits inside the gitignored generated dir.+// xtask is host tooling, not a service reading APP_SECRETS_JSON, so reading the+// process environment directly is correct here.+#[allow(clippy::disallowed_methods)]+pub fn root_dir() -> PathBuf {+ std::env::var_os("MACRO_STACK_SNAPSHOT_DIR")+ .map(PathBuf::from)+ .unwrap_or_else(|| repo_root().join("infra/local/generated/.snapshots"))+}++/// What a snapshot directory contains, recorded alongside the archives.+#[derive(Serialize, Deserialize)]+struct Manifest {+ format: u32,+ key: String,+ created_unix: u64,+ archives: Vec<String>,+}++/// A resolved snapshot decision for one instance: the content key and where+/// that key's snapshot would live. `exists()` says which way `up` goes.+pub struct Plan {+ pub key: String,+ pub dir: PathBuf,+}++impl Plan {+ /// Hash the init-defining inputs. Must be called after `prepare` (the+ /// generated kickstart is one of the inputs — it encodes the instance's+ /// ports, so a snapshot can never be restored into an instance whose+ /// FusionAuth config wouldn't match).+ pub fn compute(instance: &Instance) -> Result<Plan> {+ let mut h = Sha256::new();+ h.update(FORMAT.to_le_bytes());++ // The container platform. Volume bytes are written by arch-specific+ // images (an Apple Silicon Postgres data dir is arm64-born), and while+ // Postgres/Lucene data happens to be portable across amd64/arm64 in+ // practice, that's unsupported territory — keying on the platform makes+ // cross-arch restore a structural cache miss instead of a latent bug+ // if snapshots are ever shared between machines (e.g. an S3 cache).+ // Today each snapshot store is machine-local or CI-baked for+ // same-arch Fly machines, so this is insurance, not a behavior change.+ h.update(super::arch::detect()?.docker_platform.as_bytes());++ // Infra image pins + topology.+ for rel in [+ "docker/docker-compose.yml",+ "docker/docker-compose-databases.yml",+ "infra/stacks/fusionauth-instance/docker-compose.yml",+ ] {+ hash_file(&mut h, &repo_root().join(rel))?;+ }+ // The custom local OpenSearch image inputs.+ hash_dir(&mut h, &repo_root().join("infra/local/opensearch"), &[])?;+ // Database schema.+ hash_dir(+ &mut h,+ &workspace_root().join("crates/macro_db_client/migrations"),+ &[],+ )?;+ // Search index mappings (canonical TS helpers; node_modules excluded).+ hash_dir(+ &mut h,+ &repo_root().join("infra/stacks/opensearch/helpers"),+ &["node_modules"],+ )?;+ // The generated FusionAuth kickstart (identity + this instance's ports).+ hash_file(+ &mut h,+ &gen_compose::kickstart_dir(instance).join("kickstart.json"),+ )?;++ let key = hex(&h.finalize());+ let dir = root_dir().join(&key);+ Ok(Plan { key, dir })+ }++ /// Whether a completed snapshot exists for this key (manifest written last,+ /// so its presence implies the archives are whole).+ pub fn exists(&self) -> bool {+ std::fs::read_to_string(self.dir.join("manifest.json"))+ .ok()+ .and_then(|raw| serde_json::from_str::<Manifest>(&raw).ok())+ .is_some_and(|m| m.format == FORMAT && m.key == self.key)+ }++ /// The first 12 hex chars — enough to name it in output.+ pub fn short_key(&self) -> &str {+ &self.key[..12]+ }+}++/// Unpack the snapshot's archives into the instance's (freshly created, empty)+/// volumes. Runs before the infra containers start.+pub fn restore(stage: &Stage, instance: &Instance, plan: &Plan) -> Result<()> {+ let dir = plan.dir.clone();+ let pairs = archives(instance);+ stage.run_step(+ &format!("Restoring init snapshot ({})", plan.short_key()),+ || {+ for (archive, volume) in &pairs {+ if !dir.join(archive).exists() {+ bail!("snapshot {} is missing {archive}", dir.display());+ }+ helper_tar(+ &format!("{volume}:/vol"),+ &format!("{}:/snap:ro", dir.display()),+ &format!("tar xzf /snap/{archive} -C /vol"),+ )?;+ }+ Ok(())+ },+ )+}++/// Archive the just-initialized volumes under the plan's key. Called after the+/// infra init and BEFORE the app services start (nothing is connected yet, so+/// the stateful containers can be stopped for a consistent copy and started+/// again). Written to a temp dir and renamed in, so a crash never leaves a+/// half-snapshot that `exists()` would trust.+pub fn save(+ stage: &Stage,+ instance: &Instance,+ env: &env_layer::ResolvedEnv,+ plan: &Plan,+) -> Result<()> {+ let mut stop = super::compose_cmd(instance, env);+ stop.args(["stop", "-t", "30"]).args(STATEFUL_SERVICES);+ stage.run("Quiescing infra for snapshot", &mut stop)?;++ let tmp = root_dir().join(format!(".tmp-{}", plan.key));+ let pairs = archives(instance);+ let write = || -> Result<()> {+ if tmp.exists() {+ std::fs::remove_dir_all(&tmp)?;+ }+ std::fs::create_dir_all(&tmp)?;+ for (archive, volume) in &pairs {+ helper_tar(+ &format!("{volume}:/vol:ro"),+ &format!("{}:/snap", tmp.display()),+ &format!("tar czf /snap/{archive} -C /vol ."),+ )?;+ }+ let manifest = Manifest {+ format: FORMAT,+ key: plan.key.clone(),+ created_unix: std::time::SystemTime::now()+ .duration_since(std::time::UNIX_EPOCH)+ .map(|d| d.as_secs())+ .unwrap_or(0),+ archives: pairs.iter().map(|(a, _)| a.to_string()).collect(),+ };+ std::fs::write(+ tmp.join("manifest.json"),+ serde_json::to_string_pretty(&manifest)?,+ )?;+ if plan.dir.exists() {+ std::fs::remove_dir_all(&plan.dir)?;+ }+ std::fs::rename(&tmp, &plan.dir).context("moving snapshot into place")?;+ Ok(())+ };+ stage.run_step(+ &format!("Saving init snapshot ({})", plan.short_key()),+ write,+ )?;++ // Bring the quiesced infra back to ready before the app services start.+ let mut up = super::compose_cmd(instance, env);+ up.args(["up", "-d", "--wait", "postgres", "search", "kafka"]);+ stage.run("Restarting infra after snapshot", &mut up)?;+ let mut fa = super::compose_cmd(instance, env);+ fa.args(["up", "-d", "fusionauth"]);+ stage.run("Restarting FusionAuth", &mut fa)?;+ fusionauth::wait_ready(stage, instance)+}++/// Run `tar` in the helper container with two mounts. The stack's own stage+/// plumbing is bypassed on purpose — these run inside `run_step` parents.+fn helper_tar(vol_mount: &str, snap_mount: &str, script: &str) -> Result<()> {+ let out = Command::new("docker")+ .args(["run", "--rm", "-v", vol_mount, "-v", snap_mount])+ .arg(HELPER_IMAGE)+ .args(["sh", "-ceu", script])+ .output()+ .context("running snapshot helper container")?;+ if !out.status.success() {+ bail!(+ "snapshot helper failed ({script}):\n{}",+ String::from_utf8_lossy(&out.stderr)+ );+ }+ Ok(())+}++fn hash_file(h: &mut Sha256, path: &std::path::Path) -> Result<()> {+ let bytes = std::fs::read(path)+ .with_context(|| format!("reading snapshot input {}", path.display()))?;+ h.update(path.file_name().unwrap_or_default().as_encoded_bytes());+ h.update((bytes.len() as u64).to_le_bytes());+ h.update(&bytes);+ Ok(())+}++/// Hash a directory tree deterministically: sorted relative paths + contents.+fn hash_dir(h: &mut Sha256, dir: &std::path::Path, exclude: &[&str]) -> Result<()> {+ let mut files = Vec::new();+ collect_files(dir, dir, exclude, &mut files)?;+ files.sort();+ for rel in files {+ let path = dir.join(&rel);+ let bytes = std::fs::read(&path)+ .with_context(|| format!("reading snapshot input {}", path.display()))?;+ h.update(rel.as_bytes());+ h.update((bytes.len() as u64).to_le_bytes());+ h.update(&bytes);+ }+ Ok(())+}++fn collect_files(+ root: &std::path::Path,+ dir: &std::path::Path,+ exclude: &[&str],+ out: &mut Vec<String>,+) -> Result<()> {+ let entries = std::fs::read_dir(dir)+ .with_context(|| format!("reading snapshot input dir {}", dir.display()))?;+ for entry in entries {+ let entry = entry?;+ let name = entry.file_name().to_string_lossy().to_string();+ if exclude.contains(&name.as_str()) {+ continue;+ }+ let path = entry.path();+ if path.is_dir() {+ collect_files(root, &path, exclude, out)?;+ } else {+ out.push(+ path.strip_prefix(root)+ .expect("child path is under its root")+ .to_string_lossy()+ .to_string(),+ );+ }+ }+ Ok(())+}++fn hex(bytes: &[u8]) -> String {+ bytes.iter().map(|b| format!("{b:02x}")).collect()+}++#[cfg(test)]+mod test;diff --git a/tooling/xtask/crates/xtask_local/src/local/snapshot/test.rs b/tooling/xtask/crates/xtask_local/src/local/snapshot/test.rsnew file mode 100644index 0000000000..38775c42dc--- /dev/null+++ b/tooling/xtask/crates/xtask_local/src/local/snapshot/test.rs@@ -0,0 +1,73 @@+use super::*;+use crate::local::instance::Instance;++/// The key must change when any init-defining input changes, and only then —+/// two computes over identical inputs agree, and the kickstart (which encodes+/// the instance's ports) is part of the key, so snapshots can't cross+/// instances with different port windows.+#[test]+fn key_is_deterministic_and_kickstart_sensitive() {+ let instance = Instance::derive(None, None).unwrap();+ crate::local::fusionauth::write_kickstart(&instance).unwrap();++ let a = Plan::compute(&instance).unwrap();+ let b = Plan::compute(&instance).unwrap();+ assert_eq!(a.key, b.key, "same inputs must produce the same key");+ assert_eq!(a.key.len(), 64, "sha256 hex");++ // A named instance generates a kickstart with different ports → new key.+ let other = Instance::derive(Some("snapshot-key-test"), None).unwrap();+ crate::local::fusionauth::write_kickstart(&other).unwrap();+ let c = Plan::compute(&other).unwrap();+ assert_ne!(+ a.key, c.key,+ "instances with different ports must not share snapshots"+ );++ // Cleanup the test instance's generated dir.+ let _ = std::fs::remove_dir_all(other.artifact_dir());+}++/// `exists()` trusts only a manifest that matches the key and format — a+/// half-written or foreign directory is a cache miss, not a restore.+#[test]+fn exists_requires_a_matching_manifest() {+ let dir = std::env::temp_dir().join("macro-snapshot-exists-test");+ let _ = std::fs::remove_dir_all(&dir);+ std::fs::create_dir_all(&dir).unwrap();++ let plan = Plan {+ key: "k".repeat(64),+ dir: dir.clone(),+ };+ assert!(!plan.exists(), "no manifest → miss");++ std::fs::write(dir.join("manifest.json"), "not json").unwrap();+ assert!(!plan.exists(), "garbage manifest → miss");++ let manifest = Manifest {+ format: FORMAT,+ key: plan.key.clone(),+ created_unix: 0,+ archives: vec![],+ };+ std::fs::write(+ dir.join("manifest.json"),+ serde_json::to_string(&manifest).unwrap(),+ )+ .unwrap();+ assert!(plan.exists(), "matching manifest → hit");++ let mismatched = Manifest {+ key: "x".repeat(64),+ ..manifest+ };+ std::fs::write(+ dir.join("manifest.json"),+ serde_json::to_string(&mismatched).unwrap(),+ )+ .unwrap();+ assert!(!plan.exists(), "key mismatch → miss");++ let _ = std::fs::remove_dir_all(&dir);+}diff --git a/tooling/xtask/crates/xtask_local/src/local/stack.rs b/tooling/xtask/crates/xtask_local/src/local/stack.rsnew file mode 100644index 0000000000..5ad2cadf7d--- /dev/null+++ b/tooling/xtask/crates/xtask_local/src/local/stack.rs@@ -0,0 +1,690 @@+//! Headless stack orchestration: `cargo x stack up|update|status|down`.+//!+//! The preview/agent/CI surface. Same bring-up as `run_local`, but no TTY+//! hotkey loop and no attached dev server: the frontend is a static bundle+//! served by the instance's Caddy, so a finished `up` leaves only Docker+//! containers running — nothing to babysit, and the whole product lives behind+//! the ONE proxy origin that an agent drives with a browser and a preview+//! deploy fronts with a real hostname.+//!+//! `up` is full-delete/full-create like `run_local` (unconditionally+//! idempotent); `update` is the `r`-hotkey as a one-shot verb (rebuild, restart+//! only what changed); `status` is machine-readable state; `down` reclaims+//! everything.++use std::io::{BufReader, Read};+use std::path::{Path, PathBuf};+use std::process::Command;++use anyhow::{Context, Result, bail};+use clap::Args;+use serde::{Deserialize, Serialize};+use serde_json::json;++use super::cli::{EnvArgs, InstanceArgs, RunArgs};+use super::instance::{Instance, Port};+use super::stage::Stage;+use super::{Mode, arch, env_layer, frontend, mailpit, proxy, snapshot, summary};++#[derive(Args, Clone, Default)]+pub struct UpArgs {+ #[command(flatten)]+ pub run: RunArgs,+ /// Stage this prebuilt frontend dist instead of building (CI artifact reuse).+ #[arg(long)]+ pub frontend_dist: Option<PathBuf>,+ /// Neither restore from nor save an init snapshot — always run the full+ /// migrate/kickstart/index init.+ #[arg(long)]+ pub no_snapshot: bool,+ /// Stop after the infra bring-up + init (and the snapshot save/restore):+ /// no app services, proxy, or frontend. This is the CI bake mode — the+ /// app services need the Doppler-sourced env (AWS endpoints, shared+ /// secrets) to even boot, which a bake environment deliberately lacks,+ /// and the init snapshot only captures the infra volumes anyway.+ #[arg(long)]+ pub infra_only: bool,+ /// Print a machine-readable JSON summary as the final line.+ #[arg(long)]+ pub json: bool,+}++#[derive(Args, Clone, Default)]+pub struct SnapshotArgs {+ #[command(flatten)]+ pub instance: InstanceArgs,+ /// Print machine-readable JSON (key, dir, present).+ #[arg(long)]+ pub json: bool,+}++#[derive(Args, Clone, Default)]+pub struct UpdateArgs {+ #[command(flatten)]+ pub instance: InstanceArgs,+ #[command(flatten)]+ pub env: EnvArgs,+ /// Also rebuild + restage the static frontend and recreate the proxy.+ #[arg(long)]+ pub frontend: bool,+ /// Stage this prebuilt frontend dist instead of building (implies --frontend).+ #[arg(long)]+ pub frontend_dist: Option<PathBuf>,+ /// Apply this complete prebuilt binary set instead of invoking Cargo.+ #[arg(long, conflicts_with = "build_aux_services")]+ pub binaries_dir: Option<PathBuf>,+ /// Rebuild the Docker-built auxiliary services (sync, websocket, lexical).+ #[arg(long)]+ pub build_aux_services: bool,+ /// Recreate registry-refreshed preview auxiliary services without building.+ #[arg(long, hide = true, requires = "binaries_dir")]+ pub recreate_aux_services: bool,+ /// Stream subprocess output and show per-step timings.+ #[arg(long, short)]+ pub verbose: bool,+ /// Print a machine-readable summary as the final line.+ #[arg(long, hide = true, requires = "binaries_dir")]+ pub json: bool,+}++#[derive(Args, Clone, Default)]+pub struct StatusArgs {+ #[command(flatten)]+ pub instance: InstanceArgs,+ /// Print machine-readable JSON instead of the human summary.+ #[arg(long)]+ pub json: bool,+}++#[derive(Args, Clone, Default)]+pub struct DownArgs {+ #[command(flatten)]+ pub instance: InstanceArgs,+ /// Stop containers but keep volumes (data survives the next `stack up`...+ /// which wipes them anyway — use `just run_local`/`stop_local` semantics).+ #[arg(long)]+ pub keep_data: bool,+}++/// Durable per-instance record of what `up` brought up, so `update`/`status`+/// don't need the flags repeated. Lives in the instance artifact dir.+#[derive(Serialize, Deserialize)]+struct StackState {+ /// `local` / `dev` — mirrors [`Mode::label`].+ mode: String,+ /// `static` (Caddy serves the bundle) or `none` (`--no-frontend`).+ frontend: String,+ /// Stable directory bind-mounted into every Rust service. Prebuilt updates+ /// replace files here atomically so existing container mounts see them.+ #[serde(default)]+ binaries_dir: Option<PathBuf>,+}++fn state_path(instance: &Instance) -> PathBuf {+ instance.artifact_dir().join("stack.json")+}++fn write_state(instance: &Instance, state: &StackState) -> Result<()> {+ instance.ensure_artifact_dir()?;+ let path = state_path(instance);+ std::fs::write(&path, serde_json::to_string_pretty(state)?)+ .with_context(|| format!("writing {}", path.display()))+}++fn read_state(instance: &Instance) -> Option<StackState> {+ let raw = std::fs::read_to_string(state_path(instance)).ok()?;+ serde_json::from_str(&raw).ok()+}++/// Invalidate headless ownership before another flow replaces this instance.+/// State is written again only after a full headless stack passes its health+/// gate, so failed, infra-only, and interactive replacements cannot inherit it.+pub(super) fn clear_state(instance: &Instance) -> Result<()> {+ let path = state_path(instance);+ match std::fs::remove_file(&path) {+ Ok(()) => Ok(()),+ Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(()),+ Err(e) => Err(e).with_context(|| format!("removing {}", path.display())),+ }+}++/// `cargo x stack up` — bring the whole stack up and return, leaving only+/// Docker containers running.+pub fn up(mode: Mode, args: &UpArgs) -> Result<()> {+ let stage = Stage::from_env_cli(args.run.verbose);+ let instance = Instance::derive(+ args.run.instance.instance.as_deref(),+ args.run.instance.port_base,+ )?;+ stage.section(&format!(+ "macro {} stack (headless) — instance {}",+ mode.label(),+ instance.name()+ ));+ if !stage.is_dry_run() {+ clear_state(&instance)?;+ }++ // Same full-delete/full-create overlap as `run_stack`: tear the previous+ // stack down in the background while the host-side build runs.+ let teardown = (!stage.is_dry_run()).then(|| {+ let instance = instance.clone();+ std::thread::spawn(move || super::teardown_commands(&instance))+ });++ let static_frontend = !args.run.no_frontend && !args.infra_only;+ let (env, target) = super::prepare(&stage, mode, &instance, &args.run, static_frontend)?;+ let configured_binaries = args+ .run+ .build+ .binaries_dir+ .clone()+ .unwrap_or_else(|| super::workspace_root().join(target.debug_dir()));+ let active_binaries = super::build::BinariesDir::classify(&configured_binaries)?+ .host_dir()+ .to_path_buf();++ // Build + stage the frontend bundle in the background: it's pure host-side+ // work, independent of Docker until the proxy container mounts the staged+ // dir — so it overlaps the teardown join and the infra bring-up, and is+ // joined just before `bring_up_app` creates that container. (Dry run:+ // synchronously, so the command preview prints in order.)+ if static_frontend && stage.is_dry_run() {+ frontend::build_static(&stage, &instance, args.frontend_dist.as_deref())?;+ }+ let fe_build = (static_frontend && !stage.is_dry_run()).then(|| {+ let instance = instance.clone();+ let prebuilt = args.frontend_dist.clone();+ std::thread::spawn(move || {+ frontend::build_static(&Stage::from_env().quiet(), &instance, prebuilt.as_deref())+ })+ });++ // The init snapshot decision: hash the init-defining inputs (possible only+ // after `prepare` wrote the kickstart) and check for a stored snapshot.+ // Restores skip migrate/kickstart/index-init; a cold init saves one for+ // next time — that's how the cache seeds itself.+ let snapshot_plan = (!args.no_snapshot && !stage.is_dry_run())+ .then(|| snapshot::Plan::compute(&instance))+ .transpose()?;++ if let Some(handle) = teardown {+ stage.run_step("Tearing down previous stack", move || {+ let _ = handle.join();+ Ok(())+ })?;+ }+ super::ensure_external_resources(&stage, &instance)?;+ match &snapshot_plan {+ Some(plan) if plan.exists() => {+ snapshot::restore(&stage, &instance, plan)?;+ super::bring_up_infra(+ &stage,+ mode,+ &instance,+ &env,+ super::InfraInit::FromSnapshot,+ )?;+ }+ Some(plan) => {+ super::bring_up_infra(&stage, mode, &instance, &env, super::InfraInit::Full)?;+ snapshot::save(&stage, &instance, &env, plan)?;+ }+ None => {+ super::bring_up_infra(&stage, mode, &instance, &env, super::InfraInit::Full)?;+ }+ }+ if let Some(handle) = fe_build {+ stage.run_step("Building frontend (static bundle)", move || {+ handle+ .join()+ .map_err(|_| anyhow::anyhow!("frontend build panicked"))?+ })?;+ }+ if args.infra_only {+ // No app stack, no proxy: nothing to summarize or gate on, and no+ // stack.json — `update`/`status` on an infra-only bake should say+ // "bring the stack up first" rather than assume a running app.+ stage.note(" infra-only: app services, proxy, and frontend skipped");+ if args.json {+ println!(+ "{}",+ serde_json::to_string(&summary_json(mode, &instance, false))?+ );+ }+ return Ok(());+ }+ super::bring_up_app(&stage, mode, &instance, &env)?;++ // Headless "ready" means the backend answers through the proxy — the caller+ // (a CI step, an agent) acts on the URL the moment we return.+ if mode.spec().wait_backend_before_frontend {+ frontend::wait_backend_ready(&stage, &instance)?;+ }++ if !stage.is_dry_run() {+ write_state(+ &instance,+ &StackState {+ mode: mode.label().to_string(),+ frontend: if static_frontend { "static" } else { "none" }.to_string(),+ binaries_dir: Some(active_binaries),+ },+ )?;+ }++ let frontend_url = if static_frontend {+ frontend::static_url(&instance)+ } else {+ "(disabled — --no-frontend)".to_string()+ };+ let mailpit_url = if static_frontend {+ mailpit::proxy_ui_url(&instance)+ } else {+ mailpit::direct_ui_url(&instance)+ };+ summary::print(mode, &instance, &env, &frontend_url, &mailpit_url);+ stage.note(&format!(+ " headless: `just stack status`, `just stack update`, `just stack down`{}",+ instance_suffix(&instance)+ ));+ if args.json {+ println!(+ "{}",+ serde_json::to_string(&summary_json(mode, &instance, static_frontend))?+ );+ }+ Ok(())+}++/// `cargo x stack update` — the `r` hotkey as a one-shot verb: rebuild the+/// binaries (and optionally the frontend bundle) and restart only what changed.+pub fn update(args: &UpdateArgs) -> Result<()> {+ let stage = Stage::from_env_cli(args.verbose);+ let instance = Instance::derive(args.instance.instance.as_deref(), args.instance.port_base)?;+ let Some(state) = read_state(&instance) else {+ bail!(+ "no stack state at {} — bring the stack up first (`just stack up{}`)",+ state_path(&instance).display(),+ instance_suffix(&instance)+ );+ };+ let mode = mode_from_label(&state.mode)?;+ stage.section(&format!(+ "macro {} stack update — instance {}",+ mode.label(),+ instance.name()+ ));++ let env = env_layer::resolve(+ mode,+ &instance,+ args.env.no_doppler,+ args.env.env_file.as_deref(),+ )?;+ let changed = if let Some(source) = args.binaries_dir.as_deref() {+ let changed = apply_prebuilt_binaries(mode, &state, source)?;+ if args.recreate_aux_services {+ super::recreate_preview_aux_service_containers(&stage, &instance, &env)?;+ }+ if !changed.is_empty() {+ super::reload_services(&stage, &instance, &changed)?;+ }+ changed+ } else {+ let target = arch::detect()?;+ super::rebuild_and_reload(+ &stage,+ mode,+ &instance,+ &env,+ target,+ args.build_aux_services,+ )?;+ Vec::new()+ };++ if args.frontend || args.frontend_dist.is_some() {+ reload_static_frontend(+ &stage,+ &instance,+ &env,+ &state,+ args.frontend_dist.as_deref(),+ )?;+ }+ if mode.spec().wait_backend_before_frontend {+ frontend::wait_backend_ready(&stage, &instance)?;+ }+ if args.json {+ println!(+ "{}",+ serde_json::to_string(&json!({+ "changed_services": changed.iter().map(|s| s.compose_name).collect::<Vec<_>>(),+ "frontend_updated": args.frontend || args.frontend_dist.is_some(),+ "aux_services_recreated": args.recreate_aux_services,+ }))?+ );+ }+ Ok(())+}++fn apply_prebuilt_binaries(+ mode: Mode,+ state: &StackState,+ binaries_dir: &Path,+) -> Result<Vec<&'static super::inventory::RustService>> {+ let destination = state.binaries_dir.as_ref().ok_or_else(|| {+ anyhow::anyhow!(+ "stack state predates prebuilt updates — perform a full stack up before updating"+ )+ })?;+ let source = super::build::BinariesDir::classify(binaries_dir)?;+ source.validate(&super::inventory::local_binaries())?;+ let mut changed = Vec::new();+ for service in super::inventory::services_for_mode(mode) {+ if replace_binary_if_changed(source.host_dir(), destination, service.cargo_bin)? {+ changed.push(service);+ }+ }++ Ok(changed)+}++fn reload_static_frontend(+ stage: &Stage,+ instance: &Instance,+ env: &env_layer::ResolvedEnv,+ state: &StackState,+ frontend_dist: Option<&Path>,+) -> Result<()> {+ if state.frontend != "static" {+ bail!(+ "this stack was brought up without a static frontend (--no-frontend); \+ re-run `just stack up` to serve one"+ );+ }+ frontend::build_static(stage, instance, frontend_dist)?;+ // build_static replaces the staged directory, so the container must be+ // recreated to establish a bind mount to the new inode.+ let mut up = super::compose_cmd(instance, env);+ up.args(["up", "-d", "--force-recreate", "--no-deps", "proxy"]);+ stage.run("Recreating proxy (frontend bundle)", &mut up)+}++fn replace_binary_if_changed(+ source_dir: &Path,+ destination_dir: &Path,+ name: &str,+) -> Result<bool> {+ let source = source_dir.join(name);+ let destination = destination_dir.join(name);+ if files_equal(&source, &destination)? {+ return Ok(false);+ }+ std::fs::create_dir_all(destination_dir)+ .with_context(|| format!("creating {}", destination_dir.display()))?;+ let temporary = destination_dir.join(format!(".{name}.hot-update-{}", std::process::id()));+ let _ = std::fs::remove_file(&temporary);+ std::fs::copy(&source, &temporary).with_context(|| {+ format!(+ "copying prebuilt binary {} to {}",+ source.display(),+ temporary.display()+ )+ })?;+ std::fs::rename(&temporary, &destination).with_context(|| {+ format!(+ "atomically replacing {} with {}",+ destination.display(),+ source.display()+ )+ })?;+ Ok(true)+}++fn files_equal(left: &Path, right: &Path) -> Result<bool> {+ let left_meta = std::fs::metadata(left)+ .with_context(|| format!("reading binary metadata for {}", left.display()))?;+ let Ok(right_meta) = std::fs::metadata(right) else {+ return Ok(false);+ };+ if left_meta.len() != right_meta.len() {+ return Ok(false);+ }+ let mut left = BufReader::new(+ std::fs::File::open(left).with_context(|| format!("opening {}", left.display()))?,+ );+ let mut right = BufReader::new(+ std::fs::File::open(right).with_context(|| format!("opening {}", right.display()))?,+ );+ let mut left_buf = [0_u8; 64 * 1024];+ let mut right_buf = [0_u8; 64 * 1024];+ loop {+ let left_read = left.read(&mut left_buf)?;+ let right_read = right.read(&mut right_buf)?;+ if left_read != right_read || left_buf[..left_read] != right_buf[..right_read] {+ return Ok(false);+ }+ if left_read == 0 {+ return Ok(true);+ }+ }+}++/// `cargo x stack status` — container states + health through the proxy, as a+/// human summary or `--json` for machines.+pub fn status(args: &StatusArgs) -> Result<()> {+ let instance = Instance::derive(args.instance.instance.as_deref(), args.instance.port_base)?;+ let state = read_state(&instance);+ let services = compose_ps(&instance)?;+ let running = services.iter().any(|s| s.state == "running");+ let backend_healthy = running && probe(&format!("{}/auth/health", proxy::url(&instance)));+ let static_frontend = state.as_ref().is_some_and(|s| s.frontend == "static");+ let frontend_url = static_frontend.then(|| frontend::static_url(&instance));+ if args.json {+ let mut out = summary_json(+ state.as_ref().map_or(Mode::Local, |s| {+ mode_from_label(&s.mode).unwrap_or(Mode::Local)+ }),+ &instance,+ static_frontend,+ );+ out["running"] = json!(running);+ out["backend_healthy"] = json!(backend_healthy);+ out["services"] = json!(services);+ println!("{}", serde_json::to_string(&out)?);+ return Ok(());+ }++ println!(+ "instance {} (project {}): {}",+ instance.name(),+ instance.project_name(),+ if running { "running" } else { "not running" }+ );+ if running {+ println!(+ " backend {} ({})",+ proxy::url(&instance),+ if backend_healthy {+ "healthy"+ } else {+ "UNHEALTHY"+ }+ );+ if let Some(url) = frontend_url {+ println!(" frontend {url}");+ }+ }+ for svc in &services {+ let health = svc+ .health+ .as_deref()+ .filter(|h| !h.is_empty())+ .map(|h| format!(" ({h})"))+ .unwrap_or_default();+ println!(" {:<32} {}{health}", svc.service, svc.state);+ }+ if services.is_empty() {+ println!(" (no containers)");+ }+ Ok(())+}++/// `cargo x stack down` — reclaim the instance: containers, volumes, networks,+/// and state. `--keep-data` only stops containers.+pub fn down(args: &DownArgs) -> Result<()> {+ let instance = Instance::derive(args.instance.instance.as_deref(), args.instance.port_base)?;+ if args.keep_data {+ return super::stop(&args.instance);+ }+ super::destroy(&args.instance)?;+ clear_state(&instance)?;+ Ok(())+}++/// `cargo x stack snapshot` — report the instance's init-snapshot key and+/// whether a snapshot exists for it. CI uses `--json` to find the directory to+/// bake into preview images.+pub fn snapshot_status(args: &SnapshotArgs) -> Result<()> {+ let instance = Instance::derive(args.instance.instance.as_deref(), args.instance.port_base)?;+ // The key hashes the generated kickstart; (re)write it so the verb works+ // before any `up` has run. Deterministic, so this never changes a key.+ super::fusionauth::write_kickstart(&instance)?;+ let plan = snapshot::Plan::compute(&instance)?;+ if args.json {+ println!(+ "{}",+ serde_json::to_string(&json!({+ "key": plan.key,+ "dir": plan.dir,+ "present": plan.exists(),+ "root": snapshot::root_dir(),+ }))?+ );+ return Ok(());+ }+ println!("key {}", plan.key);+ println!("dir {}", plan.dir.display());+ println!(+ "present {}",+ if plan.exists() {+ "yes"+ } else {+ "no (next cold `stack up` will save it)"+ }+ );+ Ok(())+}++/// The machine-readable endpoint block `up --json` and `status --json` share.+fn summary_json(mode: Mode, instance: &Instance, static_frontend: bool) -> serde_json::Value {+ let mailpit_url = mode.spec().runs_local_infra.then(|| {+ if static_frontend {+ mailpit::proxy_ui_url(instance)+ } else {+ mailpit::direct_ui_url(instance)+ }+ });+ json!({+ "instance": instance.name(),+ "project": instance.project_name(),+ "mode": mode.label(),+ "proxy_url": proxy::url(instance),+ "frontend_url": static_frontend.then(|| frontend::static_url(instance)),+ "fusionauth_url": format!("http://localhost:{}", instance.port(Port::FusionAuth)),+ "mailpit_url": mailpit_url,+ "localstack_url": format!("http://localhost:{}", instance.port(Port::LocalStack)),+ "postgres_url": format!(+ "postgres://user:password@localhost:{}/macrodb",+ instance.port(Port::Postgres)+ ),+ "logs_cmd": format!("docker compose -p {} logs -f", instance.project_name()),+ })+}++/// One row of `docker compose ps`.+#[derive(Serialize)]+struct ServiceStatus {+ service: String,+ state: String,+ health: Option<String>,+}++/// `docker compose -p <project> ps --all --format json`, tolerant of both the+/// NDJSON (v2.21+) and JSON-array output shapes.+fn compose_ps(instance: &Instance) -> Result<Vec<ServiceStatus>> {+ let out = Command::new("docker")+ .args([+ "compose",+ "-p",+ instance.project_name(),+ "ps",+ "--all",+ "--format",+ "json",+ ])+ .output()+ .context("running `docker compose ps`")?;+ if !out.status.success() {+ bail!(+ "`docker compose ps` failed:\n{}",+ String::from_utf8_lossy(&out.stderr)+ );+ }+ let text = String::from_utf8_lossy(&out.stdout);+ let values: Vec<serde_json::Value> = if text.trim_start().starts_with('[') {+ serde_json::from_str(text.trim()).context("parsing compose ps json array")?+ } else {+ text.lines()+ .filter(|l| !l.trim().is_empty())+ .map(serde_json::from_str)+ .collect::<std::result::Result<_, _>>()+ .context("parsing compose ps ndjson")?+ };+ Ok(values+ .into_iter()+ .map(|v| ServiceStatus {+ service: str_field(&v, "Service"),+ state: str_field(&v, "State"),+ health: v.get("Health").and_then(|h| h.as_str()).map(str::to_string),+ })+ .collect())+}++fn str_field(v: &serde_json::Value, key: &str) -> String {+ v.get(key)+ .and_then(|s| s.as_str())+ .unwrap_or_default()+ .to_string()+}++/// One quick `curl` probe (the tooling already leans on curl for readiness).+fn probe(url: &str) -> bool {+ Command::new("curl")+ .args(["-fsS", "-m", "3", "-o", "/dev/null", url])+ .status()+ .map(|s| s.success())+ .unwrap_or(false)+}++fn mode_from_label(label: &str) -> Result<Mode> {+ match label {+ "local" => Ok(Mode::Local),+ "dev" => Ok(Mode::Dev),+ other => bail!("unknown mode '{other}' in stack.json"),+ }+}++/// The `--instance <name>` suffix for copy-pasteable hints ("" for the default).+fn instance_suffix(instance: &Instance) -> String {+ if instance.is_default() {+ String::new()+ } else {+ format!(" --instance {}", instance.name())+ }+}++#[cfg(test)]+mod test;diff --git a/tooling/xtask/crates/xtask_local/src/local/stack/test.rs b/tooling/xtask/crates/xtask_local/src/local/stack/test.rsnew file mode 100644index 0000000000..0d948adee0--- /dev/null+++ b/tooling/xtask/crates/xtask_local/src/local/stack/test.rs@@ -0,0 +1,61 @@+use super::*;++/// `up` writes stack.json and `update`/`status` read it back — the record and+/// the mode labels must stay in agreement.+#[test]+fn stack_state_roundtrips() {+ let state = StackState {+ mode: "local".to_string(),+ frontend: "static".to_string(),+ binaries_dir: Some(PathBuf::from("/tmp/binaries")),+ };+ let json = serde_json::to_string(&state).unwrap();+ let back: StackState = serde_json::from_str(&json).unwrap();+ assert_eq!(back.mode, "local");+ assert_eq!(back.frontend, "static");+ assert_eq!(back.binaries_dir, Some(PathBuf::from("/tmp/binaries")));+ assert!(mode_from_label(&back.mode).is_ok());+ assert!(mode_from_label("nonsense").is_err());+}++#[test]+fn legacy_stack_state_has_no_binaries_dir() {+ let state: StackState =+ serde_json::from_str(r#"{"mode":"local","frontend":"static"}"#).unwrap();+ assert_eq!(state.binaries_dir, None);+}++#[test]+fn clearing_state_invalidates_a_previous_headless_stack() {+ let instance =+ Instance::derive(Some(&format!("state-clear-{}", std::process::id())), None).unwrap();+ write_state(+ &instance,+ &StackState {+ mode: "local".to_string(),+ frontend: "static".to_string(),+ binaries_dir: None,+ },+ )+ .unwrap();+ assert!(read_state(&instance).is_some());+ clear_state(&instance).unwrap();+ assert!(read_state(&instance).is_none());+ clear_state(&instance).unwrap();+ let _ = std::fs::remove_dir_all(instance.artifact_dir());+}++#[test]+fn compares_binary_contents_not_just_size() {+ let root = std::env::temp_dir().join(format!("macro-stack-test-{}", std::process::id()));+ let _ = std::fs::remove_dir_all(&root);+ std::fs::create_dir_all(&root).unwrap();+ let left = root.join("left");+ let right = root.join("right");+ std::fs::write(&left, b"same").unwrap();+ std::fs::write(&right, b"same").unwrap();+ assert!(files_equal(&left, &right).unwrap());+ std::fs::write(&right, b"diff").unwrap();+ assert!(!files_equal(&left, &right).unwrap());+ std::fs::remove_dir_all(root).unwrap();+}diff --git a/tooling/xtask/crates/xtask_local/src/local/summary.rs b/tooling/xtask/crates/xtask_local/src/local/summary.rsindex 84c2c963eb..9751f21c94 100644--- a/tooling/xtask/crates/xtask_local/src/local/summary.rs+++ b/tooling/xtask/crates/xtask_local/src/local/summary.rs@@ -4,10 +4,19 @@ use console::Style;use super::env_layer::ResolvedEnv;use super::instance::{Instance, Port};-use super::{Mode, frontend, mailpit, proxy};+use super::{Mode, proxy};/// Print the mode/instance/endpoints block after a successful startup.-pub fn print(mode: Mode, instance: &Instance, env: &ResolvedEnv) {+/// `frontend_url` differs by flow: the dev-server origin for `run_local`, the+/// proxy-served bundle for headless `stack up`; `mailpit_url` follows the same+/// direct-versus-single-origin distinction.+pub fn print(+ mode: Mode,+ instance: &Instance,+ env: &ResolvedEnv,+ frontend_url: &str,+ mailpit_url: &str,+) {let key = Style::new().dim();let link = Style::new().cyan();let row = |k: &str, v: String| {@@ -51,7 +60,7 @@ pub fn print(mode: Mode, instance: &Instance, env: &ResolvedEnv) {.unwrap_or_else(|| "(none)".into()),);row("generated env", env.generated_path.display().to_string());- row("frontend", frontend::url(instance));+ row("frontend", frontend_url.to_string());if mode.spec().show_infra_in_summary {row("proxy", proxy::url(instance));@@ -59,7 +68,7 @@ pub fn print(mode: Mode, instance: &Instance, env: &ResolvedEnv) {"fusionauth",format!("http://localhost:{}", instance.port(Port::FusionAuth)),);- row("mailpit", mailpit::ui_url(instance));+ row("mailpit", mailpit_url.to_string());row("localstack",format!("http://localhost:{}", instance.port(Port::LocalStack)),diff --git a/tooling/xtask/crates/xtask_local/src/local/validate.rs b/tooling/xtask/crates/xtask_local/src/local/validate.rsindex 81e72b9afe..eb38524562 100644--- a/tooling/xtask/crates/xtask_local/src/local/validate.rs+++ b/tooling/xtask/crates/xtask_local/src/local/validate.rs@@ -30,7 +30,7 @@ pub fn local_compose(instance: &Instance, mode: Mode) -> Result<()> {// target dir — config rendering does not need the binaries to exist.let target = arch::detect()?;let binaries = BinariesDir::TargetDir(workspace_root().join(target.debug_dir()));- gen_compose::generate(mode, instance, &binaries)?;+ gen_compose::generate(mode, instance, &binaries, false)?;let resolved = env_layer::resolve(mode, instance, true, None)?;let files = gen_compose::compose_files(instance);diff --git a/tooling/xtask/crates/xtask_paths/src/lib.rs b/tooling/xtask/crates/xtask_paths/src/lib.rsindex 572c4a01bd..c2452fd7b5 100644--- a/tooling/xtask/crates/xtask_paths/src/lib.rs+++ b/tooling/xtask/crates/xtask_paths/src/lib.rs@@ -1,7 +1,7 @@#![deny(missing_docs)]//! Shared path discovery for the xtask command crates.//!-//! Every xtask command needs to locate the repository Rust workspace root (to+//! Every xtask command needs to locate the repository Cargo workspace root (to//! anchor `cargo metadata`, read member manifests, spawn `wasm-pack`, …) and//! the repository root (where `docker/`, `infra/`, `apps/`, and the root//! `justfile` live). Both are derived from this crate's own@@ -19,8 +19,14 @@ use std::path::{Path, PathBuf};/// The repository Cargo workspace root.////// `env!("CARGO_MANIFEST_DIR")` expands to this crate's directory, which is a-/// fixed location, so the depth here does not depend on the caller.+/// fixed location, so the depth here does not depend on the caller. Packaged+/// xtask binaries can set `MACRO_REPO_ROOT` when they run against a staged copy+/// of the repository rather than the checkout where they were compiled.pub fn workspace_root() -> PathBuf {+ if let Some(root) = repo_root_override() {+ return root;+ }+// <workspace>/tooling/xtask/crates/xtask_paths -> nth(4) == <workspace>.Path::new(env!("CARGO_MANIFEST_DIR")).ancestors()@@ -34,6 +40,15 @@ pub fn repo_root() -> PathBuf {workspace_root()}+/// Read the runtime repository root used by packaged xtask binaries.+///+/// Environment variables are appropriate here because this tooling crate runs+/// both inside and outside the service runtime, including in the preview VM.+#[allow(clippy::disallowed_methods)]+fn repo_root_override() -> Option<PathBuf> {+ std::env::var_os("MACRO_REPO_ROOT").map(PathBuf::from)+}+/// A UTF-8, repository-relative path that must resolve to a regular file.#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)]pub struct RepoFile<'a>(&'a str);diff --git a/tooling/xtask/crates/xtask_workflows/src/workflows/mod.rs b/tooling/xtask/crates/xtask_workflows/src/workflows/mod.rsindex 12f3fc4bf9..f37a0694ce 100644--- a/tooling/xtask/crates/xtask_workflows/src/workflows/mod.rs+++ b/tooling/xtask/crates/xtask_workflows/src/workflows/mod.rs@@ -28,6 +28,7 @@ mod deploy_preview;mod deploy_web_app;mod deploy_web_app_dev_push;mod path_validation;+mod preview_fly;mod pulumi_preview_pr;mod reusable_preview_service;mod runners;@@ -202,6 +203,16 @@ const WORKFLOWS: &[WorkflowFile] = &[)},},+ WorkflowFile {+ slug: "preview_fly",+ file_name: "preview-fly.yml",+ render_yaml: || render_gh_workflow(preview_fly::preview_fly)(),+ },+ WorkflowFile {+ slug: "preview_fly",+ file_name: "preview-fly-cleanup.yml",+ render_yaml: || render_gh_workflow(preview_fly::preview_fly_cleanup)(),+ },WorkflowFile {slug: "check_node_modules_nix",file_name: "check_node_modules_nix.yml",diff --git a/tooling/xtask/crates/xtask_workflows/src/workflows/preview_fly.rs b/tooling/xtask/crates/xtask_workflows/src/workflows/preview_fly.rsnew file mode 100644index 0000000000..086f8be3cb--- /dev/null+++ b/tooling/xtask/crates/xtask_workflows/src/workflows/preview_fly.rs@@ -0,0 +1,635 @@+//! `Fly Preview` — full-stack preview deploys on Fly Machines, plus the+//! cleanup workflow that destroys the app when the PR closes.+//!+//! Opt-in via the `preview` label. The deploy job builds every artifact on the+//! runner (service binaries, same-origin frontend bundle, init snapshot),+//! mirrors every stack image into the per-PR app's Fly registry repo, stages+//! the rest into a slim VM image (`infra/preview/`), and deploys it as the+//! per-PR Fly app `macro-pr-<N>` — which suspends when idle and wakes on+//! request. The machine pulls the stack images at boot (layer-level dedup+//! against its persistent /var/lib/docker volume) instead of shipping them+//! inside the VM image. Runtime secrets reach the machine as a config-scoped+//! Doppler service token (`DOPPLER_PREVIEW_TOKEN` → Fly secret+//! `DOPPLER_TOKEN`), which the stack's env layer pulls at boot.+//!+//! NOTE: never a required status check — previews are opt-in by design.++use gh_workflow::{+ Concurrency, Event, Expression, Job, Level, Permissions, PullRequest, PullRequestType, Run,+ Step, Use, Workflow,+};++use crate::workflows::{runners, steps, vars};++/// The per-PR Fly app name; also the preview hostname (`<app>.fly.dev`).+const APP_NAME: &str = "macro-pr-${{ github.event.pull_request.number }}";++/// Build the deploy workflow.+pub fn preview_fly() -> Workflow {+ Workflow::new("Fly Preview")+ .on(Event::default().pull_request(+ PullRequest::default()+ .add_type(PullRequestType::Opened)+ .add_type(PullRequestType::Reopened)+ .add_type(PullRequestType::Synchronize)+ .add_type(PullRequestType::Labeled),+ ))+ .concurrency(+ Concurrency::new(Expression::new(+ "fly-preview-${{ github.event.pull_request.number }}",+ ))+ .cancel_in_progress(true),+ )+ .add_job("deploy", deploy())+}++/// Build the cleanup workflow (destroys the app on close / label removal).+pub fn preview_fly_cleanup() -> Workflow {+ Workflow::new("Fly Preview Cleanup")+ .on(Event::default().pull_request(+ PullRequest::default()+ .add_type(PullRequestType::Closed)+ .add_type(PullRequestType::Unlabeled),+ ))+ .add_job("destroy", destroy())+}++fn deploy() -> Job {+ Job::default()+ // Same-repo branches only: the preview image embeds CI-built artifacts+ // and the app serves under our Fly org — never build it from fork code.+ .cond(Expression::new(+ "github.event.pull_request.head.repo.full_name == github.repository && \+ contains(github.event.pull_request.labels.*.name, 'preview')",+ ))+ // Dedicated cache volume pool (nix store + cargo target + snapshots):+ // see PREVIEW_CACHE_TAG for why sharing the check/test jobs' pool was+ // measured cold on both layers (different --target = disjoint sccache+ // keys, and their volumes never carry a cargo target dir).+ .runs_on(runners::Runner::RustCi.with_cache_tag(vars::PREVIEW_CACHE_TAG))+ .permissions(+ Permissions::default()+ .contents(Level::Read)+ .pull_requests(Level::Write),+ )+ .add_env(("FLY_API_TOKEN", vars::FLY_API_TOKEN))+ .add_env(("APP_NAME", APP_NAME))+ // Init snapshots live on the cache volume: an unchanged key skips the+ // whole infra bake and keeps snapshot bytes (→ VM image layer) stable.+ .add_env((+ "MACRO_STACK_SNAPSHOT_DIR",+ vars::PREVIEW_SNAPSHOT_VOLUME_DIR,+ ))+ // No incremental in CI: incremental caches are what bloated the cache+ // volume to 41G (making it the workspace's eviction victim — observed+ // volume loss between runs), and sccache refuses to cache incremental+ // compiles, so they also locked workspace crates out of sccache.+ // Cargo's fingerprint reuse (what makes warm runs fast) is unaffected.+ // Local run_local keeps incremental for the edit-rebuild loop.+ .add_env(("CARGO_INCREMENTAL", "0"))+ .add_step(steps::checkout(false, true))+ .add_step(steps::mount_cache_volume_with_cargo_target())+ // CARGO_INCREMENTAL=0 stops incremental caches being written, but the+ // ~25G already on older volumes never gets cleaned by cargo — it's+ // exactly the weight that made this pool the workspace quota's+ // eviction victim. Deleting is idempotent and instant when clean.+ .add_step(+ Step::new("Prune stale incremental caches")+ .run("rm -rf target/*/debug/incremental target/debug/incremental"),+ )+ // Namespace remote builder: persistent BuildKit layer cache across+ // runs, same as the deploy workflows use. The aux-image builds and the+ // preview-image build both go through it.+ .add_step(+ Step::new("Set up Namespace Docker builder").uses(+ "namespacelabs",+ "nscloud-setup-buildx-action",+ "d059ed7184f0bc7c8b27e8810cea153d02bcc6dd",+ ), // v0.0.23+ )+ // Fail fast: the Docker-built images are the most fragile part of a+ // fresh bring-up (stale local images mask their rot) and need nothing+ // from nix/cargo — build them before the expensive toolchain setup so+ // a broken Dockerfile fails in ~2 minutes, not ~12. This is also the+ // ONLY thing that builds them now: the infra-only bake never starts+ // the app layer, so the registry mirror ships exactly what's built here.+ .add_step(Step::new("Build compose service images (fail fast)").run(+ "docker compose --project-directory . -p macro -f docker/docker-compose.yml build \+ search sync_service websocket_service lexical_service ai_editing_worker",+ ))+ .add_step(steps::setup_nix())+ .add_step(steps::setup_reqs_web("Setup dev shell + web deps", false))+ .add_step(steps::configure_namespace_sccache(+ vars::PREVIEW_SCCACHE_NAME,+ ))+ .add_step(Step::new("Build service binaries").run(indoc::indoc! {r#"+ cargo x zigbuild+ sccache --show-stats+ "#}))+ .add_step(+ // xtask_local, not the dependency-free `xtask` launcher (which+ // just re-invokes cargo — useless in the VM). Deliberately WITHOUT+ // the `local-stack` feature: the VM restores the baked snapshot,+ // whose Kafka volume already carries the topics, so it never+ // provisions Kafka — and skipping rdkafka means no+ // dynamically-linked librdkafka to ship into the VM. zigbuild,+ // not a host build: the nix dev shell links the /nix/store ELF+ // interpreter, which doesn't exist in the VM image (exit 127,+ // "required file not found").+ Step::new("Build xtask_local (runs inside the preview VM)").run(+ "cargo zigbuild --release --target x86_64-unknown-linux-gnu.2.36 -p xtask_local",+ ),+ )+ .add_step(+ Step::new("Build frontend bundle (same-origin)")+ .run("bun run --bun build")+ .working_directory(xtask_paths::repo_dir!("apps/web"))+ .add_env(("MODE", "development"))+ .add_env(("NODE_ENV", "production"))+ .add_env(("VITE_LOCAL_SERVERS", "ALL"))+ .add_env(("VITE_LOCAL_BACKEND_ORIGIN", "same-origin")),+ )+ .add_step(bake_snapshot())+ .add_step(dump_stack_diagnostics())+ .add_step(stage_context())+ .add_step(setup_flyctl())+ .add_step(deploy_to_fly())+ .add_step(dump_fly_diagnostics())+ .add_step(dump_boot_timings())+ .add_step(comment_preview_url())+}++/// When the machine never passes its health check, the answer is inside the+/// VM (dockerd, image load, `stack up`) — dump its logs and state so a failed+/// deploy is diagnosable from CI output alone.+fn dump_fly_diagnostics() -> Step<Run> {+ Step::new("Dump Fly diagnostics")+ .run(indoc::indoc! {r#"+ flyctl machine list --app "$APP_NAME" || true+ flyctl logs --app "$APP_NAME" --no-tail || true+ "#})+ .if_condition(Expression::new("failure()"))+}++/// The entrypoint logs how long each boot phase took (image pulls, stack up),+/// but Fly's log retention is short and nobody looks at a healthy machine —+/// surface the `[preview]` lines in CI so every deploy records its boot+/// breakdown next to the deploy timings.+fn dump_boot_timings() -> Step<Run> {+ Step::new("Dump boot timings")+ .run(r#"flyctl logs --app "$APP_NAME" --no-tail | grep -E '\[preview\]|✓' || true"#)+}++/// A cold `stack up --infra-only` on the runner runs the real init (migrate,+/// kickstart, Kafka topics, indices) and saves the content-addressed snapshot+/// the VM restores from. Infra only: the app services need the Doppler-sourced+/// env to boot, which this runner deliberately lacks — the snapshot captures+/// only the infra volumes anyway. Via `just` (not the bare `cargo x` alias)+/// because the justfile enables the `local-stack` feature the Kafka+/// provisioning needs.+///+/// The store lives on the cache volume (MACRO_STACK_SNAPSHOT_DIR) for a+/// zero-copy hit. Every content-addressed snapshot is also stored as one tar in+/// Namespace artifact storage, keyed by the same hash. Cache-volume onboarding+/// misses therefore download the exact snapshot instead of paying for a full+/// init again. Artifact access is an optimization: download/upload failures+/// fall back to the existing volume/cold-bake behavior.+///+/// The final `snapshot --json` line lands in $RUNNER_TEMP for the stage step,+/// which ships exactly that key's directory.+fn bake_snapshot() -> Step<Run> {+ Step::new("Bake init snapshot").run(indoc::indoc! {r#"+ set -euo pipefail+ mkdir -p "$MACRO_STACK_SNAPSHOT_DIR"+ status=$(just stack snapshot --json | tail -n 1)+ key=$(echo "$status" | jq -r '.key')+ root=$(echo "$status" | jq -r '.root')+ artifact="macro-preview/init-snapshots/${key}.tar"+ archive="$RUNNER_TEMP/init-snapshot-${key}.tar"+ artifact_hit=++ # The Namespace cache-volume pool has an onboarding period where a+ # runner can receive an empty fork. Artifact storage is the durable+ # fallback: restore into a temporary directory and only publish it to+ # the live store after validating the embedded manifest through xtask.+ if ! echo "$status" | jq -e '.present' >/dev/null; then+ restored="$RUNNER_TEMP/restored-init-snapshot"+ rm -rf "$restored"+ mkdir -p "$restored"+ if nsc artifact download "$artifact" "$archive" \+ && tar -xf "$archive" -C "$restored" \+ && [ -f "$restored/$key/manifest.json" ]; then+ rm -rf "$root/$key"+ cp -a "$restored/$key" "$root/$key"+ status=$(just stack snapshot --json | tail -n 1)+ if echo "$status" | jq -e '.present' >/dev/null; then+ artifact_hit=1+ echo "init snapshot artifact hit: $status"+ else+ rm -rf "$root/$key"+ echo "downloaded init snapshot failed validation; baking cold" >&2+ fi+ else+ echo "init snapshot artifact miss; baking cold" >&2+ fi+ fi++ if echo "$status" | jq -e '.present' >/dev/null; then+ echo "init snapshot cache hit: $status"+ cargo run --quiet --manifest-path Cargo.toml \+ -p xtask_local --features local-stack -- gen-compose+ # The cold path builds the runtime image inside `stack up`; on a+ # hit nothing else does, and the registry mirror needs it in the+ # local daemon (its pull fallback only covers public images —+ # macro-local-runtime:dev is not on Docker Hub).+ cargo run --quiet --manifest-path Cargo.toml \+ -p xtask_local --features local-stack -- runtime-image+ else+ just stack up --infra-only --no-doppler --no-build+ status=$(just stack snapshot --json | tail -n 1)+ echo "$status" | jq -e '.present' >/dev/null+ fi++ # Seed artifact storage from either a volume hit or a cold bake. Avoid+ # uploading another version when the artifact already exists; a race+ # between concurrent PRs is harmless and must not fail the preview.+ if [ -z "$artifact_hit" ] && ! nsc artifact describe "$artifact" >/dev/null 2>&1; then+ snap_dir=$(echo "$status" | jq -r '.dir')+ tar -cf "$archive" -C "$root" "$(basename "$snap_dir")"+ nsc artifact upload "$archive" "$artifact" --expires_in 720h \+ || echo "warning: failed to upload init snapshot artifact" >&2+ fi+ echo "$status" > "$RUNNER_TEMP/preview-snapshot.json"+ "#})+}++/// When the bake dies (typically the backend health gate), the answer is in+/// the service logs — dump container states and recent logs so a failed run is+/// diagnosable from CI output alone.+fn dump_stack_diagnostics() -> Step<Run> {+ Step::new("Dump stack diagnostics")+ .run(indoc::indoc! {r#"+ docker compose -p macro ps --all || true+ for svc in authentication-service proxy fusionauth postgres kafka \+ connection_gateway document_storage_service email_service \+ notification_service contacts_service; do+ echo "==================== $svc ===================="+ docker compose -p macro logs --no-color --tail 80 "$svc" 2>&1 || true+ done+ "#})+ .if_condition(Expression::new("failure()"))+}++fn stage_context() -> Step<Run> {+ Step::new("Stage preview build context").run(indoc::indoc! {r#"+ set -euo pipefail+ # The dev-shell env (BASH_ENV) exports a Nix LD_LIBRARY_PATH; host+ # binaries like rsync then resolve Nix-store libs whose deps drag in+ # Nix glibc mid-process and crash on glibc symbol versions. This step+ # is pure file shuffling — drop it.+ unset LD_LIBRARY_PATH+ ctx=preview-ctx+ rm -rf "$ctx"+ mkdir -p "$ctx/repo" "$ctx/artifacts/binaries" "$ctx/bin"+ # The minimal repo layout xtask reads at runtime — including every+ # snapshot-key input, byte-identical to this checkout, so the VM's+ # key matches the snapshot baked above.+ rsync -a --relative --exclude node_modules \+ docker/docker-compose.yml \+ docker/docker-compose-databases.yml \+ infra/stacks/fusionauth-instance/docker-compose.yml \+ infra/local/nginx \+ infra/local/opensearch \+ infra/stacks/opensearch/helpers \+ crates/macro_db_client/migrations \+ "$ctx/repo/"+ # Service binaries only — the target dir also holds gigabytes of+ # build intermediates. `-p` everywhere: preserved mtimes mean files+ # cargo didn't relink produce byte-identical Docker layers, so the+ # registry push/pull skips them (the Dockerfile orders its COPYs+ # stable → volatile for the same reason).+ find target/x86_64-unknown-linux-gnu/debug \+ -maxdepth 1 -type f -executable ! -name '*.d' ! -name '*.so' \+ -exec cp -p {} "$ctx/artifacts/binaries/" \;+ cp -a apps/web/dist "$ctx/artifacts/frontend-dist"+ # Only the current key's snapshot — the volume store accumulates keys.+ snap_dir=$(jq -r '.dir' "$RUNNER_TEMP/preview-snapshot.json")+ mkdir -p "$ctx/artifacts/snapshots"+ cp -a "$snap_dir" "$ctx/artifacts/snapshots/"+ snapshot_key=$(jq -r '.key' "$RUNNER_TEMP/preview-snapshot.json")+ runtime_key=$(cd "$ctx/repo" \+ && find . -type f -print0 \+ | sort -z \+ | xargs -0 sha256sum \+ | sha256sum \+ | cut -d ' ' -f 1)+ frontend_key=$(cd "$ctx/artifacts/frontend-dist" \+ && find . -type f -print0 \+ | sort -z \+ | xargs -0 sha256sum \+ | sha256sum \+ | cut -d ' ' -f 1)+ jq -n \+ --arg snapshot_key "$snapshot_key" \+ --arg runtime_key "$runtime_key" \+ --arg frontend_key "$frontend_key" \+ --arg commit "${{ github.sha }}" \+ '{format: 1, snapshot_key: $snapshot_key, runtime_key: $runtime_key, frontend_key: $frontend_key, commit: $commit}' \+ > "$ctx/deployment.json"+ cp -p target/x86_64-unknown-linux-gnu/release/xtask_local "$ctx/bin/xtask"+ cp -p infra/preview/hot-update.sh "$ctx/bin/hot-update"+ cp -p infra/preview/entrypoint.sh "$ctx/entrypoint.sh"+ cp -p infra/preview/Dockerfile "$ctx/Dockerfile"+ cp -p infra/preview/update.Dockerfile "$ctx/update.Dockerfile"+ "#})+}++fn setup_flyctl() -> Step<Use> {+ Step::new("Setup flyctl").uses(+ "superfly",+ "flyctl-actions/setup-flyctl",+ "ed8efb33836e8b2096c7fd3ba1c8afe303ebbff1",+ ) // 1.6+}++/// Select bootstrap / hot update / rehydrate from the running machine's durable+/// compatibility marker. All modes mirror content-addressed stack images. A+/// compatible machine pulls a scratch artifact carrier and applies it in place;+/// an absent, incompatible, or failed hot update falls through to the immutable+/// full-image deploy and snapshot restore.+fn deploy_to_fly() -> Step<Run> {+ Step::new("Deploy to Fly")+ .run(indoc::indoc! {r#"+ set -euo pipefail+ if [ -z "$DOPPLER_PREVIEW_TOKEN" ]; then+ echo "DOPPLER_PREVIEW_TOKEN secret is not set (a Doppler service token scoped to the preview config)" >&2+ exit 1+ fi+ if [ -z "$FLY_ORG" ]; then+ echo "FLY_ORG is not set (repo variable or secret with the Fly org slug)" >&2+ exit 1+ fi++ app_existed=+ if flyctl status --app "$APP_NAME" >/dev/null 2>&1; then+ app_existed=1+ else+ # A transient status error must not turn an existing app into a+ # hard create failure: create, then prove the app is reachable.+ flyctl apps create "$APP_NAME" --org "$FLY_ORG" \+ || flyctl status --app "$APP_NAME" >/dev/null+ fi++ # A read-only, app-scoped, time-boxed pull token so the machine's+ # inner dockerd can pull the mirrored stack images from this app's+ # registry repo — and nothing else (PR code can read machine+ # secrets, so scope matters). The org deploy token can't mint new+ # tokens (createLimitedAccessToken is not authorized), but macaroon+ # attenuation is pure client-side crypto: append an Apps caveat+ # (numeric app id, mask "r") and a validity window to our own token.+ app_id=$(curl -sf https://api.fly.io/graphql \+ -H "Authorization: Bearer $FLY_API_TOKEN" \+ -H "content-type: application/json" \+ -d "{\"query\":\"{ app(name: \\\"$APP_NAME\\\") { internalNumericId } }\"}" \+ | jq -re '.data.app.internalNumericId')+ now=$(date +%s)+ pull_token=$(printf '[{"type":"Apps","body":{"apps":{"%s":"r"}}},{"type":"ValidityWindow","body":{"not_before":%d,"not_after":%d}}]' \+ "$app_id" $((now - 60)) $((now + 604800)) | flyctl tokens attenuate)++ snapshot_key=$(jq -r '.key' "$RUNNER_TEMP/preview-snapshot.json")+ runtime_key=$(jq -r '.runtime_key' preview-ctx/deployment.json)+ mode=bootstrap+ machine_id=+ if [ -n "$app_existed" ]; then+ mode=rehydrate+ machine_id=$(flyctl machine list --app "$APP_NAME" --json \+ | jq -r '[.[] | select((.config.mounts // []) | length > 0)][0].id // empty')+ if [ -n "$machine_id" ]; then+ # A suspended preview needs demand before SSH is reachable.+ curl -s -o /dev/null --max-time 10 "https://$APP_NAME.fly.dev/" || true+ for _ in $(seq 1 12); do+ marker=$(flyctl ssh console --app "$APP_NAME" --machine "$machine_id" \+ --quiet --command 'if [ -f /var/lib/docker/.macro-preview/deployment.json ]; then cat /var/lib/docker/.macro-preview/deployment.json; else echo __NO_MARKER__; fi' \+ 2>/dev/null || true)+ if echo "$marker" | jq -e \+ --arg key "$snapshot_key" \+ --arg runtime "$runtime_key" \+ '.format == 1 and .snapshot_key == $key and .runtime_key == $runtime and (.frontend_key | type == "string")' \+ >/dev/null 2>&1; then+ mode=hot+ break+ fi+ # SSH succeeded and returned a marker (or proved it absent):+ # incompatibility is real, not a wake-up race.+ if [ "$marker" = __NO_MARKER__ ] \+ || echo "$marker" | jq -e '.format != null' >/dev/null 2>&1; then+ break+ fi+ sleep 5+ done+ fi+ fi+ echo "preview deployment mode: $mode (snapshot $snapshot_key)"++ flyctl auth docker+ registry="registry.fly.io/$APP_NAME"+ # Mirror every stack image into the app's registry repo instead of+ # baking docker-save tars into the VM image (which made every+ # deploy re-ship ~5GB the machine's volume already had). Pushes+ # dedup at the layer level against the repo, so a redeploy only+ # uploads layers that actually changed; the entrypoint pulls with+ # the same dedup against the machine's persistent layer store. The+ # manifest maps image ID -> local tag -> registry ref so boots can+ # skip images already present. alpine:3 rides along for the+ # snapshot-restore helper container.+ # On a snapshot cache hit only gen-compose ran, which doesn't write+ # the env file; compose hard-fails on a missing --env-file, and the+ # image names don't interpolate env vars anyway.+ envfile=infra/local/generated/macro/local.generated.env+ [ -f "$envfile" ] || : > "$envfile"+ images=$(docker compose --project-directory . -p macro \+ -f docker/docker-compose.yml \+ -f infra/local/generated/macro/docker-compose.override.yml \+ --env-file "$envfile" \+ config --images | sort -u)+ echo "mirroring images:" $images+ docker pull alpine:3+ mkdir -p preview-ctx/preload+ : > preview-ctx/preload/manifest.txt+ : > "$RUNNER_TEMP/push-refs.txt"+ for img in $images alpine:3; do+ # Images the bake didn't leave in the daemon (snapshot cache hit,+ # or app-layer-only images like proxy/mailpit) get pulled here.+ docker image inspect "$img" >/dev/null 2>&1 || docker pull "$img"+ id=$(docker image inspect -f '{{.Id}}' "$img")+ # Content-addressed tag: same image content = same ref, so a+ # redeploy's push is skippable by a manifest existence check and+ # concurrent PRs' deploys can never clobber each other's tags.+ ref="$registry:img-$(echo "$img" | tr '/:' '__')-$(echo "$id" | cut -c8-19)"+ docker tag "$img" "$ref"+ echo "$id $img $ref" >> preview-ctx/preload/manifest.txt+ if ! docker manifest inspect "$ref" >/dev/null 2>&1; then+ echo "$ref" >> "$RUNNER_TEMP/push-refs.txt"+ fi+ done+ echo "pushing $(wc -l < "$RUNNER_TEMP/push-refs.txt") of $(wc -l < preview-ctx/preload/manifest.txt) images"+ # The Fly registry occasionally aborts large uploads ("s3aws:+ # append to zero-size path unsupported") — retry; layers that made+ # it are reused. Four pushes at a time keeps the first (cold) push+ # moving without saturating the registry.+ if [ -s "$RUNNER_TEMP/push-refs.txt" ]; then+ xargs -P 4 -I {} sh -c '+ for _ in 1 2 3; do+ docker push "{}" && exit 0+ echo "push of {} failed, retrying" >&2+ sleep 15+ done+ exit 1' < "$RUNNER_TEMP/push-refs.txt"+ fi++ push_image() {+ image_to_push=$1+ for _ in 1 2 3; do+ if docker push "$image_to_push"; then return 0; fi+ echo "docker push failed for $image_to_push, retrying" >&2+ sleep 15+ done+ return 1+ }++ if [ "$mode" = hot ]; then+ update_image="$registry:update-${{ github.sha }}"+ docker build -f preview-ctx/update.Dockerfile -t "$update_image" preview-ctx+ push_image "$update_image"+ token_file="$RUNNER_TEMP/registry-pull-token"+ printf '%s' "$pull_token" > "$token_file"+ chmod 600 "$token_file"+ hot_rc=0+ flyctl ssh sftp put "$token_file" /tmp/macro-registry-token \+ --app "$APP_NAME" --machine "$machine_id" --mode 0600 \+ || hot_rc=$?+ if [ "$hot_rc" = 0 ]; then+ hot_command="/srv/macro/bin/hot-update '$update_image' /tmp/macro-registry-token"+ flyctl ssh console --app "$APP_NAME" --machine "$machine_id" \+ --command "$hot_command" || hot_rc=$?+ fi+ if [ "$hot_rc" = 0 ]; then+ echo "hot update completed without restarting the Fly machine"+ exit 0+ fi+ echo "hot update failed (exit $hot_rc); falling back to full rehydrate" >&2+ mode=rehydrate+ fi++ # Bootstrap/rehydrate refreshes app secrets and reconciles the+ # persistent Docker volume before replacing the machine image.+ flyctl secrets set --app "$APP_NAME" --stage \+ "DOPPLER_TOKEN=$DOPPLER_PREVIEW_TOKEN" \+ "REGISTRY_PULL_TOKEN=$pull_token"+ if ! flyctl volumes list --app "$APP_NAME" --json \+ | jq -e 'map(select(.name == "docker_data")) | length > 0' >/dev/null; then+ flyctl volumes create docker_data --app "$APP_NAME" --region ewr --size 40 --yes+ fi+ # Machines created before the volume existed can't take the mounts+ # config update. A destroyed/partial machine is recreated by deploy.+ flyctl machine list --app "$APP_NAME" --json \+ | jq -r '.[] | select((.config.mounts // []) | length == 0) | .id' \+ | while read -r id; do+ [ -z "$id" ] || flyctl machine destroy "$id" --app "$APP_NAME" --force || true+ done++ image="$registry:${{ github.sha }}"+ docker build -t "$image" preview-ctx+ push_image "$image"+ # Keep the machine in demand while it boots: with zero inbound+ # traffic, fly-proxy counts a health-failing machine as excess+ # capacity and suspends it mid-boot (observed mid-docker-load).+ # Any response status counts as traffic, and a request also+ # resumes an already-suspended machine.+ (while :; do+ curl -s -o /dev/null --max-time 10 "https://$APP_NAME.fly.dev/" || true+ sleep 15+ done) &+ keepalive=$!+ # First boot does real work before 8090 opens (pulling the stack+ # images, snapshot restore, compose up, FusionAuth's JVM) —+ # give the health check more runway than flyctl's default wait.+ rc=0+ flyctl deploy --app "$APP_NAME" \+ --config infra/preview/fly.toml \+ --image "$image" \+ --wait-timeout 1800 \+ --yes || rc=$?+ kill "$keepalive" 2>/dev/null || true+ exit "$rc"+ "#})+ // Accept the org slug from either a repo variable or a repo secret —+ // it's not sensitive, but people reasonably reach for secrets first.+ .add_env(("FLY_ORG", "${{ vars.FLY_ORG || secrets.FLY_ORG }}"))+ .add_env(("DOPPLER_PREVIEW_TOKEN", vars::DOPPLER_PREVIEW_TOKEN))+}++fn comment_preview_url() -> Step<Use> {+ Step::new("Comment preview URL")+ .uses(+ "actions",+ "github-script",+ "f28e40c7f34bde8b3046d885e986cb6290c5673b",+ )+ .add_with(("github-token", "${{ secrets.GITHUB_TOKEN }}"))+ .add_with((+ "script",+ indoc::indoc! {r#"+ const app = process.env.APP_NAME;+ const url = `https://${app}.fly.dev`;+ const marker = '<!-- fly-preview -->';+ const body = [+ marker,+ `🚀 **Full-stack preview**: ${url}`,+ '',+ `- Suspended when idle — the first request wakes it (a few seconds).`,+ `- Log in with any email; the passwordless code lands in [Mailpit](${url}/mailpit/).`,+ `- Deployed from ${context.payload.pull_request.head.sha.slice(0, 7)}.`,+ ].join('\n');+ const { data: comments } = await github.rest.issues.listComments({+ owner: context.repo.owner,+ repo: context.repo.repo,+ issue_number: context.issue.number,+ per_page: 100,+ });+ const existing = comments.find(c => c.body && c.body.startsWith(marker));+ if (existing) {+ await github.rest.issues.updateComment({+ owner: context.repo.owner,+ repo: context.repo.repo,+ comment_id: existing.id,+ body,+ });+ } else {+ await github.rest.issues.createComment({+ owner: context.repo.owner,+ repo: context.repo.repo,+ issue_number: context.issue.number,+ body,+ });+ }+ "#},+ ))+}++fn destroy() -> Job {+ Job::default()+ // Closed PRs always attempt cleanup (destroy is a no-op if no app);+ // unlabeled only cleans up when the removed label is `preview`.+ .cond(Expression::new(+ "github.event.action == 'closed' || github.event.label.name == 'preview'",+ ))+ .runs_on(runners::Runner::TinyNoCache.to_string())+ .permissions(Permissions::default().contents(Level::Read))+ .add_env(("FLY_API_TOKEN", vars::FLY_API_TOKEN))+ .add_step(setup_flyctl())+ .add_step(+ Step::new("Destroy app")+ .run(format!("flyctl apps destroy \"{APP_NAME}\" --yes || true")),+ )+}diff --git a/tooling/xtask/crates/xtask_workflows/src/workflows/steps.rs b/tooling/xtask/crates/xtask_workflows/src/workflows/steps.rsindex 1f53a2319a..02f8a1f61e 100644--- a/tooling/xtask/crates/xtask_workflows/src/workflows/steps.rs+++ b/tooling/xtask/crates/xtask_workflows/src/workflows/steps.rs@@ -104,6 +104,51 @@ pub fn mount_cache_volume() -> Step<Use> {.continue_on_error(true)}+/// [`mount_cache_volume`] plus the checkout's cargo target dir and the init+/// snapshot store. Persisting `target/` is what makes the preview job's+/// zigbuild incremental — cargo's own fingerprints carry across runs, where+/// remote sccache alone leaves build scripts, native (cmake/zig) compiles, and+/// linking cold every time. Persisting the snapshot store gives the bake step+/// a zero-copy fast path; Namespace artifact storage is its durable fallback.+/// The volume is a block-device mount, so multi-GB trees cost nothing to save+/// or restore when it hits.+pub fn mount_cache_volume_with_cargo_target() -> Step<Use> {+ Step::new("Mount Namespace cache volume")+ .uses(+ "namespacelabs",+ "nscloud-cache-action",+ "15799a6b54e5765f85b2aac25b3f0df43ed571c0", // v1.4.3+ )+ .add_with(("cache", "rust"))+ .add_with((+ "path",+ format!(+ "/nix\n${{{{ github.workspace }}}}/target\n{}",+ vars::PREVIEW_SNAPSHOT_VOLUME_DIR,+ ),+ ))+ .continue_on_error(true)+}++/// Configure Namespace's artifact-backed remote sccache. `setup-reqs-web`+/// already installs sccache and exports `RUSTC_WRAPPER=sccache`; this replaces+/// the local-disk backend with short-lived WebDAV credentials that work across+/// runners and cache-volume misses.+pub fn configure_namespace_sccache(cache_name: &str) -> Step<Run> {+ Step::new("Configure Namespace remote sccache").run(format!(+ r#"set -euo pipefail+env_file="$RUNNER_TEMP/namespace-sccache.env"+if nsc cache sccache setup --cache_name {cache_name} > "$env_file"; then+ cat "$env_file" >> "$GITHUB_ENV"+ # Force the next compiler invocation to start a server with the new remote+ # backend even if a setup hook happened to launch one already.+ sccache --stop-server >/dev/null 2>&1 || true+else+ echo "::warning::Namespace remote sccache setup failed; using local cache fallback"+fi"#+ ))+}+/// Repoint sccache at the persisted volume. Runs AFTER `setup-cachix`; this keeps/// the compiled artifact cache directory aligned with the Namespace mounted path.pub fn pin_sccache_dir() -> Step<Run> {diff --git a/tooling/xtask/crates/xtask_workflows/src/workflows/vars.rs b/tooling/xtask/crates/xtask_workflows/src/workflows/vars.rsindex b7ab1c8aaf..bbf55745c9 100644--- a/tooling/xtask/crates/xtask_workflows/src/workflows/vars.rs+++ b/tooling/xtask/crates/xtask_workflows/src/workflows/vars.rs@@ -18,7 +18,9 @@ secret!(CLOUDFLARE_API_TOKEN);secret!(DD_API_KEY);secret!(DD_APP_KEY);secret!(DD_WEB_APP_TOKEN);+secret!(DOPPLER_PREVIEW_TOKEN);secret!(DOPPLER_TOKEN);+secret!(FLY_API_TOKEN);secret!(MACOS_DEVELOPER_ID_CERTIFICATE_BASE64);secret!(MACOS_DEVELOPER_ID_CERTIFICATE_PASSWORD);secret!(POSTHOG_API_KEY);@@ -40,6 +42,23 @@ pub const NEXTEST_TEST_THREADS: u32 = 32;/// global across all branches — see [`crate::workflows::runners::Runner::with_cache_tag`].pub const CI_CACHE_TAG: &str = "sccache-ci";+/// Namespace cache tag for the Fly preview deploy job. Its own pool, NOT+/// [`CI_CACHE_TAG`]: sharing looked economical but was measured cold both ways+/// (run 28968155599: sccache 2.65% hits, cargo target dir absent). The+/// check/test jobs compile for the host while this job zigbuilds with+/// `--target x86_64-unknown-linux-gnu.2.36`, so their sccache entries hash+/// differently and never serve this job — and they don't persist the cargo+/// target dir at all, so a volume from the shared pool almost never carries+/// one. A dedicated low-concurrency pool gives the job's target dir and init+/// snapshots the best chance of a zero-copy hit; Rust objects use the durable+/// remote sccache below instead of depending on this volume's placement.+pub const PREVIEW_CACHE_TAG: &str = "fly-preview";++/// Durable Namespace remote sccache shared by Fly preview jobs. Unlike the+/// local cache volume, this has an artifact-backed cold tier, so a job placed+/// on a runner that has never seen the volume can still reuse Rust objects.+pub const PREVIEW_SCCACHE_NAME: &str = "fly-preview";+/// Namespace cache tag for the web-app jobs (PR checks + preview deploys)./// Cache volumes are keyed workspace-wide by tag alone, so a dedicated tag/// gives the frontend its own volume — isolated both from the Rust CI volume@@ -51,6 +70,12 @@ pub const WEB_CI_CACHE_TAG: &str = "web-ci";/// volume so it persists across runs — this is what replaces the S3 bucket.pub const SCCACHE_VOLUME_DIR: &str = "/home/runner/.cache/sccache";+/// Init-snapshot store for the preview job (`MACRO_STACK_SNAPSHOT_DIR`). Lives+/// on the preview cache volume for the zero-copy fast path; the workflow also+/// backs each content-addressed snapshot up to Namespace artifact storage so a+/// cache-volume miss does not force another infra bake.+pub const PREVIEW_SNAPSHOT_VOLUME_DIR: &str = "/home/runner/.cache/macro-preview-snapshots";+/// Max on-disk size for the sccache cache. Larger than the setup default since/// the persisted volume can hold a full-workspace cache.pub const SCCACHE_CACHE_SIZE: &str = "20G";