Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

chore(ci): migrate the deploy pipeline family to cargo x workflows (#4529)

teof90d8b713be2
diff --git a/.github/actionlint.yaml b/.github/actionlint.yaml
index 579545a746..0f45ea3d1c 100644
--- a/.github/actionlint.yaml
+++ b/.github/actionlint.yaml
@@ -1,5 +1,6 @@
self-hosted-runner:
labels:
+ - db-migrator
- linux-extra-beefy
- linux-latest-middy
- namespace-profile-linux-mid
diff --git a/.github/workflows/cancel_stuck_cloud_storage_deploys.yml b/.github/workflows/cancel_stuck_cloud_storage_deploys.yml
index 4a2a13c901..dea6411f06 100644
--- a/.github/workflows/cancel_stuck_cloud_storage_deploys.yml
+++ b/.github/workflows/cancel_stuck_cloud_storage_deploys.yml
@@ -24,7 +24,7 @@ jobs:
name: Cancel runs stuck queued or in_progress past threshold
runs-on: namespace-profile-linux-tiny-no-cache
env:
- WORKFLOW_FILE: deploy-cloud-storage-on-push.yml
+ WORKFLOW_FILE: deploy_cloud_storage_on_push.yml
MAX_AGE_MINUTES: ${{ inputs.max-age-minutes || '90' }}
DRY_RUN: ${{ inputs.dry-run || 'false' }}
GH_TOKEN: ${{ github.token }}
diff --git a/.github/workflows/deploy-all-services.yml b/.github/workflows/deploy-all-services.yml
deleted file mode 100644
index b02e030046..0000000000
--- a/.github/workflows/deploy-all-services.yml
+++ /dev/null
@@ -1,437 +0,0 @@
-name: Deploy All Services
-on:
- workflow_dispatch:
- inputs:
- environment:
- type: choice
- required: true
- default: 'dev'
- options:
- - dev
- - prod
- description: The environment we are deploying to.
- workflow_call:
- inputs:
- environment:
- type: string
- required: true
- description: The environment we are deploying to.
- secrets:
- AWS_ACCESS_KEY:
- required: true
- AWS_SECRET_ACCESS_KEY:
- required: true
- PULUMI_ACCESS_TOKEN:
- required: true
- DD_APP_KEY:
- required: true
- DD_API_KEY:
- required: true
- CACHIX_AUTH_TOKEN:
- required: true
-
-# Only run 1 deploy-all workflow at a time per environment. Literal prefix:
-# for workflow_call runs `github.workflow` expands to the *caller's* name,
-# which would split dispatches and callers into separate groups and let them
-# race the same Pulumi stacks. Groups are repo-global strings, so the on-push
-# caller shares this group by using the same literal.
-concurrency:
- group: deploy-all-services-${{ inputs.environment }}
- cancel-in-progress: false
-
-jobs:
- setup:
- name: Setup Deployment Matrix
- runs-on: ubuntu-latest
- outputs:
- matrix: ${{ steps.set-matrix.outputs.matrix }}
- binaries: ${{ steps.set-matrix.outputs.binaries }}
- lambdas: ${{ steps.set-matrix.outputs.lambdas }}
- steps:
- - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd
- - name: Set deployment matrix
- id: set-matrix
- run: |
- set -euo pipefail
- cfg=.github/services-config.json
- # Full list drives deploy-services; the filtered lists keep each build
- # matrix to only the services that actually produce that artifact, so
- # no build job spins up Nix + a cache volume for nothing.
- services=$(jq -c '.services | keys' "$cfg")
- binaries=$(jq -c '[.services | to_entries[] | select((.value.deploy_binaries // []) | length > 0) | .key]' "$cfg")
- lambdas=$(jq -c '[.services | to_entries[] | select((.value.deploy_lambdas // []) | length > 0) | .key]' "$cfg")
- echo "matrix=${services}" >> "$GITHUB_OUTPUT"
- echo "binaries=${binaries}" >> "$GITHUB_OUTPUT"
- echo "lambdas=${lambdas}" >> "$GITHUB_OUTPUT"
- echo "All services: ${services}"
- echo "With binaries: ${binaries}"
- echo "With lambdas: ${lambdas}"
-
- # Warm the two shared dep closures in parallel. Each build matrix depends
- # only on its own warm job, so a slow lambda-closure warm never blocks
- # binary builds and vice versa. The /nix cache volume is an optimization;
- # Cachix is the consistency backstop either way.
- warm-binaries:
- name: Warm shared binary deps onto sticky disk
- needs: setup
- if: ${{ needs.setup.outputs.binaries != '[]' }}
- runs-on: namespace-profile-linux-mid
- steps:
- - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd
-
- # Namespace cache volume backing /nix. Purely an optimization: if the
- # profile has no cache volumes (or the mount fails), setup-nix performs
- # a cold install and Cachix substitutes the warm layers — slower, never
- # wrong. Hence continue-on-error.
- - name: Mount /nix cache volume
- continue-on-error: true
- uses: namespacelabs/nscloud-cache-action@15799a6b54e5765f85b2aac25b3f0df43ed571c0 # v1.4.3
- with:
- path: /nix
-
- - name: Setup Nix
- uses: ./.github/actions/setup-nix
-
- - name: Configure Cachix fallback
- uses: ./.github/actions/setup-cachix
- with:
- cachix-auth-token: ${{ secrets.CACHIX_AUTH_TOKEN }}
-
- - name: Warm shared release deps
- shell: bash
- run: |
- set -euo pipefail
- cachix_pid=
- if command -v cachix >/dev/null 2>&1 && [ -n "${CACHIX_CACHE_NAME:-}" ]; then
- cachix watch-store "$CACHIX_CACHE_NAME" >/tmp/cachix-watch-store.log 2>&1 &
- cachix_pid=$!
- trap 'if [ -n "${cachix_pid:-}" ]; then kill "$cachix_pid" 2>/dev/null || true; wait "$cachix_pid" 2>/dev/null || true; fi' EXIT
- fi
- nix build --print-build-logs ".#deployCargoArtifacts"
- echo "Shared release deps realised into /nix/store; committed on job exit."
-
- - name: Teardown Nix (free cache volume for commit)
- if: always()
- uses: ./.github/actions/teardown-nix
-
- warm-lambdas:
- name: Warm shared lambda deps onto sticky disk
- needs: setup
- if: ${{ needs.setup.outputs.lambdas != '[]' }}
- runs-on: namespace-profile-linux-mid
- steps:
- - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd
-
- # Namespace cache volume backing /nix. Purely an optimization: if the
- # profile has no cache volumes (or the mount fails), setup-nix performs
- # a cold install and Cachix substitutes the warm layers — slower, never
- # wrong. Hence continue-on-error.
- - name: Mount /nix cache volume
- continue-on-error: true
- uses: namespacelabs/nscloud-cache-action@15799a6b54e5765f85b2aac25b3f0df43ed571c0 # v1.4.3
- with:
- path: /nix
-
- - name: Setup Nix
- uses: ./.github/actions/setup-nix
-
- - name: Configure Cachix fallback
- uses: ./.github/actions/setup-cachix
- with:
- cachix-auth-token: ${{ secrets.CACHIX_AUTH_TOKEN }}
-
- - name: Warm shared lambda dep closure
- shell: bash
- run: |
- set -euo pipefail
- cachix_pid=
- if command -v cachix >/dev/null 2>&1 && [ -n "${CACHIX_CACHE_NAME:-}" ]; then
- cachix watch-store "$CACHIX_CACHE_NAME" >/tmp/cachix-watch-store.log 2>&1 &
- cachix_pid=$!
- trap 'if [ -n "${cachix_pid:-}" ]; then kill "$cachix_pid" 2>/dev/null || true; wait "$cachix_pid" 2>/dev/null || true; fi' EXIT
- fi
- nix build --print-build-logs ".#lambdaDeployCargoArtifacts" ".#callRecordingPreviewFfmpegLayer"
- echo "Shared lambda deps realised into /nix/store; committed on job exit."
-
- - name: Teardown Nix (free cache volume for commit)
- if: always()
- uses: ./.github/actions/teardown-nix
-
- build-service-binaries:
- name: Build ${{ matrix.service }} binaries
- needs: [setup, warm-binaries]
- if: ${{ needs.setup.outputs.binaries != '[]' }}
- # Clones the warm /nix cache volume populated by warm-binaries so only
- # each service's own crate compiles (cold volume -> Cachix substitution).
- runs-on: namespace-profile-linux-mid
- strategy:
- fail-fast: false
- # No max-parallel: all binary services build concurrently (they only clone
- # the warm /nix disk and compile their own crate), instead of batching 8.
- matrix:
- service: ${{ fromJson(needs.setup.outputs.binaries) }}
- steps:
- - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd
- with:
- clean: false
-
- # Namespace cache volume backing /nix. Purely an optimization: if the
- # profile has no cache volumes (or the mount fails), setup-nix performs
- # a cold install and Cachix substitutes the warm layers — slower, never
- # wrong. Hence continue-on-error.
- - name: Mount /nix cache volume
- continue-on-error: true
- uses: namespacelabs/nscloud-cache-action@15799a6b54e5765f85b2aac25b3f0df43ed571c0 # v1.4.3
- with:
- path: /nix
-
- - name: Setup Nix
- uses: ./.github/actions/setup-nix
-
- - name: Configure Cachix fallback
- uses: ./.github/actions/setup-cachix
- with:
- cachix-auth-token: ${{ secrets.CACHIX_AUTH_TOKEN }}
-
- - name: Build prebuilt binaries
- shell: bash
- env:
- SERVICE: ${{ matrix.service }}
- run: |
- set -euo pipefail
- mkdir -p prebuilt
-
- binaries=$(jq -r --arg service "$SERVICE" '.services[$service].deploy_binaries // [] | @tsv' .github/services-config.json)
- if [[ -n "$binaries" ]]; then
- cachix_pid=
- if command -v cachix >/dev/null 2>&1 && [ -n "${CACHIX_CACHE_NAME:-}" ]; then
- cachix watch-store "$CACHIX_CACHE_NAME" >/tmp/cachix-watch-store.log 2>&1 &
- cachix_pid=$!
- trap 'if [ -n "${cachix_pid:-}" ]; then kill "$cachix_pid" 2>/dev/null || true; wait "$cachix_pid" 2>/dev/null || true; fi' EXIT
- fi
- nix build --print-build-logs ".#deploy-service-binaries-${SERVICE}"
- cp -r result/bin/* prebuilt/
- mkdir -p prebuilt/nix-store
- while IFS= read -r store_path; do
- cp -a "$store_path" prebuilt/nix-store/
- done < <(nix-store -qR result)
- else
- echo "No deploy binaries configured for $SERVICE; uploading an empty artifact"
- fi
-
- touch prebuilt/.keep
- tar -C prebuilt -czf prebuilt-binaries.tar.gz .
- # Receipt: the deploy job logs the same hash on read.
- echo "handoff receipt: $(sha256sum prebuilt-binaries.tar.gz | cut -d' ' -f1) ($(stat -c%s prebuilt-binaries.tar.gz) bytes)"
-
- # Handoff to the matching deploy job via Namespace artifact storage:
- # strongly consistent object storage (no snapshot-visibility races),
- # rides Namespace's network rather than the GitHub artifacts API.
- # Attempt-scoped path so re-runs never collide with stale uploads.
- - name: Upload handoff artifact
- shell: bash
- env:
- DEST: handoff/${{ github.run_id }}-${{ github.run_attempt }}/${{ matrix.service }}/prebuilt-binaries.tar.gz
- run: nsc artifact upload prebuilt-binaries.tar.gz "$DEST" --expires_in=24h
-
- - name: Teardown Nix (free cache volume for commit)
- if: always()
- uses: ./.github/actions/teardown-nix
-
- build-lambda-artifacts:
- name: Build ${{ matrix.service }} Lambda artifacts
- needs: [setup, warm-lambdas]
- if: ${{ needs.setup.outputs.lambdas != '[]' }}
- # Each handler is a crane + cargo-zigbuild nix package. This job builds all
- # of a service's handlers via `nix build .#deploy-lambda-<name>`, cloning the
- # warm lambda /nix disk committed by warm-lambdas. Unchanged handlers are
- # pure cache hits (substituted from the disk/Cachix, no recompile).
- runs-on: namespace-profile-linux-mid
- strategy:
- fail-fast: false
- # No max-parallel: runners autoscale, so all lambda services
- # build concurrently instead of batching 8.
- matrix:
- service: ${{ fromJson(needs.setup.outputs.lambdas) }}
- steps:
- - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd
- with:
- clean: false
-
- # Namespace cache volume backing /nix. Purely an optimization: if the
- # profile has no cache volumes (or the mount fails), setup-nix performs
- # a cold install and Cachix substitutes the warm layers — slower, never
- # wrong. Hence continue-on-error.
- - name: Mount /nix cache volume
- continue-on-error: true
- uses: namespacelabs/nscloud-cache-action@15799a6b54e5765f85b2aac25b3f0df43ed571c0 # v1.4.3
- with:
- path: /nix
-
- - name: Setup Nix
- uses: ./.github/actions/setup-nix
-
- - name: Configure Cachix fallback
- uses: ./.github/actions/setup-cachix
- with:
- cachix-auth-token: ${{ secrets.CACHIX_AUTH_TOKEN }}
-
- - name: Build Lambda artifacts
- env:
- SERVICE: ${{ matrix.service }}
- run: .github/scripts/build-cloud-storage-lambdas-nix.sh
-
- - name: Log handoff receipt
- shell: bash
- run: |
- set -euo pipefail
- # The build script writes lambda-artifacts.tar.gz to the workspace
- # root; the deploy job logs the same hash on read.
- echo "handoff receipt: $(sha256sum lambda-artifacts.tar.gz | cut -d' ' -f1) ($(stat -c%s lambda-artifacts.tar.gz) bytes)"
-
- # Handoff via Namespace artifact storage (see build-service-binaries).
- - name: Upload handoff artifact
- shell: bash
- env:
- DEST: handoff/${{ github.run_id }}-${{ github.run_attempt }}/${{ matrix.service }}/lambda-artifacts.tar.gz
- run: nsc artifact upload lambda-artifacts.tar.gz "$DEST" --expires_in=24h
-
- - name: Teardown Nix (free cache volume for commit)
- if: always()
- uses: ./.github/actions/teardown-nix
-
- migrate-db:
- name: Run Database Migrations
- runs-on: db-migrator
- needs: [build-service-binaries, build-lambda-artifacts]
- steps:
- - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd
- with:
- sparse-checkout: .github/
- - name: Run migrations
- uses: ./.github/actions/migrate-cloud-storage-db
- with:
- environment: ${{ inputs.environment }}
-
- deploy-services:
- name: Deploy ${{ matrix.service }}
- needs: [setup, build-service-binaries, build-lambda-artifacts, migrate-db]
- if: ${{ needs.setup.outputs.matrix != '[]' }}
- # Deploys via Pulumi + Docker; AWS auth is via explicit static keys
- # (configure-aws-credentials).
- runs-on: namespace-profile-linux-small
- # Pin PULUMI_HOME to a fixed path (outside the workspace, which the deploy
- # action's checkout cleans) so the plugins subdir can be backed by a sticky
- # disk. Propagates into the composite action's pulumi step via the job env.
- env:
- PULUMI_HOME: /pulumi
- strategy:
- fail-fast: false
- # No max-parallel: all services deploy concurrently.
- matrix:
- service: ${{ fromJson(needs.setup.outputs.matrix) }}
- steps:
- - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd
-
- - name: Get project name
- id: project-name
- uses: ./.github/actions/get-project-name
- with:
- service-name: ${{ matrix.service }}
-
- - name: Check artifact config
- id: check-artifacts
- env:
- SERVICE: ${{ matrix.service }}
- run: |
- has_binaries=$(jq -r --arg service "$SERVICE" '((.services[$service].deploy_binaries // []) | length) > 0' .github/services-config.json)
- has_lambdas=$(jq -r --arg service "$SERVICE" '((.services[$service].deploy_lambdas // []) | length) > 0' .github/services-config.json)
- echo "has_binaries=$has_binaries" >> "$GITHUB_OUTPUT"
- echo "has_lambdas=$has_lambdas" >> "$GITHUB_OUTPUT"
-
- # Pull the handoff tars from Namespace artifact storage into runner.temp
- # (outside the workspace, which the composite action's checkout cleans).
- # The composite's tar-path branch handles receipts + the extract guard.
- - name: Download handoff artifacts
- if: ${{ steps.check-artifacts.outputs.has_binaries == 'true' || steps.check-artifacts.outputs.has_lambdas == 'true' }}
- shell: bash
- env:
- HAS_BINARIES: ${{ steps.check-artifacts.outputs.has_binaries }}
- HAS_LAMBDAS: ${{ steps.check-artifacts.outputs.has_lambdas }}
- BASE: handoff/${{ github.run_id }}-${{ github.run_attempt }}/${{ matrix.service }}
- run: |
- set -euo pipefail
- if ! command -v nsc >/dev/null 2>&1; then
- echo "::error::nsc CLI not found — this job expects a Namespace runner (or add namespacelabs/nscloud-setup)"
- exit 1
- fi
- mkdir -p "$RUNNER_TEMP/handoff"
- if [[ "$HAS_BINARIES" == "true" ]]; then
- nsc artifact download "$BASE/prebuilt-binaries.tar.gz" "$RUNNER_TEMP/handoff/prebuilt-binaries.tar.gz"
- fi
- if [[ "$HAS_LAMBDAS" == "true" ]]; then
- nsc artifact download "$BASE/lambda-artifacts.tar.gz" "$RUNNER_TEMP/handoff/lambda-artifacts.tar.gz"
- fi
-
- # Cache Pulumi's provider plugins ($PULUMI_HOME/plugins) on a Namespace
- # cache volume. Plugins are version-pinned by infra/ and identical across
- # services. Optimization-only: a cold volume just re-downloads (~45s).
- - name: Cache Pulumi plugins
- continue-on-error: true
- uses: namespacelabs/nscloud-cache-action@15799a6b54e5765f85b2aac25b3f0df43ed571c0 # v1.4.3
- with:
- path: /pulumi/plugins
-
- # PULUMI_HOME (/pulumi) and its mounted plugins subdir come up root-owned;
- # make them writable so pulumi can write credentials + plugins as the
- # runner. No-op when the runner already runs as root.
- - name: Ensure Pulumi home is writable
- shell: bash
- run: |
- set -euo pipefail
- if [ "$(id -u)" -ne 0 ]; then sudo chown -R "$(id -u):$(id -g)" /pulumi; fi
-
- - name: Deploy ${{ matrix.service }}
- uses: ./.github/actions/deploy-cloud-storage-pulumi
- with:
- environment: ${{ inputs.environment }}
- aws-access-key: ${{ secrets.AWS_ACCESS_KEY }}
- aws-secret-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
- pulumi-access-token: ${{ secrets.PULUMI_ACCESS_TOKEN }}
- pulumi-service-name: ${{ steps.project-name.outputs.project-name }}
- use-namespace-builder: 'true'
- prebuilt-binaries-tar: ${{ steps.check-artifacts.outputs.has_binaries == 'true' && format('{0}/handoff/prebuilt-binaries.tar.gz', runner.temp) || '' }}
- lambda-artifacts-tar: ${{ steps.check-artifacts.outputs.has_lambdas == 'true' && format('{0}/handoff/lambda-artifacts.tar.gz', runner.temp) || '' }}
- dd-app-key: ${{ secrets.DD_APP_KEY }}
- dd-api-key: ${{ secrets.DD_API_KEY }}
-
- deployment-summary:
- name: Deployment Summary
- runs-on: ubuntu-latest
- needs: [setup, warm-binaries, warm-lambdas, build-service-binaries, build-lambda-artifacts, deploy-services]
- if: always()
- steps:
- - name: Check deployment results
- run: |
- if [[ "${{ needs.setup.result }}" == "failure" ]]; then
- echo "❌ Deployment setup failed"
- exit 1
- elif [[ "${{ needs.setup.result }}" == "skipped" ]]; then
- echo "⏭️ Deployment setup was skipped"
- elif [[ "${{ needs.warm-binaries.result }}" == "failure" || "${{ needs.warm-lambdas.result }}" == "failure" ]]; then
- echo "❌ Warming shared deps onto a sticky disk failed"
- exit 1
- elif [[ "${{ needs.build-service-binaries.result }}" == "failure" ]]; then
- echo "❌ One or more service binary builds failed"
- exit 1
- elif [[ "${{ needs.build-lambda-artifacts.result }}" == "failure" ]]; then
- echo "❌ One or more Lambda artifact builds failed"
- exit 1
- elif [[ "${{ needs.deploy-services.result }}" == "failure" ]]; then
- echo "❌ One or more service deployments failed"
- exit 1
- elif [[ "${{ needs.deploy-services.result }}" == "skipped" ]]; then
- echo "⏭️ No services to deploy"
- else
- echo "✅ All service deployments completed successfully for ${{ inputs.environment }} environment"
- fi
diff --git a/.github/workflows/deploy-cloud-storage-on-push.yml b/.github/workflows/deploy-cloud-storage-on-push.yml
deleted file mode 100644
index 8cf5339557..0000000000
--- a/.github/workflows/deploy-cloud-storage-on-push.yml
+++ /dev/null
@@ -1,63 +0,0 @@
-# Thin trigger: every push to main touching the Rust services (or the deploy
-# machinery) deploys all services to dev via the shared deploy-all-services
-# pipeline — the same warm sticky-disk nix builds, crane/zigbuild lambdas, and
-# handoff topology used by manual dispatches and prod releases. This replaced
-# a standalone pipeline that built binaries on Cachix-only runners and lambdas
-# via per-handler cargo-lambda jobs.
-name: Deploy Cloud Storage Services on Push
-on:
- push:
- branches: [ 'main' ]
- paths:
- - 'Cargo.toml'
- - 'Cargo.lock'
- - 'Cross.toml'
- - 'clippy.toml'
- - 'deny.toml'
- - 'rust-toolchain.toml'
- - '.cargo/**'
- - '.config/**'
- - '.sqlx/**'
- - 'crates/**'
- - 'services/**'
- - 'tooling/xtask/**'
- - 'tooling/just/**'
- - 'tooling/scripts/**'
- - 'static_assets/**'
- - 'docker/**'
- - 'nix/**'
- - 'nix-support/**'
- - 'infra/**'
- - '.github/workflows/deploy-cloud-storage-on-push.yml'
- - '.github/workflows/deploy-all-services.yml'
- - '.github/actions/deploy-cloud-storage-pulumi/**'
- - '.github/actions/setup-nix/**'
- - '.github/actions/teardown-nix/**'
- - '.github/actions/setup-cachix/**'
- - '.github/actions/migrate-cloud-storage-db/**'
- - '.github/scripts/build-cloud-storage-lambdas-nix.sh'
- - '.github/services-config.json'
- - '.github/workspace-dep-closures.json'
- - 'flake.nix'
- - 'flake.lock'
-
-# NOTE: deliberately no concurrency block here. The called workflow's own
-# top-level group (deploy-all-services-${{ inputs.environment }}) IS honored
-# for workflow_call runs and is what serializes push deploys against manual
-# dev dispatches (running deploys finish; queued pushes coalesce to the
-# newest). Declaring the same group in this wrapper makes GitHub detect a
-# self-deadlock ("between a top level workflow and ...") and cancel the run.
-
-jobs:
- deploy-all:
- name: Deploy Cloud Storage Services
- uses: ./.github/workflows/deploy-all-services.yml
- with:
- environment: dev
- secrets:
- AWS_ACCESS_KEY: ${{ secrets.AWS_ACCESS_KEY }}
- AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
- PULUMI_ACCESS_TOKEN: ${{ secrets.PULUMI_ACCESS_TOKEN }}
- DD_APP_KEY: ${{ secrets.DD_APP_KEY }}
- DD_API_KEY: ${{ secrets.DD_API_KEY }}
- CACHIX_AUTH_TOKEN: ${{ secrets.CACHIX_AUTH_TOKEN }}
diff --git a/.github/workflows/deploy-service-generic.yml b/.github/workflows/deploy-service-generic.yml
index 9366f58d16..ff5ab7c678 100644
--- a/.github/workflows/deploy-service-generic.yml
+++ b/.github/workflows/deploy-service-generic.yml
@@ -52,7 +52,7 @@ concurrency:
jobs:
deploy:
name: Deploy ${{ inputs.service }}
- uses: ./.github/workflows/reusable-deploy-service.yml
+ uses: ./.github/workflows/reusable_deploy_service.yml
with:
environment: ${{ inputs.environment }}
service-name: ${{ inputs.service }}
diff --git a/.github/workflows/deploy_all_services.yml b/.github/workflows/deploy_all_services.yml
new file mode 100644
index 0000000000..965ad02b94
--- /dev/null
+++ b/.github/workflows/deploy_all_services.yml
@@ -0,0 +1,390 @@
+# DO NOT EDIT — regenerate with `cargo x workflows` (from the repository root).
+# Source: tooling/xtask/crates/xtask_workflows/src/workflows/deploy_all_services.rs
+name: Deploy All Services
+on:
+ workflow_call:
+ inputs:
+ environment:
+ type: string
+ required: true
+ description: The environment we are deploying to.
+ secrets:
+ AWS_ACCESS_KEY:
+ required: true
+ AWS_SECRET_ACCESS_KEY:
+ required: true
+ PULUMI_ACCESS_TOKEN:
+ required: true
+ DD_APP_KEY:
+ required: true
+ DD_API_KEY:
+ required: true
+ CACHIX_AUTH_TOKEN:
+ required: true
+ workflow_dispatch:
+ inputs:
+ environment:
+ type: choice
+ required: true
+ default: dev
+ options:
+ - dev
+ - prod
+ description: The environment we are deploying to.
+jobs:
+ setup:
+ name: Setup Deployment Matrix
+ runs-on: namespace-profile-linux-tiny-no-cache
+ outputs:
+ matrix: ${{ steps.set-matrix.outputs.matrix }}
+ binaries: ${{ steps.set-matrix.outputs.binaries }}
+ lambdas: ${{ steps.set-matrix.outputs.lambdas }}
+ steps:
+ - name: Checkout Repo
+ uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd
+ with:
+ persist-credentials: 'false'
+ - id: set-matrix
+ name: Set deployment matrix
+ run: |
+ set -euo pipefail
+ cfg=.github/services-config.json
+ # Full list drives deploy-services; the filtered lists keep each build
+ # matrix to only the services that actually produce that artifact, so
+ # no build job spins up Nix + a cache volume for nothing.
+ services=$(jq -c '.services | keys' "$cfg")
+ binaries=$(jq -c '[.services | to_entries[] | select((.value.deploy_binaries // []) | length > 0) | .key]' "$cfg")
+ lambdas=$(jq -c '[.services | to_entries[] | select((.value.deploy_lambdas // []) | length > 0) | .key]' "$cfg")
+ echo "matrix=${services}" >> "$GITHUB_OUTPUT"
+ echo "binaries=${binaries}" >> "$GITHUB_OUTPUT"
+ echo "lambdas=${lambdas}" >> "$GITHUB_OUTPUT"
+ echo "All services: ${services}"
+ echo "With binaries: ${binaries}"
+ echo "With lambdas: ${lambdas}"
+ warm-binaries:
+ needs:
+ - setup
+ if: ${{ needs.setup.outputs.binaries != '[]' }}
+ name: Warm shared binary deps onto sticky disk
+ runs-on: namespace-profile-linux-mid
+ steps:
+ - name: Checkout Repo
+ uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd
+ with:
+ persist-credentials: 'false'
+ - name: Mount /nix cache volume
+ uses: namespacelabs/nscloud-cache-action@15799a6b54e5765f85b2aac25b3f0df43ed571c0
+ with:
+ path: /nix
+ continue-on-error: true
+ - name: Setup Nix
+ uses: ./.github/actions/setup-nix
+ - name: Configure Cachix fallback
+ uses: ./.github/actions/setup-cachix
+ with:
+ cachix-auth-token: ${{ secrets.CACHIX_AUTH_TOKEN }}
+ - name: Warm shared release deps
+ run: |
+ set -euo pipefail
+ cachix_pid=
+ if command -v cachix >/dev/null 2>&1 && [ -n "${CACHIX_CACHE_NAME:-}" ]; then
+ cachix watch-store "$CACHIX_CACHE_NAME" >/tmp/cachix-watch-store.log 2>&1 &
+ cachix_pid=$!
+ trap 'if [ -n "${cachix_pid:-}" ]; then kill "$cachix_pid" 2>/dev/null || true; wait "$cachix_pid" 2>/dev/null || true; fi' EXIT
+ fi
+ nix build --print-build-logs ".#deployCargoArtifacts"
+ echo "Shared release deps realised into /nix/store; committed on job exit."
+ shell: bash
+ - name: Teardown Nix
+ if: always()
+ uses: ./.github/actions/teardown-nix
+ warm-lambdas:
+ needs:
+ - setup
+ if: ${{ needs.setup.outputs.lambdas != '[]' }}
+ name: Warm shared lambda deps onto sticky disk
+ runs-on: namespace-profile-linux-mid
+ steps:
+ - name: Checkout Repo
+ uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd
+ with:
+ persist-credentials: 'false'
+ - name: Mount /nix cache volume
+ uses: namespacelabs/nscloud-cache-action@15799a6b54e5765f85b2aac25b3f0df43ed571c0
+ with:
+ path: /nix
+ continue-on-error: true
+ - name: Setup Nix
+ uses: ./.github/actions/setup-nix
+ - name: Configure Cachix fallback
+ uses: ./.github/actions/setup-cachix
+ with:
+ cachix-auth-token: ${{ secrets.CACHIX_AUTH_TOKEN }}
+ - name: Warm shared lambda dep closure
+ run: |
+ set -euo pipefail
+ cachix_pid=
+ if command -v cachix >/dev/null 2>&1 && [ -n "${CACHIX_CACHE_NAME:-}" ]; then
+ cachix watch-store "$CACHIX_CACHE_NAME" >/tmp/cachix-watch-store.log 2>&1 &
+ cachix_pid=$!
+ trap 'if [ -n "${cachix_pid:-}" ]; then kill "$cachix_pid" 2>/dev/null || true; wait "$cachix_pid" 2>/dev/null || true; fi' EXIT
+ fi
+ nix build --print-build-logs ".#lambdaDeployCargoArtifacts" ".#callRecordingPreviewFfmpegLayer"
+ echo "Shared lambda deps realised into /nix/store; committed on job exit."
+ shell: bash
+ - name: Teardown Nix
+ if: always()
+ uses: ./.github/actions/teardown-nix
+ build-service-binaries:
+ needs:
+ - setup
+ - warm-binaries
+ if: ${{ needs.setup.outputs.binaries != '[]' }}
+ name: Build ${{ matrix.service }} binaries
+ runs-on: namespace-profile-linux-mid
+ strategy:
+ matrix:
+ service: ${{ fromJson(needs.setup.outputs.binaries) }}
+ fail-fast: false
+ steps:
+ - name: Checkout Repo
+ uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd
+ with:
+ persist-credentials: 'false'
+ clean: 'false'
+ - name: Mount /nix cache volume
+ uses: namespacelabs/nscloud-cache-action@15799a6b54e5765f85b2aac25b3f0df43ed571c0
+ with:
+ path: /nix
+ continue-on-error: true
+ - name: Setup Nix
+ uses: ./.github/actions/setup-nix
+ - name: Configure Cachix fallback
+ uses: ./.github/actions/setup-cachix
+ with:
+ cachix-auth-token: ${{ secrets.CACHIX_AUTH_TOKEN }}
+ - name: Build prebuilt binaries
+ run: |+
+ set -euo pipefail
+ cachix_pid=
+ if command -v cachix >/dev/null 2>&1 && [ -n "${CACHIX_CACHE_NAME:-}" ]; then
+ cachix watch-store "$CACHIX_CACHE_NAME" >/tmp/cachix-watch-store.log 2>&1 &
+ cachix_pid=$!
+ trap 'if [ -n "${cachix_pid:-}" ]; then kill "$cachix_pid" 2>/dev/null || true; wait "$cachix_pid" 2>/dev/null || true; fi' EXIT
+ fi
+ mkdir -p prebuilt
+ nix build --print-build-logs ".#deploy-service-binaries-${SERVICE}"
+ cp -r result/bin/* prebuilt/
+ mkdir -p prebuilt/nix-store
+ while IFS= read -r store_path; do
+ cp -a "$store_path" prebuilt/nix-store/
+ done < <(nix-store -qR result)
+ touch prebuilt/.keep
+ tar -C prebuilt -czf prebuilt-binaries.tar.gz .
+ # Receipt: the deploy job logs the same hash on read.
+ echo "handoff receipt: $(sha256sum prebuilt-binaries.tar.gz | cut -d' ' -f1) ($(stat -c%s prebuilt-binaries.tar.gz) bytes)"
+
+ shell: bash
+ env:
+ SERVICE: ${{ matrix.service }}
+ - name: Upload handoff artifact
+ run: nsc artifact upload prebuilt-binaries.tar.gz "$DEST" --expires_in=24h
+ shell: bash
+ env:
+ DEST: handoff/${{ github.run_id }}-${{ github.run_attempt }}/${{ matrix.service }}/prebuilt-binaries.tar.gz
+ - name: Teardown Nix
+ if: always()
+ uses: ./.github/actions/teardown-nix
+ build-lambda-artifacts:
+ needs:
+ - setup
+ - warm-lambdas
+ if: ${{ needs.setup.outputs.lambdas != '[]' }}
+ name: Build ${{ matrix.service }} Lambda artifacts
+ runs-on: namespace-profile-linux-mid
+ strategy:
+ matrix:
+ service: ${{ fromJson(needs.setup.outputs.lambdas) }}
+ fail-fast: false
+ steps:
+ - name: Checkout Repo
+ uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd
+ with:
+ persist-credentials: 'false'
+ clean: 'false'
+ - name: Mount /nix cache volume
+ uses: namespacelabs/nscloud-cache-action@15799a6b54e5765f85b2aac25b3f0df43ed571c0
+ with:
+ path: /nix
+ continue-on-error: true
+ - name: Setup Nix
+ uses: ./.github/actions/setup-nix
+ - name: Configure Cachix fallback
+ uses: ./.github/actions/setup-cachix
+ with:
+ cachix-auth-token: ${{ secrets.CACHIX_AUTH_TOKEN }}
+ - name: Build Lambda artifacts
+ run: .github/scripts/build-cloud-storage-lambdas-nix.sh
+ env:
+ SERVICE: ${{ matrix.service }}
+ - name: Log handoff receipt
+ run: |
+ set -euo pipefail
+ # The build script writes lambda-artifacts.tar.gz to the workspace
+ # root; the deploy job logs the same hash on read.
+ echo "handoff receipt: $(sha256sum lambda-artifacts.tar.gz | cut -d' ' -f1) ($(stat -c%s lambda-artifacts.tar.gz) bytes)"
+ shell: bash
+ - name: Upload handoff artifact
+ run: nsc artifact upload lambda-artifacts.tar.gz "$DEST" --expires_in=24h
+ shell: bash
+ env:
+ DEST: handoff/${{ github.run_id }}-${{ github.run_attempt }}/${{ matrix.service }}/lambda-artifacts.tar.gz
+ - name: Teardown Nix
+ if: always()
+ uses: ./.github/actions/teardown-nix
+ migrate-db:
+ needs:
+ - setup
+ - warm-binaries
+ - warm-lambdas
+ - build-service-binaries
+ - build-lambda-artifacts
+ if: ${{ !cancelled() && needs.setup.outputs.matrix != '[]' && !contains(needs.*.result, 'failure') && !contains(needs.*.result, 'cancelled') }}
+ name: Run Database Migrations
+ runs-on: db-migrator
+ steps:
+ - name: Checkout Repo
+ uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd
+ with:
+ persist-credentials: 'false'
+ sparse-checkout: .github/
+ - name: Run migrations
+ uses: ./.github/actions/migrate-cloud-storage-db
+ with:
+ environment: ${{ inputs.environment }}
+ deploy-services:
+ needs:
+ - setup
+ - warm-binaries
+ - warm-lambdas
+ - build-service-binaries
+ - build-lambda-artifacts
+ - migrate-db
+ if: ${{ !cancelled() && needs.setup.outputs.matrix != '[]' && needs.migrate-db.result == 'success' && !contains(needs.*.result, 'failure') && !contains(needs.*.result, 'cancelled') }}
+ name: Deploy ${{ matrix.service }}
+ runs-on: namespace-profile-linux-small
+ env:
+ PULUMI_HOME: /pulumi
+ strategy:
+ matrix:
+ service: ${{ fromJson(needs.setup.outputs.matrix) }}
+ fail-fast: false
+ steps:
+ - name: Checkout Repo
+ uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd
+ with:
+ persist-credentials: 'false'
+ - id: project-name
+ name: Get project name
+ uses: ./.github/actions/get-project-name
+ with:
+ service-name: ${{ matrix.service }}
+ - id: check-artifacts
+ name: Check artifact config
+ run: |
+ has_binaries=$(jq -r --arg service "$SERVICE" '((.services[$service].deploy_binaries // []) | length) > 0' .github/services-config.json)
+ has_lambdas=$(jq -r --arg service "$SERVICE" '((.services[$service].deploy_lambdas // []) | length) > 0' .github/services-config.json)
+ echo "has_binaries=$has_binaries" >> "$GITHUB_OUTPUT"
+ echo "has_lambdas=$has_lambdas" >> "$GITHUB_OUTPUT"
+ env:
+ SERVICE: ${{ matrix.service }}
+ - name: Download handoff artifacts
+ if: ${{ steps.check-artifacts.outputs.has_binaries == 'true' || steps.check-artifacts.outputs.has_lambdas == 'true' }}
+ run: |
+ set -euo pipefail
+ if ! command -v nsc >/dev/null 2>&1; then
+ echo "::error::nsc CLI not found — this job expects a Namespace runner (or add namespacelabs/nscloud-setup)"
+ exit 1
+ fi
+ mkdir -p "$RUNNER_TEMP/handoff"
+ if [[ "$HAS_BINARIES" == "true" ]]; then
+ nsc artifact download "$BASE/prebuilt-binaries.tar.gz" "$RUNNER_TEMP/handoff/prebuilt-binaries.tar.gz"
+ fi
+ if [[ "$HAS_LAMBDAS" == "true" ]]; then
+ nsc artifact download "$BASE/lambda-artifacts.tar.gz" "$RUNNER_TEMP/handoff/lambda-artifacts.tar.gz"
+ fi
+ shell: bash
+ env:
+ HAS_BINARIES: ${{ steps.check-artifacts.outputs.has_binaries }}
+ HAS_LAMBDAS: ${{ steps.check-artifacts.outputs.has_lambdas }}
+ BASE: handoff/${{ github.run_id }}-${{ github.run_attempt }}/${{ matrix.service }}
+ - name: Cache Pulumi plugins
+ uses: namespacelabs/nscloud-cache-action@15799a6b54e5765f85b2aac25b3f0df43ed571c0
+ with:
+ path: /pulumi/plugins
+ continue-on-error: true
+ - name: Ensure Pulumi home is writable
+ run: |
+ set -euo pipefail
+ sudo mkdir -p /pulumi
+ if [ "$(id -u)" -ne 0 ]; then sudo chown -R "$(id -u):$(id -g)" /pulumi; fi
+ shell: bash
+ - name: Deploy ${{ matrix.service }}
+ uses: ./.github/actions/deploy-cloud-storage-pulumi
+ with:
+ environment: ${{ inputs.environment }}
+ aws-access-key: ${{ secrets.AWS_ACCESS_KEY }}
+ aws-secret-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+ pulumi-access-token: ${{ secrets.PULUMI_ACCESS_TOKEN }}
+ pulumi-service-name: ${{ steps.project-name.outputs.project-name }}
+ use-namespace-builder: 'true'
+ prebuilt-binaries-tar: ${{ steps.check-artifacts.outputs.has_binaries == 'true' && format('{0}/handoff/prebuilt-binaries.tar.gz', runner.temp) || '' }}
+ lambda-artifacts-tar: ${{ steps.check-artifacts.outputs.has_lambdas == 'true' && format('{0}/handoff/lambda-artifacts.tar.gz', runner.temp) || '' }}
+ dd-app-key: ${{ secrets.DD_APP_KEY }}
+ dd-api-key: ${{ secrets.DD_API_KEY }}
+ deployment-summary:
+ needs:
+ - setup
+ - warm-binaries
+ - warm-lambdas
+ - build-service-binaries
+ - build-lambda-artifacts
+ - migrate-db
+ - deploy-services
+ if: always()
+ name: Deployment Summary
+ runs-on: namespace-profile-linux-tiny-no-cache
+ steps:
+ - name: Check deployment results
+ run: |
+ if [[ "${{ needs.setup.result }}" == "failure" ]]; then
+ echo "❌ Deployment setup failed"
+ exit 1
+ elif [[ "${{ needs.setup.result }}" == "skipped" ]]; then
+ echo "⏭️ Deployment setup was skipped"
+ elif [[ "${{ needs.warm-binaries.result }}" == "failure" || "${{ needs.warm-lambdas.result }}" == "failure" ]]; then
+ echo "❌ Warming shared deps onto a sticky disk failed"
+ exit 1
+ elif [[ "${{ needs.build-service-binaries.result }}" == "failure" ]]; then
+ echo "❌ One or more service binary builds failed"
+ exit 1
+ elif [[ "${{ needs.build-lambda-artifacts.result }}" == "failure" ]]; then
+ echo "❌ One or more Lambda artifact builds failed"
+ exit 1
+ elif [[ "${{ needs.migrate-db.result }}" == "failure" ]]; then
+ echo "❌ Database migrations failed"
+ exit 1
+ elif [[ "${{ needs.deploy-services.result }}" == "failure" ]]; then
+ echo "❌ One or more service deployments failed"
+ exit 1
+ elif [[ "${{ needs.deploy-services.result }}" == "skipped" ]]; then
+ echo "⏭️ No services to deploy"
+ else
+ echo "✅ All service deployments completed successfully for $ENVIRONMENT environment"
+ fi
+ shell: bash
+ env:
+ ENVIRONMENT: ${{ inputs.environment }}
+concurrency:
+ group: deploy-all-services-${{ inputs.environment }}
+ cancel-in-progress: false
diff --git a/.github/workflows/deploy_cloud_storage_on_push.yml b/.github/workflows/deploy_cloud_storage_on_push.yml
new file mode 100644
index 0000000000..001b943852
--- /dev/null
+++ b/.github/workflows/deploy_cloud_storage_on_push.yml
@@ -0,0 +1,52 @@
+# DO NOT EDIT — regenerate with `cargo x workflows` (from the repository root).
+# Source: tooling/xtask/crates/xtask_workflows/src/workflows/deploy_cloud_storage_on_push.rs
+name: Deploy Cloud Storage Services on Push
+on:
+ push:
+ branches:
+ - main
+ paths:
+ - Cargo.toml
+ - Cargo.lock
+ - Cross.toml
+ - clippy.toml
+ - deny.toml
+ - rust-toolchain.toml
+ - .cargo/**
+ - .config/**
+ - .sqlx/**
+ - crates/**
+ - services/**
+ - tooling/xtask/**
+ - tooling/just/**
+ - tooling/scripts/**
+ - static_assets/**
+ - docker/**
+ - nix/**
+ - nix-support/**
+ - infra/**
+ - .github/workflows/deploy_cloud_storage_on_push.yml
+ - .github/workflows/deploy_all_services.yml
+ - .github/actions/deploy-cloud-storage-pulumi/**
+ - .github/actions/setup-nix/**
+ - .github/actions/teardown-nix/**
+ - .github/actions/setup-cachix/**
+ - .github/actions/migrate-cloud-storage-db/**
+ - .github/scripts/build-cloud-storage-lambdas-nix.sh
+ - .github/services-config.json
+ - .github/workspace-dep-closures.json
+ - flake.nix
+ - flake.lock
+jobs:
+ deploy-all:
+ name: Deploy Cloud Storage Services
+ uses: ./.github/workflows/deploy_all_services.yml
+ with:
+ environment: dev
+ secrets:
+ AWS_ACCESS_KEY: ${{ secrets.AWS_ACCESS_KEY }}
+ AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+ PULUMI_ACCESS_TOKEN: ${{ secrets.PULUMI_ACCESS_TOKEN }}
+ DD_APP_KEY: ${{ secrets.DD_APP_KEY }}
+ DD_API_KEY: ${{ secrets.DD_API_KEY }}
+ CACHIX_AUTH_TOKEN: ${{ secrets.CACHIX_AUTH_TOKEN }}
diff --git a/.github/workflows/release-production.yml b/.github/workflows/release-production.yml
index b0e1074860..8e5de91f1b 100644
--- a/.github/workflows/release-production.yml
+++ b/.github/workflows/release-production.yml
@@ -13,7 +13,7 @@ env:
jobs:
deploy-cloud-storage:
name: Deploy Cloud Storage Services
- uses: ./.github/workflows/deploy-all-services.yml
+ uses: ./.github/workflows/deploy_all_services.yml
with:
environment: prod
secrets:
diff --git a/.github/workflows/reusable-deploy-service.yml b/.github/workflows/reusable-deploy-service.yml
deleted file mode 100644
index 21b382ea2c..0000000000
--- a/.github/workflows/reusable-deploy-service.yml
+++ /dev/null
@@ -1,282 +0,0 @@
-# Single-service deploy on the shared namespace + nix + crane pipeline — the
-# same warm sticky-disk nix builds, crane/cargo-zigbuild lambdas, nsc artifact
-# handoff, and Namespace Docker builder used by deploy-all-services, scoped to
-# one service. This replaced a pipeline that built binaries on Cachix-only
-# linux-extra-beefy runners, built lambdas per-handler, and handed off via the
-# GitHub artifacts API.
-#
-# deploy-all-services warms a shared dep closure once and fans a build matrix
-# out across it; with a single service there is no fan-out to amortise, so the
-# per-service build realises its own closure directly (the /nix sticky disk +
-# Cachix still substitute everything unchanged from prior runs).
-name: Reusable Deploy Service
-on:
- workflow_call:
- inputs:
- environment:
- required: true
- type: string
- description: The environment to deploy to
- service-name:
- required: true
- type: string
- description: The name of the service to deploy
- pulumi-stack-name:
- required: false
- type: string
- description: Override pulumi stack name (defaults to service-name)
- use-docker:
- required: false
- type: boolean
- default: true
- description: Whether to setup docker
- use-lfs:
- required: false
- type: boolean
- default: false
- description: Whether to checkout LFS content
- secrets:
- AWS_ACCESS_KEY:
- required: true
- AWS_SECRET_ACCESS_KEY:
- required: true
- PULUMI_ACCESS_TOKEN:
- required: true
- DD_APP_KEY:
- required: true
- DD_API_KEY:
- required: true
- CACHIX_AUTH_TOKEN:
- required: true
-
-jobs:
- # Resolve which artifact kinds this service produces so the heavy nix build
- # jobs only spin up a Namespace runner + /nix volume when there's something to
- # build (mirrors deploy-all-services' matrix filtering, for one service).
- setup:
- name: Check ${{ inputs.service-name }} artifacts
- runs-on: ubuntu-latest
- outputs:
- has_binaries: ${{ steps.check.outputs.has_binaries }}
- has_lambdas: ${{ steps.check.outputs.has_lambdas }}
- steps:
- # Pinned actions/checkout v6: https://github.com/actions/checkout/commit/de0fac2e4500dabe0009e67214ff5f5447ce83dd
- - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd
- with:
- sparse-checkout: .github/
- - name: Check artifact config
- id: check
- env:
- SERVICE: ${{ inputs.service-name }}
- run: |
- set -euo pipefail
- cfg=.github/services-config.json
- has_binaries=$(jq -r --arg service "$SERVICE" '((.services[$service].deploy_binaries // []) | length) > 0' "$cfg")
- has_lambdas=$(jq -r --arg service "$SERVICE" '((.services[$service].deploy_lambdas // []) | length) > 0' "$cfg")
- echo "has_binaries=$has_binaries" >> "$GITHUB_OUTPUT"
- echo "has_lambdas=$has_lambdas" >> "$GITHUB_OUTPUT"
- echo "service=$SERVICE binaries=$has_binaries lambdas=$has_lambdas"
-
- build-service-binaries:
- name: Build ${{ inputs.service-name }} binaries
- needs: setup
- if: ${{ needs.setup.outputs.has_binaries == 'true' }}
- runs-on: namespace-profile-linux-mid
- steps:
- - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd
- with:
- clean: false
-
- # Namespace cache volume backing /nix. Purely an optimization: if the
- # profile has no cache volumes (or the mount fails), setup-nix performs
- # a cold install and Cachix substitutes the warm layers — slower, never
- # wrong. Hence continue-on-error.
- - name: Mount /nix cache volume
- continue-on-error: true
- uses: namespacelabs/nscloud-cache-action@15799a6b54e5765f85b2aac25b3f0df43ed571c0 # v1.4.3
- with:
- path: /nix
-
- - name: Setup Nix
- uses: ./.github/actions/setup-nix
-
- - name: Configure Cachix fallback
- uses: ./.github/actions/setup-cachix
- with:
- cachix-auth-token: ${{ secrets.CACHIX_AUTH_TOKEN }}
-
- - name: Build prebuilt binaries
- shell: bash
- env:
- SERVICE: ${{ inputs.service-name }}
- run: |
- set -euo pipefail
- mkdir -p prebuilt
- cachix_pid=
- if command -v cachix >/dev/null 2>&1 && [ -n "${CACHIX_CACHE_NAME:-}" ]; then
- cachix watch-store "$CACHIX_CACHE_NAME" >/tmp/cachix-watch-store.log 2>&1 &
- cachix_pid=$!
- trap 'if [ -n "${cachix_pid:-}" ]; then kill "$cachix_pid" 2>/dev/null || true; wait "$cachix_pid" 2>/dev/null || true; fi' EXIT
- fi
- nix build --print-build-logs ".#deploy-service-binaries-${SERVICE}"
- cp -r result/bin/* prebuilt/
- mkdir -p prebuilt/nix-store
- while IFS= read -r store_path; do
- cp -a "$store_path" prebuilt/nix-store/
- done < <(nix-store -qR result)
- touch prebuilt/.keep
- tar -C prebuilt -czf prebuilt-binaries.tar.gz .
- # Receipt: the deploy job logs the same hash on read.
- echo "handoff receipt: $(sha256sum prebuilt-binaries.tar.gz | cut -d' ' -f1) ($(stat -c%s prebuilt-binaries.tar.gz) bytes)"
-
- # Handoff to the deploy job via Namespace artifact storage: strongly
- # consistent object storage (no snapshot-visibility races), rides
- # Namespace's network rather than the GitHub artifacts API. Attempt-scoped
- # path so re-runs never collide with stale uploads.
- - name: Upload handoff artifact
- shell: bash
- env:
- DEST: handoff/${{ github.run_id }}-${{ github.run_attempt }}/${{ inputs.service-name }}/prebuilt-binaries.tar.gz
- run: nsc artifact upload prebuilt-binaries.tar.gz "$DEST" --expires_in=24h
-
- - name: Teardown Nix (free cache volume for commit)
- if: always()
- uses: ./.github/actions/teardown-nix
-
- build-lambda-artifacts:
- name: Build ${{ inputs.service-name }} Lambda artifacts
- needs: setup
- if: ${{ needs.setup.outputs.has_lambdas == 'true' }}
- # Each handler is a crane + cargo-zigbuild nix package; the build script runs
- # `nix build .#deploy-lambda-<name>` for the service's handlers. Unchanged
- # handlers are pure cache hits (substituted from the /nix disk or Cachix).
- runs-on: namespace-profile-linux-mid
- steps:
- - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd
- with:
- clean: false
-
- # See note on the binary build's volume mount.
- - name: Mount /nix cache volume
- continue-on-error: true
- uses: namespacelabs/nscloud-cache-action@15799a6b54e5765f85b2aac25b3f0df43ed571c0 # v1.4.3
- with:
- path: /nix
-
- - name: Setup Nix
- uses: ./.github/actions/setup-nix
-
- - name: Configure Cachix fallback
- uses: ./.github/actions/setup-cachix
- with:
- cachix-auth-token: ${{ secrets.CACHIX_AUTH_TOKEN }}
-
- - name: Build Lambda artifacts
- env:
- SERVICE: ${{ inputs.service-name }}
- run: .github/scripts/build-cloud-storage-lambdas-nix.sh
-
- - name: Log handoff receipt
- shell: bash
- run: |
- set -euo pipefail
- # The build script writes lambda-artifacts.tar.gz to the workspace
- # root; the deploy job logs the same hash on read.
- echo "handoff receipt: $(sha256sum lambda-artifacts.tar.gz | cut -d' ' -f1) ($(stat -c%s lambda-artifacts.tar.gz) bytes)"
-
- # Handoff via Namespace artifact storage (see build-service-binaries).
- - name: Upload handoff artifact
- shell: bash
- env:
- DEST: handoff/${{ github.run_id }}-${{ github.run_attempt }}/${{ inputs.service-name }}/lambda-artifacts.tar.gz
- run: nsc artifact upload lambda-artifacts.tar.gz "$DEST" --expires_in=24h
-
- - name: Teardown Nix (free cache volume for commit)
- if: always()
- uses: ./.github/actions/teardown-nix
-
- deploy:
- name: Deploy ${{ inputs.service-name }}
- needs: [setup, build-service-binaries, build-lambda-artifacts]
- # A build job is skipped when the service has no binaries/lambdas (and a
- # service may legitimately have neither). Proceed as long as nothing
- # actually failed or was cancelled — skipped needs are expected here.
- if: ${{ !failure() && !cancelled() }}
- # Deploys via Pulumi + Docker; AWS auth is via explicit static keys
- # (configure-aws-credentials).
- runs-on: namespace-profile-linux-small
- # Pin PULUMI_HOME to a fixed path (outside the workspace, which the deploy
- # action's checkout cleans) so the plugins subdir can be backed by a sticky
- # disk. Propagates into the composite action's pulumi step via the job env.
- env:
- PULUMI_HOME: /pulumi
- steps:
- - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd
-
- - name: Get project name
- id: project-name
- uses: ./.github/actions/get-project-name
- with:
- service-name: ${{ inputs.service-name }}
-
- # Pull the handoff tars from Namespace artifact storage into runner.temp
- # (outside the workspace, which the composite action's checkout cleans).
- # The composite's tar-path branch handles receipts + the extract guard.
- - name: Download handoff artifacts
- if: ${{ needs.setup.outputs.has_binaries == 'true' || needs.setup.outputs.has_lambdas == 'true' }}
- shell: bash
- env:
- HAS_BINARIES: ${{ needs.setup.outputs.has_binaries }}
- HAS_LAMBDAS: ${{ needs.setup.outputs.has_lambdas }}
- BASE: handoff/${{ github.run_id }}-${{ github.run_attempt }}/${{ inputs.service-name }}
- run: |
- set -euo pipefail
- if ! command -v nsc >/dev/null 2>&1; then
- echo "::error::nsc CLI not found — this job expects a Namespace runner (or add namespacelabs/nscloud-setup)"
- exit 1
- fi
- mkdir -p "$RUNNER_TEMP/handoff"
- if [[ "$HAS_BINARIES" == "true" ]]; then
- nsc artifact download "$BASE/prebuilt-binaries.tar.gz" "$RUNNER_TEMP/handoff/prebuilt-binaries.tar.gz"
- fi
- if [[ "$HAS_LAMBDAS" == "true" ]]; then
- nsc artifact download "$BASE/lambda-artifacts.tar.gz" "$RUNNER_TEMP/handoff/lambda-artifacts.tar.gz"
- fi
-
- # Cache Pulumi's provider plugins ($PULUMI_HOME/plugins) on a Namespace
- # cache volume. Plugins are version-pinned by infra/ and identical across
- # services. Optimization-only: a cold volume just re-downloads (~45s).
- - name: Cache Pulumi plugins
- continue-on-error: true
- uses: namespacelabs/nscloud-cache-action@15799a6b54e5765f85b2aac25b3f0df43ed571c0 # v1.4.3
- with:
- path: /pulumi/plugins
-
- # PULUMI_HOME (/pulumi) and its mounted plugins subdir come up root-owned;
- # make them writable so pulumi can write credentials + plugins as the
- # runner. No-op when the runner already runs as root. mkdir -p first: the
- # plugin cache mount is continue-on-error, so on a mount failure /pulumi
- # may not exist yet — create it so the cold-cache fallback works instead
- # of the chown hard-failing the deploy.
- - name: Ensure Pulumi home is writable
- shell: bash
- run: |
- set -euo pipefail
- sudo mkdir -p /pulumi
- if [ "$(id -u)" -ne 0 ]; then sudo chown -R "$(id -u):$(id -g)" /pulumi; fi
-
- - name: Deploy ${{ inputs.service-name }}
- uses: ./.github/actions/deploy-cloud-storage-pulumi
- with:
- environment: ${{ inputs.environment }}
- aws-access-key: ${{ secrets.AWS_ACCESS_KEY }}
- aws-secret-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
- pulumi-access-token: ${{ secrets.PULUMI_ACCESS_TOKEN }}
- pulumi-service-name: ${{ inputs.pulumi-stack-name || steps.project-name.outputs.project-name }}
- use-namespace-builder: 'true'
- use-docker: ${{ inputs.use-docker }}
- use-lfs: ${{ inputs.use-lfs }}
- prebuilt-binaries-tar: ${{ needs.setup.outputs.has_binaries == 'true' && format('{0}/handoff/prebuilt-binaries.tar.gz', runner.temp) || '' }}
- lambda-artifacts-tar: ${{ needs.setup.outputs.has_lambdas == 'true' && format('{0}/handoff/lambda-artifacts.tar.gz', runner.temp) || '' }}
- dd-app-key: ${{ secrets.DD_APP_KEY }}
- dd-api-key: ${{ secrets.DD_API_KEY }}
diff --git a/.github/workflows/reusable_deploy_service.yml b/.github/workflows/reusable_deploy_service.yml
new file mode 100644
index 0000000000..868004f40f
--- /dev/null
+++ b/.github/workflows/reusable_deploy_service.yml
@@ -0,0 +1,229 @@
+# DO NOT EDIT — regenerate with `cargo x workflows` (from the repository root).
+# Source: tooling/xtask/crates/xtask_workflows/src/workflows/reusable_deploy_service.rs
+name: Reusable Deploy Service
+on:
+ workflow_call:
+ inputs:
+ environment:
+ required: true
+ type: string
+ description: The environment to deploy to
+ service-name:
+ required: true
+ type: string
+ description: The name of the service to deploy
+ pulumi-stack-name:
+ required: false
+ type: string
+ description: Override pulumi stack name (defaults to service-name)
+ use-docker:
+ required: false
+ type: boolean
+ default: true
+ description: Whether to setup docker
+ use-lfs:
+ required: false
+ type: boolean
+ default: false
+ description: Whether to checkout LFS content
+ secrets:
+ AWS_ACCESS_KEY:
+ required: true
+ AWS_SECRET_ACCESS_KEY:
+ required: true
+ PULUMI_ACCESS_TOKEN:
+ required: true
+ DD_APP_KEY:
+ required: true
+ DD_API_KEY:
+ required: true
+ CACHIX_AUTH_TOKEN:
+ required: true
+jobs:
+ setup:
+ name: Check ${{ inputs.service-name }} artifacts
+ runs-on: namespace-profile-linux-tiny-no-cache
+ outputs:
+ has_binaries: ${{ steps.check.outputs.has_binaries }}
+ has_lambdas: ${{ steps.check.outputs.has_lambdas }}
+ steps:
+ - name: Checkout Repo
+ uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd
+ with:
+ persist-credentials: 'false'
+ sparse-checkout: .github/
+ - id: check
+ name: Check artifact config
+ run: |
+ set -euo pipefail
+ cfg=.github/services-config.json
+ has_binaries=$(jq -r --arg service "$SERVICE" '((.services[$service].deploy_binaries // []) | length) > 0' "$cfg")
+ has_lambdas=$(jq -r --arg service "$SERVICE" '((.services[$service].deploy_lambdas // []) | length) > 0' "$cfg")
+ echo "has_binaries=$has_binaries" >> "$GITHUB_OUTPUT"
+ echo "has_lambdas=$has_lambdas" >> "$GITHUB_OUTPUT"
+ echo "service=$SERVICE binaries=$has_binaries lambdas=$has_lambdas"
+ env:
+ SERVICE: ${{ inputs.service-name }}
+ build-service-binaries:
+ needs:
+ - setup
+ if: ${{ needs.setup.outputs.has_binaries == 'true' }}
+ name: Build ${{ inputs.service-name }} binaries
+ runs-on: namespace-profile-linux-mid
+ steps:
+ - name: Checkout Repo
+ uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd
+ with:
+ persist-credentials: 'false'
+ clean: 'false'
+ - name: Mount /nix cache volume
+ uses: namespacelabs/nscloud-cache-action@15799a6b54e5765f85b2aac25b3f0df43ed571c0
+ with:
+ path: /nix
+ continue-on-error: true
+ - name: Setup Nix
+ uses: ./.github/actions/setup-nix
+ - name: Configure Cachix fallback
+ uses: ./.github/actions/setup-cachix
+ with:
+ cachix-auth-token: ${{ secrets.CACHIX_AUTH_TOKEN }}
+ - name: Build prebuilt binaries
+ run: |+
+ set -euo pipefail
+ cachix_pid=
+ if command -v cachix >/dev/null 2>&1 && [ -n "${CACHIX_CACHE_NAME:-}" ]; then
+ cachix watch-store "$CACHIX_CACHE_NAME" >/tmp/cachix-watch-store.log 2>&1 &
+ cachix_pid=$!
+ trap 'if [ -n "${cachix_pid:-}" ]; then kill "$cachix_pid" 2>/dev/null || true; wait "$cachix_pid" 2>/dev/null || true; fi' EXIT
+ fi
+ mkdir -p prebuilt
+ nix build --print-build-logs ".#deploy-service-binaries-${SERVICE}"
+ cp -r result/bin/* prebuilt/
+ mkdir -p prebuilt/nix-store
+ while IFS= read -r store_path; do
+ cp -a "$store_path" prebuilt/nix-store/
+ done < <(nix-store -qR result)
+ touch prebuilt/.keep
+ tar -C prebuilt -czf prebuilt-binaries.tar.gz .
+ # Receipt: the deploy job logs the same hash on read.
+ echo "handoff receipt: $(sha256sum prebuilt-binaries.tar.gz | cut -d' ' -f1) ($(stat -c%s prebuilt-binaries.tar.gz) bytes)"
+
+ shell: bash
+ env:
+ SERVICE: ${{ inputs.service-name }}
+ - name: Upload handoff artifact
+ run: nsc artifact upload prebuilt-binaries.tar.gz "$DEST" --expires_in=24h
+ shell: bash
+ env:
+ DEST: handoff/${{ github.run_id }}-${{ github.run_attempt }}/${{ inputs.service-name }}/prebuilt-binaries.tar.gz
+ - name: Teardown Nix
+ if: always()
+ uses: ./.github/actions/teardown-nix
+ build-lambda-artifacts:
+ needs:
+ - setup
+ if: ${{ needs.setup.outputs.has_lambdas == 'true' }}
+ name: Build ${{ inputs.service-name }} Lambda artifacts
+ runs-on: namespace-profile-linux-mid
+ steps:
+ - name: Checkout Repo
+ uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd
+ with:
+ persist-credentials: 'false'
+ clean: 'false'
+ - name: Mount /nix cache volume
+ uses: namespacelabs/nscloud-cache-action@15799a6b54e5765f85b2aac25b3f0df43ed571c0
+ with:
+ path: /nix
+ continue-on-error: true
+ - name: Setup Nix
+ uses: ./.github/actions/setup-nix
+ - name: Configure Cachix fallback
+ uses: ./.github/actions/setup-cachix
+ with:
+ cachix-auth-token: ${{ secrets.CACHIX_AUTH_TOKEN }}
+ - name: Build Lambda artifacts
+ run: .github/scripts/build-cloud-storage-lambdas-nix.sh
+ env:
+ SERVICE: ${{ inputs.service-name }}
+ - name: Log handoff receipt
+ run: |
+ set -euo pipefail
+ # The build script writes lambda-artifacts.tar.gz to the workspace
+ # root; the deploy job logs the same hash on read.
+ echo "handoff receipt: $(sha256sum lambda-artifacts.tar.gz | cut -d' ' -f1) ($(stat -c%s lambda-artifacts.tar.gz) bytes)"
+ shell: bash
+ - name: Upload handoff artifact
+ run: nsc artifact upload lambda-artifacts.tar.gz "$DEST" --expires_in=24h
+ shell: bash
+ env:
+ DEST: handoff/${{ github.run_id }}-${{ github.run_attempt }}/${{ inputs.service-name }}/lambda-artifacts.tar.gz
+ - name: Teardown Nix
+ if: always()
+ uses: ./.github/actions/teardown-nix
+ deploy:
+ needs:
+ - setup
+ - build-service-binaries
+ - build-lambda-artifacts
+ if: ${{ !failure() && !cancelled() }}
+ name: Deploy ${{ inputs.service-name }}
+ runs-on: namespace-profile-linux-small
+ env:
+ PULUMI_HOME: /pulumi
+ steps:
+ - name: Checkout Repo
+ uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd
+ with:
+ persist-credentials: 'false'
+ - id: project-name
+ name: Get project name
+ uses: ./.github/actions/get-project-name
+ with:
+ service-name: ${{ inputs.service-name }}
+ - name: Download handoff artifacts
+ if: ${{ needs.setup.outputs.has_binaries == 'true' || needs.setup.outputs.has_lambdas == 'true' }}
+ run: |
+ set -euo pipefail
+ if ! command -v nsc >/dev/null 2>&1; then
+ echo "::error::nsc CLI not found — this job expects a Namespace runner (or add namespacelabs/nscloud-setup)"
+ exit 1
+ fi
+ mkdir -p "$RUNNER_TEMP/handoff"
+ if [[ "$HAS_BINARIES" == "true" ]]; then
+ nsc artifact download "$BASE/prebuilt-binaries.tar.gz" "$RUNNER_TEMP/handoff/prebuilt-binaries.tar.gz"
+ fi
+ if [[ "$HAS_LAMBDAS" == "true" ]]; then
+ nsc artifact download "$BASE/lambda-artifacts.tar.gz" "$RUNNER_TEMP/handoff/lambda-artifacts.tar.gz"
+ fi
+ shell: bash
+ env:
+ HAS_BINARIES: ${{ needs.setup.outputs.has_binaries }}
+ HAS_LAMBDAS: ${{ needs.setup.outputs.has_lambdas }}
+ BASE: handoff/${{ github.run_id }}-${{ github.run_attempt }}/${{ inputs.service-name }}
+ - name: Cache Pulumi plugins
+ uses: namespacelabs/nscloud-cache-action@15799a6b54e5765f85b2aac25b3f0df43ed571c0
+ with:
+ path: /pulumi/plugins
+ continue-on-error: true
+ - name: Ensure Pulumi home is writable
+ run: |
+ set -euo pipefail
+ sudo mkdir -p /pulumi
+ if [ "$(id -u)" -ne 0 ]; then sudo chown -R "$(id -u):$(id -g)" /pulumi; fi
+ shell: bash
+ - name: Deploy ${{ inputs.service-name }}
+ uses: ./.github/actions/deploy-cloud-storage-pulumi
+ with:
+ environment: ${{ inputs.environment }}
+ aws-access-key: ${{ secrets.AWS_ACCESS_KEY }}
+ aws-secret-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+ pulumi-access-token: ${{ secrets.PULUMI_ACCESS_TOKEN }}
+ pulumi-service-name: ${{ inputs.pulumi-stack-name || steps.project-name.outputs.project-name }}
+ use-namespace-builder: 'true'
+ use-docker: ${{ inputs.use-docker }}
+ use-lfs: ${{ inputs.use-lfs }}
+ prebuilt-binaries-tar: ${{ needs.setup.outputs.has_binaries == 'true' && format('{0}/handoff/prebuilt-binaries.tar.gz', runner.temp) || '' }}
+ lambda-artifacts-tar: ${{ needs.setup.outputs.has_lambdas == 'true' && format('{0}/handoff/lambda-artifacts.tar.gz', runner.temp) || '' }}
+ dd-app-key: ${{ secrets.DD_APP_KEY }}
+ dd-api-key: ${{ secrets.DD_API_KEY }}
diff --git a/tooling/xtask/crates/xtask_workflows/src/workflows/cancel_stuck_cloud_storage_deploys.rs b/tooling/xtask/crates/xtask_workflows/src/workflows/cancel_stuck_cloud_storage_deploys.rs
index eada9e9956..f9b6fb3955 100644
--- a/tooling/xtask/crates/xtask_workflows/src/workflows/cancel_stuck_cloud_storage_deploys.rs
+++ b/tooling/xtask/crates/xtask_workflows/src/workflows/cancel_stuck_cloud_storage_deploys.rs
@@ -56,9 +56,9 @@ fn cancel_stuck() -> Job {
.name("Cancel runs stuck queued or in_progress past threshold")
.runs_on(runners::Runner::TinyNoCache.to_string())
.timeout_minutes(5u32)
- // Keep in sync with the deploy trigger's generated filename when the
- // deploy family is ported.
- .add_env(("WORKFLOW_FILE", "deploy-cloud-storage-on-push.yml"))
+ // Keep in sync with the deploy trigger's generated filename
+ // (see deploy_cloud_storage_on_push.rs).
+ .add_env(("WORKFLOW_FILE", "deploy_cloud_storage_on_push.yml"))
.add_env(("MAX_AGE_MINUTES", "${{ inputs.max-age-minutes || '90' }}"))
.add_env(("DRY_RUN", "${{ inputs.dry-run || 'false' }}"))
.add_env(("GH_TOKEN", "${{ github.token }}"))
diff --git a/tooling/xtask/crates/xtask_workflows/src/workflows/deploy_all_services.rs b/tooling/xtask/crates/xtask_workflows/src/workflows/deploy_all_services.rs
new file mode 100644
index 0000000000..5859abe8fa
--- /dev/null
+++ b/tooling/xtask/crates/xtask_workflows/src/workflows/deploy_all_services.rs
@@ -0,0 +1,463 @@
+//! `Deploy All Services` — the shared cloud-storage deploy pipeline: warm the
+//! two shared dep closures onto sticky disks, fan a build matrix out per
+//! service (binaries via crane, lambdas via crane + cargo-zigbuild), hand the
+//! artifacts off through Namespace artifact storage, run DB migrations, then
+//! deploy every service via Pulumi. Called by `deploy_cloud_storage_on_push`
+//! (dev) and `release-production` (prod), and manually dispatchable.
+//! Generated into `deploy_all_services.yml` (replaces the hand-written
+//! `deploy-all-services.yml`).
+
+use anyhow::Result;
+use gh_workflow::{
+ Concurrency, Env, Event, Expression, Job, Run, Step, Strategy, Use, Workflow, WorkflowCall,
+ WorkflowDispatch,
+};
+
+use crate::workflows::{runners, steps, vars};
+
+/// The in-VPC self-hosted runner with network access to the databases. Stays
+/// off Namespace deliberately — migrations need to reach RDS.
+const DB_MIGRATOR_RUNNER: &str = "db-migrator";
+
+/// Build the workflow. The dispatch/call input blocks are filled in by
+/// [`patch`].
+pub fn deploy_all_services() -> Workflow {
+ Workflow::new("Deploy All Services")
+ .on(Event::default()
+ .workflow_dispatch(WorkflowDispatch::default())
+ .workflow_call(WorkflowCall::default()))
+ .concurrency(
+ // Only run 1 deploy-all workflow at a time per environment.
+ // Literal prefix: for workflow_call runs `github.workflow` expands
+ // to the *caller's* name, which would split dispatches and callers
+ // into separate groups and let them race the same Pulumi stacks.
+ // Groups are repo-global strings, so the on-push caller shares
+ // this group by using the same literal.
+ Concurrency::new(Expression::new(
+ "deploy-all-services-${{ inputs.environment }}",
+ ))
+ .cancel_in_progress(false),
+ )
+ .add_job("setup", setup())
+ .add_job(
+ "warm-binaries",
+ warm_job(
+ "Warm shared binary deps onto sticky disk",
+ "binaries",
+ "Warm shared release deps",
+ "\".#deployCargoArtifacts\"",
+ "Shared release deps realised into /nix/store; committed on job exit.",
+ ),
+ )
+ .add_job(
+ "warm-lambdas",
+ warm_job(
+ "Warm shared lambda deps onto sticky disk",
+ "lambdas",
+ "Warm shared lambda dep closure",
+ "\".#lambdaDeployCargoArtifacts\" \".#callRecordingPreviewFfmpegLayer\"",
+ "Shared lambda deps realised into /nix/store; committed on job exit.",
+ ),
+ )
+ .add_job("build-service-binaries", build_service_binaries())
+ .add_job("build-lambda-artifacts", build_lambda_artifacts())
+ .add_job("migrate-db", migrate_db())
+ .add_job("deploy-services", deploy_services())
+ .add_job("deployment-summary", deployment_summary())
+}
+
+/// Fill in the ordered dispatch/call input blocks.
+pub fn patch(root: &mut serde_yaml::Value) -> Result<()> {
+ let on = root
+ .get_mut("on")
+ .and_then(serde_yaml::Value::as_mapping_mut)
+ .ok_or_else(|| anyhow::anyhow!("rendered workflow has no `on` mapping"))?;
+ on.insert(
+ "workflow_dispatch".into(),
+ crate::workflows::yaml_fragment(indoc::indoc! {r#"
+ inputs:
+ environment:
+ type: choice
+ required: true
+ default: 'dev'
+ options:
+ - dev
+ - prod
+ description: The environment we are deploying to.
+ "#})?,
+ );
+ on.insert(
+ "workflow_call".into(),
+ crate::workflows::yaml_fragment(indoc::indoc! {r#"
+ inputs:
+ environment:
+ type: string
+ required: true
+ description: The environment we are deploying to.
+ secrets:
+ AWS_ACCESS_KEY:
+ required: true
+ AWS_SECRET_ACCESS_KEY:
+ required: true
+ PULUMI_ACCESS_TOKEN:
+ required: true
+ DD_APP_KEY:
+ required: true
+ DD_API_KEY:
+ required: true
+ CACHIX_AUTH_TOKEN:
+ required: true
+ "#})?,
+ );
+ Ok(())
+}
+
+fn setup() -> Job {
+ Job::default()
+ .name("Setup Deployment Matrix")
+ .runs_on(runners::Runner::TinyNoCache.to_string())
+ .add_output("matrix", "${{ steps.set-matrix.outputs.matrix }}")
+ .add_output("binaries", "${{ steps.set-matrix.outputs.binaries }}")
+ .add_output("lambdas", "${{ steps.set-matrix.outputs.lambdas }}")
+ .add_step(steps::checkout_v4())
+ .add_step(set_matrix())
+}
+
+fn set_matrix() -> Step<Run> {
+ Step::new("Set deployment matrix")
+ .run(indoc::indoc! {r#"
+ set -euo pipefail
+ cfg=.github/services-config.json
+ # Full list drives deploy-services; the filtered lists keep each build
+ # matrix to only the services that actually produce that artifact, so
+ # no build job spins up Nix + a cache volume for nothing.
+ services=$(jq -c '.services | keys' "$cfg")
+ binaries=$(jq -c '[.services | to_entries[] | select((.value.deploy_binaries // []) | length > 0) | .key]' "$cfg")
+ lambdas=$(jq -c '[.services | to_entries[] | select((.value.deploy_lambdas // []) | length > 0) | .key]' "$cfg")
+ echo "matrix=${services}" >> "$GITHUB_OUTPUT"
+ echo "binaries=${binaries}" >> "$GITHUB_OUTPUT"
+ echo "lambdas=${lambdas}" >> "$GITHUB_OUTPUT"
+ echo "All services: ${services}"
+ echo "With binaries: ${binaries}"
+ echo "With lambdas: ${lambdas}"
+ "#})
+ .id("set-matrix")
+}
+
+/// Warm one shared dep closure onto the /nix sticky disk. The two warm jobs
+/// run in parallel and each build matrix depends only on its own warm job, so
+/// a slow lambda-closure warm never blocks binary builds and vice versa. The
+/// /nix cache volume is an optimization; Cachix is the consistency backstop
+/// either way.
+fn warm_job(
+ name: &str,
+ gate_output: &str,
+ build_step_name: &str,
+ targets: &str,
+ done_msg: &str,
+) -> Job {
+ Job::default()
+ .name(name)
+ .needs(vec!["setup".to_string()])
+ .cond(Expression::new(format!(
+ "${{{{ needs.setup.outputs.{gate_output} != '[]' }}}}"
+ )))
+ .runs_on(runners::Runner::Mid.to_string())
+ .add_step(steps::checkout_v4())
+ .add_step(steps::mount_nix_cache_volume())
+ .add_step(steps::setup_nix())
+ .add_step(steps::setup_cachix())
+ .add_step(steps::nix_build_watched(build_step_name, targets, done_msg))
+ .add_step(steps::teardown_nix())
+}
+
+/// Base for the per-service build matrix jobs: clones the warm /nix cache
+/// volume populated by the matching warm job, so only each service's own
+/// crate compiles (cold volume -> Cachix substitution). No max-parallel:
+/// runners autoscale, so all services build concurrently.
+fn build_job(name: &str, warm_job_id: &str, gate_output: &str) -> Job {
+ Job::default()
+ .name(name)
+ .needs(vec!["setup".to_string(), warm_job_id.to_string()])
+ .cond(Expression::new(format!(
+ "${{{{ needs.setup.outputs.{gate_output} != '[]' }}}}"
+ )))
+ .runs_on(runners::Runner::Mid.to_string())
+ .strategy(Strategy {
+ fail_fast: Some(false),
+ matrix: Some(serde_json::json!({
+ "service": format!("${{{{ fromJson(needs.setup.outputs.{gate_output}) }}}}"),
+ })),
+ max_parallel: None,
+ })
+ .add_step(steps::checkout_v4().add_with(("clean", false)))
+ .add_step(steps::mount_nix_cache_volume())
+ .add_step(steps::setup_nix())
+ .add_step(steps::setup_cachix())
+}
+
+fn build_service_binaries() -> Job {
+ build_job(
+ "Build ${{ matrix.service }} binaries",
+ "warm-binaries",
+ "binaries",
+ )
+ .add_step(build_prebuilt_binaries())
+ .add_step(steps::upload_handoff_artifact(
+ "prebuilt-binaries.tar.gz",
+ "${{ matrix.service }}",
+ ))
+ .add_step(steps::teardown_nix())
+}
+
+fn build_prebuilt_binaries() -> Step<Run> {
+ let script = steps::with_cachix_watch(indoc::indoc! {r#"
+ mkdir -p prebuilt
+ nix build --print-build-logs ".#deploy-service-binaries-${SERVICE}"
+ cp -r result/bin/* prebuilt/
+ mkdir -p prebuilt/nix-store
+ while IFS= read -r store_path; do
+ cp -a "$store_path" prebuilt/nix-store/
+ done < <(nix-store -qR result)
+ touch prebuilt/.keep
+ tar -C prebuilt -czf prebuilt-binaries.tar.gz .
+ # Receipt: the deploy job logs the same hash on read.
+ echo "handoff receipt: $(sha256sum prebuilt-binaries.tar.gz | cut -d' ' -f1) ($(stat -c%s prebuilt-binaries.tar.gz) bytes)"
+ "#});
+ Step::new("Build prebuilt binaries")
+ .run(script)
+ .shell("bash")
+ .add_env(Env::new("SERVICE", "${{ matrix.service }}"))
+}
+
+/// Each handler is a crane + cargo-zigbuild nix package. This job builds all
+/// of a service's handlers via `nix build .#deploy-lambda-<name>`, cloning the
+/// warm lambda /nix disk committed by warm-lambdas. Unchanged handlers are
+/// pure cache hits (substituted from the disk/Cachix, no recompile).
+fn build_lambda_artifacts() -> Job {
+ build_job(
+ "Build ${{ matrix.service }} Lambda artifacts",
+ "warm-lambdas",
+ "lambdas",
+ )
+ .add_step(build_lambdas())
+ .add_step(log_lambda_receipt())
+ .add_step(steps::upload_handoff_artifact(
+ "lambda-artifacts.tar.gz",
+ "${{ matrix.service }}",
+ ))
+ .add_step(steps::teardown_nix())
+}
+
+fn build_lambdas() -> Step<Run> {
+ Step::new("Build Lambda artifacts")
+ .run(".github/scripts/build-cloud-storage-lambdas-nix.sh")
+ .add_env(Env::new("SERVICE", "${{ matrix.service }}"))
+}
+
+fn log_lambda_receipt() -> Step<Run> {
+ Step::new("Log handoff receipt")
+ .run(indoc::indoc! {r#"
+ set -euo pipefail
+ # The build script writes lambda-artifacts.tar.gz to the workspace
+ # root; the deploy job logs the same hash on read.
+ echo "handoff receipt: $(sha256sum lambda-artifacts.tar.gz | cut -d' ' -f1) ($(stat -c%s lambda-artifacts.tar.gz) bytes)"
+ "#})
+ .shell("bash")
+}
+
+fn migrate_db() -> Job {
+ Job::default()
+ .name("Run Database Migrations")
+ .needs(vec![
+ "setup".to_string(),
+ "warm-binaries".to_string(),
+ "warm-lambdas".to_string(),
+ "build-service-binaries".to_string(),
+ "build-lambda-artifacts".to_string(),
+ ])
+ .cond(Expression::new(
+ "${{ !cancelled() && needs.setup.outputs.matrix != '[]' && !contains(needs.*.result, 'failure') && !contains(needs.*.result, 'cancelled') }}",
+ ))
+ .runs_on(DB_MIGRATOR_RUNNER)
+ .add_step(steps::checkout_v4().add_with(("sparse-checkout", ".github/")))
+ .add_step(run_migrations())
+}
+
+fn run_migrations() -> Step<Use> {
+ steps::uses_local(
+ "Run migrations",
+ xtask_paths::repo_dir!(".github/actions/migrate-cloud-storage-db"),
+ )
+ .add_with(("environment", "${{ inputs.environment }}"))
+}
+
+/// Deploys via Pulumi + Docker; AWS auth is via explicit static keys
+/// (configure-aws-credentials). PULUMI_HOME is pinned to a fixed path (outside
+/// the workspace, which the deploy action's checkout cleans) so the plugins
+/// subdir can be backed by a sticky disk; it propagates into the composite
+/// action's pulumi step via the job env.
+fn deploy_services() -> Job {
+ Job::default()
+ .name("Deploy ${{ matrix.service }}")
+ .needs(vec![
+ "setup".to_string(),
+ "warm-binaries".to_string(),
+ "warm-lambdas".to_string(),
+ "build-service-binaries".to_string(),
+ "build-lambda-artifacts".to_string(),
+ "migrate-db".to_string(),
+ ])
+ .cond(Expression::new(
+ "${{ !cancelled() && needs.setup.outputs.matrix != '[]' && needs.migrate-db.result == 'success' && !contains(needs.*.result, 'failure') && !contains(needs.*.result, 'cancelled') }}",
+ ))
+ .runs_on(runners::Runner::Small.to_string())
+ .add_env(("PULUMI_HOME", "/pulumi"))
+ .strategy(Strategy {
+ fail_fast: Some(false),
+ matrix: Some(serde_json::json!({
+ "service": "${{ fromJson(needs.setup.outputs.matrix) }}",
+ })),
+ max_parallel: None,
+ })
+ .add_step(steps::checkout_v4())
+ .add_step(get_project_name())
+ .add_step(check_artifact_config())
+ .add_step(download_handoff_artifacts())
+ .add_step(steps::cache_pulumi_plugins())
+ .add_step(steps::ensure_pulumi_home_writable())
+ .add_step(deploy_service())
+}
+
+fn get_project_name() -> Step<Use> {
+ steps::uses_local(
+ "Get project name",
+ xtask_paths::repo_dir!(".github/actions/get-project-name"),
+ )
+ .id("project-name")
+ .add_with(("service-name", "${{ matrix.service }}"))
+}
+
+fn check_artifact_config() -> Step<Run> {
+ Step::new("Check artifact config")
+ .run(indoc::indoc! {r#"
+ has_binaries=$(jq -r --arg service "$SERVICE" '((.services[$service].deploy_binaries // []) | length) > 0' .github/services-config.json)
+ has_lambdas=$(jq -r --arg service "$SERVICE" '((.services[$service].deploy_lambdas // []) | length) > 0' .github/services-config.json)
+ echo "has_binaries=$has_binaries" >> "$GITHUB_OUTPUT"
+ echo "has_lambdas=$has_lambdas" >> "$GITHUB_OUTPUT"
+ "#})
+ .id("check-artifacts")
+ .add_env(Env::new("SERVICE", "${{ matrix.service }}"))
+}
+
+/// Pull the handoff tars from Namespace artifact storage into runner.temp
+/// (outside the workspace, which the composite action's checkout cleans).
+/// The composite's tar-path branch handles receipts + the extract guard.
+fn download_handoff_artifacts() -> Step<Run> {
+ Step::new("Download handoff artifacts")
+ .run(indoc::indoc! {r#"
+ set -euo pipefail
+ if ! command -v nsc >/dev/null 2>&1; then
+ echo "::error::nsc CLI not found — this job expects a Namespace runner (or add namespacelabs/nscloud-setup)"
+ exit 1
+ fi
+ mkdir -p "$RUNNER_TEMP/handoff"
+ if [[ "$HAS_BINARIES" == "true" ]]; then
+ nsc artifact download "$BASE/prebuilt-binaries.tar.gz" "$RUNNER_TEMP/handoff/prebuilt-binaries.tar.gz"
+ fi
+ if [[ "$HAS_LAMBDAS" == "true" ]]; then
+ nsc artifact download "$BASE/lambda-artifacts.tar.gz" "$RUNNER_TEMP/handoff/lambda-artifacts.tar.gz"
+ fi
+ "#})
+ .if_condition(Expression::new(
+ "${{ steps.check-artifacts.outputs.has_binaries == 'true' || steps.check-artifacts.outputs.has_lambdas == 'true' }}",
+ ))
+ .shell("bash")
+ .add_env(Env::new(
+ "HAS_BINARIES",
+ "${{ steps.check-artifacts.outputs.has_binaries }}",
+ ))
+ .add_env(Env::new(
+ "HAS_LAMBDAS",
+ "${{ steps.check-artifacts.outputs.has_lambdas }}",
+ ))
+ .add_env(Env::new(
+ "BASE",
+ "handoff/${{ github.run_id }}-${{ github.run_attempt }}/${{ matrix.service }}",
+ ))
+}
+
+fn deploy_service() -> Step<Use> {
+ steps::uses_local(
+ "Deploy ${{ matrix.service }}",
+ xtask_paths::repo_dir!(".github/actions/deploy-cloud-storage-pulumi"),
+ )
+ .add_with(("environment", "${{ inputs.environment }}"))
+ .add_with(("aws-access-key", vars::AWS_ACCESS_KEY))
+ .add_with(("aws-secret-key", vars::AWS_SECRET_ACCESS_KEY))
+ .add_with(("pulumi-access-token", vars::PULUMI_ACCESS_TOKEN))
+ .add_with((
+ "pulumi-service-name",
+ "${{ steps.project-name.outputs.project-name }}",
+ ))
+ .add_with(("use-namespace-builder", "true"))
+ .add_with((
+ "prebuilt-binaries-tar",
+ "${{ steps.check-artifacts.outputs.has_binaries == 'true' && format('{0}/handoff/prebuilt-binaries.tar.gz', runner.temp) || '' }}",
+ ))
+ .add_with((
+ "lambda-artifacts-tar",
+ "${{ steps.check-artifacts.outputs.has_lambdas == 'true' && format('{0}/handoff/lambda-artifacts.tar.gz', runner.temp) || '' }}",
+ ))
+ .add_with(("dd-app-key", vars::DD_APP_KEY))
+ .add_with(("dd-api-key", vars::DD_API_KEY))
+}
+
+fn deployment_summary() -> Job {
+ Job::default()
+ .name("Deployment Summary")
+ .runs_on(runners::Runner::TinyNoCache.to_string())
+ .needs(vec![
+ "setup".to_string(),
+ "warm-binaries".to_string(),
+ "warm-lambdas".to_string(),
+ "build-service-binaries".to_string(),
+ "build-lambda-artifacts".to_string(),
+ "migrate-db".to_string(),
+ "deploy-services".to_string(),
+ ])
+ .cond(Expression::new("always()"))
+ .add_step(check_deployment_results())
+}
+
+fn check_deployment_results() -> Step<Run> {
+ Step::new("Check deployment results").run(indoc::indoc! {r#"
+ if [[ "${{ needs.setup.result }}" == "failure" ]]; then
+ echo "❌ Deployment setup failed"
+ exit 1
+ elif [[ "${{ needs.setup.result }}" == "skipped" ]]; then
+ echo "⏭️ Deployment setup was skipped"
+ elif [[ "${{ needs.warm-binaries.result }}" == "failure" || "${{ needs.warm-lambdas.result }}" == "failure" ]]; then
+ echo "❌ Warming shared deps onto a sticky disk failed"
+ exit 1
+ elif [[ "${{ needs.build-service-binaries.result }}" == "failure" ]]; then
+ echo "❌ One or more service binary builds failed"
+ exit 1
+ elif [[ "${{ needs.build-lambda-artifacts.result }}" == "failure" ]]; then
+ echo "❌ One or more Lambda artifact builds failed"
+ exit 1
+ elif [[ "${{ needs.migrate-db.result }}" == "failure" ]]; then
+ echo "❌ Database migrations failed"
+ exit 1
+ elif [[ "${{ needs.deploy-services.result }}" == "failure" ]]; then
+ echo "❌ One or more service deployments failed"
+ exit 1
+ elif [[ "${{ needs.deploy-services.result }}" == "skipped" ]]; then
+ echo "⏭️ No services to deploy"
+ else
+ echo "✅ All service deployments completed successfully for $ENVIRONMENT environment"
+ fi
+ "#})
+ .shell("bash")
+ .add_env(Env::new("ENVIRONMENT", "${{ inputs.environment }}"))
+}
diff --git a/tooling/xtask/crates/xtask_workflows/src/workflows/deploy_cloud_storage_on_push.rs b/tooling/xtask/crates/xtask_workflows/src/workflows/deploy_cloud_storage_on_push.rs
new file mode 100644
index 0000000000..b1b07518a0
--- /dev/null
+++ b/tooling/xtask/crates/xtask_workflows/src/workflows/deploy_cloud_storage_on_push.rs
@@ -0,0 +1,103 @@
+//! `Deploy Cloud Storage Services on Push` — thin trigger: every push to main
+//! touching the Rust services (or the deploy machinery) deploys all services
+//! to dev via the shared [`crate::workflows::deploy_all_services`] pipeline —
+//! the same warm sticky-disk nix builds, crane/zigbuild lambdas, and handoff
+//! topology used by manual dispatches and prod releases. Generated into
+//! `deploy_cloud_storage_on_push.yml` (replaces the hand-written
+//! `deploy-cloud-storage-on-push.yml`; keep
+//! [`crate::workflows::cancel_stuck_cloud_storage_deploys`]'s `WORKFLOW_FILE`
+//! in sync with this filename).
+//!
+//! NOTE: deliberately no concurrency block here. The called workflow's own
+//! top-level group (`deploy-all-services-${{ inputs.environment }}`) IS
+//! honored for workflow_call runs and is what serializes push deploys against
+//! manual dev dispatches (running deploys finish; queued pushes coalesce to
+//! the newest). Declaring the same group in this wrapper makes GitHub detect
+//! a self-deadlock ("between a top level workflow and ...") and cancel the
+//! run.
+
+use anyhow::Result;
+use gh_workflow::{Event, Job, Push, Workflow};
+
+/// Build the workflow. The caller job's `with:`/`secrets:` are filled in by
+/// [`patch`].
+pub fn deploy_cloud_storage_on_push() -> Workflow {
+ Workflow::new("Deploy Cloud Storage Services on Push")
+ .on(Event::default().push(
+ Push::default()
+ .add_branch("main")
+ .add_path(xtask_paths::repo_glob!("Cargo.toml"))
+ .add_path(xtask_paths::repo_glob!("Cargo.lock"))
+ .add_path(xtask_paths::repo_glob!("Cross.toml"))
+ .add_path(xtask_paths::repo_glob!("clippy.toml"))
+ .add_path(xtask_paths::repo_glob!("deny.toml"))
+ .add_path(xtask_paths::repo_glob!("rust-toolchain.toml"))
+ .add_path(xtask_paths::repo_glob!(".cargo/**"))
+ .add_path(xtask_paths::repo_glob!(".config/**"))
+ .add_path(xtask_paths::repo_glob!(".sqlx/**"))
+ .add_path(xtask_paths::repo_glob!("crates/**"))
+ .add_path(xtask_paths::repo_glob!("services/**"))
+ .add_path(xtask_paths::repo_glob!("tooling/xtask/**"))
+ .add_path(xtask_paths::repo_glob!("tooling/just/**"))
+ .add_path(xtask_paths::repo_glob!("tooling/scripts/**"))
+ .add_path(xtask_paths::repo_glob!("static_assets/**"))
+ .add_path(xtask_paths::repo_glob!("docker/**"))
+ .add_path(xtask_paths::repo_glob!("nix/**"))
+ .add_path(xtask_paths::repo_glob!("nix-support/**"))
+ .add_path(xtask_paths::repo_glob!("infra/**"))
+ .add_path(xtask_paths::repo_glob!(
+ ".github/workflows/deploy_cloud_storage_on_push.yml"
+ ))
+ .add_path(xtask_paths::repo_glob!(
+ ".github/workflows/deploy_all_services.yml"
+ ))
+ .add_path(xtask_paths::repo_glob!(
+ ".github/actions/deploy-cloud-storage-pulumi/**"
+ ))
+ .add_path(xtask_paths::repo_glob!(".github/actions/setup-nix/**"))
+ .add_path(xtask_paths::repo_glob!(".github/actions/teardown-nix/**"))
+ .add_path(xtask_paths::repo_glob!(".github/actions/setup-cachix/**"))
+ .add_path(xtask_paths::repo_glob!(
+ ".github/actions/migrate-cloud-storage-db/**"
+ ))
+ .add_path(xtask_paths::repo_glob!(
+ ".github/scripts/build-cloud-storage-lambdas-nix.sh"
+ ))
+ .add_path(xtask_paths::repo_glob!(".github/services-config.json"))
+ .add_path(xtask_paths::repo_glob!(
+ ".github/workspace-dep-closures.json"
+ ))
+ .add_path(xtask_paths::repo_glob!("flake.nix"))
+ .add_path(xtask_paths::repo_glob!("flake.lock")),
+ ))
+ .add_job("deploy-all", deploy_all())
+}
+
+/// Add the caller job's `with:` and explicit `secrets:` map, and drop the
+/// `runs-on` that `Job::default()` injects (invalid alongside `uses:`).
+pub fn patch(root: &mut serde_yaml::Value) -> Result<()> {
+ let job = crate::workflows::job_mut(root, "deploy-all")?;
+ job.remove("runs-on");
+ job.insert(
+ "with".into(),
+ crate::workflows::yaml_fragment("environment: dev")?,
+ );
+ job.insert(
+ "secrets".into(),
+ crate::workflows::yaml_fragment(indoc::indoc! {r#"
+ AWS_ACCESS_KEY: ${{ secrets.AWS_ACCESS_KEY }}
+ AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+ PULUMI_ACCESS_TOKEN: ${{ secrets.PULUMI_ACCESS_TOKEN }}
+ DD_APP_KEY: ${{ secrets.DD_APP_KEY }}
+ DD_API_KEY: ${{ secrets.DD_API_KEY }}
+ CACHIX_AUTH_TOKEN: ${{ secrets.CACHIX_AUTH_TOKEN }}
+ "#})?,
+ );
+ Ok(())
+}
+
+fn deploy_all() -> Job {
+ Job::default()
+ .name("Deploy Cloud Storage Services")
+ .uses("./.github/workflows/deploy_all_services.yml")
+}
diff --git a/tooling/xtask/crates/xtask_workflows/src/workflows/mod.rs b/tooling/xtask/crates/xtask_workflows/src/workflows/mod.rs
index f37a0694ce..baedc155f6 100644
--- a/tooling/xtask/crates/xtask_workflows/src/workflows/mod.rs
+++ b/tooling/xtask/crates/xtask_workflows/src/workflows/mod.rs
@@ -24,12 +24,15 @@ mod code_check_cloud_storage;
mod code_check_conventions;
mod code_check_infra;
mod deploy_ai_editing_worker;
+mod deploy_all_services;
+mod deploy_cloud_storage_on_push;
mod deploy_preview;
mod deploy_web_app;
mod deploy_web_app_dev_push;
mod path_validation;
mod preview_fly;
mod pulumi_preview_pr;
+mod reusable_deploy_service;
mod reusable_preview_service;
mod runners;
mod steps;
@@ -163,6 +166,26 @@ const WORKFLOWS: &[WorkflowFile] = &[
file_name: "deploy_ai_editing_worker.yml",
render_yaml: || render_gh_workflow(deploy_ai_editing_worker::deploy_ai_editing_worker)(),
},
+ WorkflowFile {
+ slug: "deploy_all_services",
+ file_name: "deploy_all_services.yml",
+ render_yaml: || {
+ render_patched(
+ deploy_all_services::deploy_all_services,
+ deploy_all_services::patch,
+ )
+ },
+ },
+ WorkflowFile {
+ slug: "deploy_cloud_storage_on_push",
+ file_name: "deploy_cloud_storage_on_push.yml",
+ render_yaml: || {
+ render_patched(
+ deploy_cloud_storage_on_push::deploy_cloud_storage_on_push,
+ deploy_cloud_storage_on_push::patch,
+ )
+ },
+ },
WorkflowFile {
slug: "deploy_preview",
file_name: "deploy_preview.yml",
@@ -193,6 +216,16 @@ const WORKFLOWS: &[WorkflowFile] = &[
)
},
},
+ WorkflowFile {
+ slug: "reusable_deploy_service",
+ file_name: "reusable_deploy_service.yml",
+ render_yaml: || {
+ render_patched(
+ reusable_deploy_service::reusable_deploy_service,
+ reusable_deploy_service::patch,
+ )
+ },
+ },
WorkflowFile {
slug: "reusable_preview_service",
file_name: "reusable_preview_service.yml",
diff --git a/tooling/xtask/crates/xtask_workflows/src/workflows/reusable_deploy_service.rs b/tooling/xtask/crates/xtask_workflows/src/workflows/reusable_deploy_service.rs
new file mode 100644
index 0000000000..518e0f1625
--- /dev/null
+++ b/tooling/xtask/crates/xtask_workflows/src/workflows/reusable_deploy_service.rs
@@ -0,0 +1,283 @@
+//! `Reusable Deploy Service` — single-service deploy on the shared
+//! namespace + nix + crane pipeline: the same warm sticky-disk nix builds,
+//! crane/cargo-zigbuild lambdas, nsc artifact handoff, and Namespace Docker
+//! builder used by [`crate::workflows::deploy_all_services`], scoped to one
+//! service. Generated into `reusable_deploy_service.yml` (replaces the
+//! hand-written `reusable-deploy-service.yml`).
+//!
+//! deploy-all-services warms a shared dep closure once and fans a build
+//! matrix out across it; with a single service there is no fan-out to
+//! amortise, so the per-service build realises its own closure directly (the
+//! /nix sticky disk + Cachix still substitute everything unchanged from
+//! prior runs).
+
+use anyhow::Result;
+use gh_workflow::{Env, Event, Expression, Job, Run, Step, Use, Workflow, WorkflowCall};
+
+use crate::workflows::{runners, steps, vars};
+
+/// Build the workflow. The `workflow_call` input/secret block is filled in by
+/// [`patch`].
+pub fn reusable_deploy_service() -> Workflow {
+ Workflow::new("Reusable Deploy Service")
+ .on(Event::default().workflow_call(WorkflowCall::default()))
+ .add_job("setup", setup())
+ .add_job("build-service-binaries", build_service_binaries())
+ .add_job("build-lambda-artifacts", build_lambda_artifacts())
+ .add_job("deploy", deploy())
+}
+
+/// Fill in the ordered `workflow_call` inputs/secrets block.
+pub fn patch(root: &mut serde_yaml::Value) -> Result<()> {
+ let on = root
+ .get_mut("on")
+ .and_then(serde_yaml::Value::as_mapping_mut)
+ .ok_or_else(|| anyhow::anyhow!("rendered workflow has no `on` mapping"))?;
+ on.insert(
+ "workflow_call".into(),
+ crate::workflows::yaml_fragment(indoc::indoc! {r#"
+ inputs:
+ environment:
+ required: true
+ type: string
+ description: The environment to deploy to
+ service-name:
+ required: true
+ type: string
+ description: The name of the service to deploy
+ pulumi-stack-name:
+ required: false
+ type: string
+ description: Override pulumi stack name (defaults to service-name)
+ use-docker:
+ required: false
+ type: boolean
+ default: true
+ description: Whether to setup docker
+ use-lfs:
+ required: false
+ type: boolean
+ default: false
+ description: Whether to checkout LFS content
+ secrets:
+ AWS_ACCESS_KEY:
+ required: true
+ AWS_SECRET_ACCESS_KEY:
+ required: true
+ PULUMI_ACCESS_TOKEN:
+ required: true
+ DD_APP_KEY:
+ required: true
+ DD_API_KEY:
+ required: true
+ CACHIX_AUTH_TOKEN:
+ required: true
+ "#})?,
+ );
+ Ok(())
+}
+
+/// Resolve which artifact kinds this service produces so the heavy nix build
+/// jobs only spin up a Namespace runner + /nix volume when there's something
+/// to build (mirrors deploy-all-services' matrix filtering, for one service).
+fn setup() -> Job {
+ Job::default()
+ .name("Check ${{ inputs.service-name }} artifacts")
+ .runs_on(runners::Runner::TinyNoCache.to_string())
+ .add_output("has_binaries", "${{ steps.check.outputs.has_binaries }}")
+ .add_output("has_lambdas", "${{ steps.check.outputs.has_lambdas }}")
+ .add_step(steps::checkout_v4().add_with(("sparse-checkout", ".github/")))
+ .add_step(check_artifact_config())
+}
+
+fn check_artifact_config() -> Step<Run> {
+ Step::new("Check artifact config")
+ .run(indoc::indoc! {r#"
+ set -euo pipefail
+ cfg=.github/services-config.json
+ has_binaries=$(jq -r --arg service "$SERVICE" '((.services[$service].deploy_binaries // []) | length) > 0' "$cfg")
+ has_lambdas=$(jq -r --arg service "$SERVICE" '((.services[$service].deploy_lambdas // []) | length) > 0' "$cfg")
+ echo "has_binaries=$has_binaries" >> "$GITHUB_OUTPUT"
+ echo "has_lambdas=$has_lambdas" >> "$GITHUB_OUTPUT"
+ echo "service=$SERVICE binaries=$has_binaries lambdas=$has_lambdas"
+ "#})
+ .id("check")
+ .add_env(Env::new("SERVICE", "${{ inputs.service-name }}"))
+}
+
+/// Base for the two nix build jobs: warm /nix volume + Nix + Cachix fallback.
+fn build_job(name: &str, gate_output: &str) -> Job {
+ Job::default()
+ .name(name)
+ .needs(vec!["setup".to_string()])
+ .cond(Expression::new(format!(
+ "${{{{ needs.setup.outputs.{gate_output} == 'true' }}}}"
+ )))
+ .runs_on(runners::Runner::Mid.to_string())
+ .add_step(steps::checkout_v4().add_with(("clean", false)))
+ .add_step(steps::mount_nix_cache_volume())
+ .add_step(steps::setup_nix())
+ .add_step(steps::setup_cachix())
+}
+
+fn build_service_binaries() -> Job {
+ build_job("Build ${{ inputs.service-name }} binaries", "has_binaries")
+ .add_step(build_prebuilt_binaries())
+ .add_step(steps::upload_handoff_artifact(
+ "prebuilt-binaries.tar.gz",
+ "${{ inputs.service-name }}",
+ ))
+ .add_step(steps::teardown_nix())
+}
+
+fn build_prebuilt_binaries() -> Step<Run> {
+ let script = steps::with_cachix_watch(indoc::indoc! {r#"
+ mkdir -p prebuilt
+ nix build --print-build-logs ".#deploy-service-binaries-${SERVICE}"
+ cp -r result/bin/* prebuilt/
+ mkdir -p prebuilt/nix-store
+ while IFS= read -r store_path; do
+ cp -a "$store_path" prebuilt/nix-store/
+ done < <(nix-store -qR result)
+ touch prebuilt/.keep
+ tar -C prebuilt -czf prebuilt-binaries.tar.gz .
+ # Receipt: the deploy job logs the same hash on read.
+ echo "handoff receipt: $(sha256sum prebuilt-binaries.tar.gz | cut -d' ' -f1) ($(stat -c%s prebuilt-binaries.tar.gz) bytes)"
+ "#});
+ Step::new("Build prebuilt binaries")
+ .run(script)
+ .shell("bash")
+ .add_env(Env::new("SERVICE", "${{ inputs.service-name }}"))
+}
+
+/// Each handler is a crane + cargo-zigbuild nix package; the build script
+/// runs `nix build .#deploy-lambda-<name>` for the service's handlers.
+/// Unchanged handlers are pure cache hits (substituted from the /nix disk or
+/// Cachix).
+fn build_lambda_artifacts() -> Job {
+ build_job(
+ "Build ${{ inputs.service-name }} Lambda artifacts",
+ "has_lambdas",
+ )
+ .add_step(build_lambdas())
+ .add_step(log_lambda_receipt())
+ .add_step(steps::upload_handoff_artifact(
+ "lambda-artifacts.tar.gz",
+ "${{ inputs.service-name }}",
+ ))
+ .add_step(steps::teardown_nix())
+}
+
+fn build_lambdas() -> Step<Run> {
+ Step::new("Build Lambda artifacts")
+ .run(".github/scripts/build-cloud-storage-lambdas-nix.sh")
+ .add_env(Env::new("SERVICE", "${{ inputs.service-name }}"))
+}
+
+fn log_lambda_receipt() -> Step<Run> {
+ Step::new("Log handoff receipt")
+ .run(indoc::indoc! {r#"
+ set -euo pipefail
+ # The build script writes lambda-artifacts.tar.gz to the workspace
+ # root; the deploy job logs the same hash on read.
+ echo "handoff receipt: $(sha256sum lambda-artifacts.tar.gz | cut -d' ' -f1) ($(stat -c%s lambda-artifacts.tar.gz) bytes)"
+ "#})
+ .shell("bash")
+}
+
+/// A build job is skipped when the service has no binaries/lambdas (and a
+/// service may legitimately have neither). Proceed as long as nothing
+/// actually failed or was cancelled — skipped needs are expected here.
+/// Deploys via Pulumi + Docker; AWS auth is via explicit static keys.
+fn deploy() -> Job {
+ Job::default()
+ .name("Deploy ${{ inputs.service-name }}")
+ .needs(vec![
+ "setup".to_string(),
+ "build-service-binaries".to_string(),
+ "build-lambda-artifacts".to_string(),
+ ])
+ .cond(Expression::new("${{ !failure() && !cancelled() }}"))
+ .runs_on(runners::Runner::Small.to_string())
+ .add_env(("PULUMI_HOME", "/pulumi"))
+ .add_step(steps::checkout_v4())
+ .add_step(get_project_name())
+ .add_step(download_handoff_artifacts())
+ .add_step(steps::cache_pulumi_plugins())
+ .add_step(steps::ensure_pulumi_home_writable())
+ .add_step(deploy_service())
+}
+
+fn get_project_name() -> Step<Use> {
+ steps::uses_local(
+ "Get project name",
+ xtask_paths::repo_dir!(".github/actions/get-project-name"),
+ )
+ .id("project-name")
+ .add_with(("service-name", "${{ inputs.service-name }}"))
+}
+
+/// Pull the handoff tars from Namespace artifact storage into runner.temp
+/// (outside the workspace, which the composite action's checkout cleans).
+/// The composite's tar-path branch handles receipts + the extract guard.
+fn download_handoff_artifacts() -> Step<Run> {
+ Step::new("Download handoff artifacts")
+ .run(indoc::indoc! {r#"
+ set -euo pipefail
+ if ! command -v nsc >/dev/null 2>&1; then
+ echo "::error::nsc CLI not found — this job expects a Namespace runner (or add namespacelabs/nscloud-setup)"
+ exit 1
+ fi
+ mkdir -p "$RUNNER_TEMP/handoff"
+ if [[ "$HAS_BINARIES" == "true" ]]; then
+ nsc artifact download "$BASE/prebuilt-binaries.tar.gz" "$RUNNER_TEMP/handoff/prebuilt-binaries.tar.gz"
+ fi
+ if [[ "$HAS_LAMBDAS" == "true" ]]; then
+ nsc artifact download "$BASE/lambda-artifacts.tar.gz" "$RUNNER_TEMP/handoff/lambda-artifacts.tar.gz"
+ fi
+ "#})
+ .if_condition(Expression::new(
+ "${{ needs.setup.outputs.has_binaries == 'true' || needs.setup.outputs.has_lambdas == 'true' }}",
+ ))
+ .shell("bash")
+ .add_env(Env::new(
+ "HAS_BINARIES",
+ "${{ needs.setup.outputs.has_binaries }}",
+ ))
+ .add_env(Env::new(
+ "HAS_LAMBDAS",
+ "${{ needs.setup.outputs.has_lambdas }}",
+ ))
+ .add_env(Env::new(
+ "BASE",
+ "handoff/${{ github.run_id }}-${{ github.run_attempt }}/${{ inputs.service-name }}",
+ ))
+}
+
+fn deploy_service() -> Step<Use> {
+ steps::uses_local(
+ "Deploy ${{ inputs.service-name }}",
+ xtask_paths::repo_dir!(".github/actions/deploy-cloud-storage-pulumi"),
+ )
+ .add_with(("environment", "${{ inputs.environment }}"))
+ .add_with(("aws-access-key", vars::AWS_ACCESS_KEY))
+ .add_with(("aws-secret-key", vars::AWS_SECRET_ACCESS_KEY))
+ .add_with(("pulumi-access-token", vars::PULUMI_ACCESS_TOKEN))
+ .add_with((
+ "pulumi-service-name",
+ "${{ inputs.pulumi-stack-name || steps.project-name.outputs.project-name }}",
+ ))
+ .add_with(("use-namespace-builder", "true"))
+ .add_with(("use-docker", "${{ inputs.use-docker }}"))
+ .add_with(("use-lfs", "${{ inputs.use-lfs }}"))
+ .add_with((
+ "prebuilt-binaries-tar",
+ "${{ needs.setup.outputs.has_binaries == 'true' && format('{0}/handoff/prebuilt-binaries.tar.gz', runner.temp) || '' }}",
+ ))
+ .add_with((
+ "lambda-artifacts-tar",
+ "${{ needs.setup.outputs.has_lambdas == 'true' && format('{0}/handoff/lambda-artifacts.tar.gz', runner.temp) || '' }}",
+ ))
+ .add_with(("dd-app-key", vars::DD_APP_KEY))
+ .add_with(("dd-api-key", vars::DD_API_KEY))
+}
diff --git a/tooling/xtask/crates/xtask_workflows/src/workflows/steps.rs b/tooling/xtask/crates/xtask_workflows/src/workflows/steps.rs
index 02f8a1f61e..96b3ac9383 100644
--- a/tooling/xtask/crates/xtask_workflows/src/workflows/steps.rs
+++ b/tooling/xtask/crates/xtask_workflows/src/workflows/steps.rs
@@ -4,7 +4,7 @@
//! Third-party actions are pinned to a SHA with the human-readable version in a
//! trailing comment, matching the rest of the repo's workflows.
-use gh_workflow::{Expression, Job, Run, Step, Use};
+use gh_workflow::{Env, Expression, Job, Run, Step, Use};
use xtask_paths::{RepoDir, RuntimePath};
use crate::workflows::vars;
@@ -211,6 +211,94 @@ pub fn gated_job() -> Job {
))
}
+// ---------------------------------------------------------------------------
+// Deploy family (deploy_all_services / reusable_deploy_service) shared steps
+// ---------------------------------------------------------------------------
+
+/// `actions/checkout` pinned to the v4 SHA the deploy pipelines use. Compose
+/// options at the call site (`clean: false` on jobs reusing a mounted /nix
+/// volume, `sparse-checkout` for action-only jobs).
+pub fn checkout_v4() -> Step<Use> {
+ Step::new("Checkout Repo")
+ .uses(
+ "actions",
+ "checkout",
+ "de0fac2e4500dabe0009e67214ff5f5447ce83dd",
+ ) // v4
+ .add_with(("persist-credentials", false))
+}
+
+/// Wrap a shell fragment in the standard Cachix watch-store lifecycle.
+///
+/// The watcher is optional and always cleaned up, so Nix builds still succeed
+/// when Cachix is unavailable while every deploy-family caller shares the same
+/// process and trap semantics.
+pub fn with_cachix_watch(inner: &str) -> String {
+ indoc::formatdoc! {r#"
+ set -euo pipefail
+ cachix_pid=
+ if command -v cachix >/dev/null 2>&1 && [ -n "${{CACHIX_CACHE_NAME:-}}" ]; then
+ cachix watch-store "$CACHIX_CACHE_NAME" >/tmp/cachix-watch-store.log 2>&1 &
+ cachix_pid=$!
+ trap 'if [ -n "${{cachix_pid:-}}" ]; then kill "$cachix_pid" 2>/dev/null || true; wait "$cachix_pid" 2>/dev/null || true; fi' EXIT
+ fi
+ {inner}
+ "#}
+}
+
+/// `nix build` wrapped in `cachix watch-store`, so realised store paths are
+/// pushed to Cachix as they build — the consistency backstop when the /nix
+/// volume is cold or evicted.
+pub fn nix_build_watched(name: &str, targets: &str, done_msg: &str) -> Step<Run> {
+ let script = with_cachix_watch(&format!(
+ "nix build --print-build-logs {targets}\necho \"{done_msg}\""
+ ));
+ Step::new(name).run(script).shell("bash")
+}
+
+/// Upload a build's handoff tarball to Namespace artifact storage: strongly
+/// consistent object storage that rides Namespace's network rather than the
+/// GitHub artifacts API. Attempt-scoped path so re-runs never collide with
+/// stale uploads; the deploy job logs the same hash on read.
+pub fn upload_handoff_artifact(file: &str, service_expr: &str) -> Step<Run> {
+ Step::new("Upload handoff artifact")
+ .run(format!("nsc artifact upload {file} \"$DEST\" --expires_in=24h"))
+ .shell("bash")
+ .add_env(Env::new(
+ "DEST",
+ format!("handoff/${{{{ github.run_id }}}}-${{{{ github.run_attempt }}}}/{service_expr}/{file}"),
+ ))
+}
+
+/// Mount Pulumi's provider-plugin dir on a Namespace cache volume. Plugins are
+/// version-pinned by infra/ and identical across services; a cold volume just
+/// re-downloads (~45s). Requires the job to pin `PULUMI_HOME: /pulumi`.
+pub fn cache_pulumi_plugins() -> Step<Use> {
+ Step::new("Cache Pulumi plugins")
+ .uses(
+ "namespacelabs",
+ "nscloud-cache-action",
+ "15799a6b54e5765f85b2aac25b3f0df43ed571c0", // v1.4.3
+ )
+ .add_with(("path", "/pulumi/plugins"))
+ .continue_on_error(true)
+}
+
+/// Make PULUMI_HOME (/pulumi) and its mounted plugins subdir writable so
+/// pulumi can write credentials + plugins as the runner (no-op when already
+/// root). `mkdir -p` first: the plugin cache mount is continue-on-error, so on
+/// a mount failure /pulumi may not exist yet — create it so the cold-cache
+/// fallback works instead of the chown hard-failing the deploy.
+pub fn ensure_pulumi_home_writable() -> Step<Run> {
+ Step::new("Ensure Pulumi home is writable")
+ .run(indoc::indoc! {r#"
+ set -euo pipefail
+ sudo mkdir -p /pulumi
+ if [ "$(id -u)" -ne 0 ]; then sudo chown -R "$(id -u):$(id -g)" /pulumi; fi
+ "#})
+ .shell("bash")
+}
+
// ---------------------------------------------------------------------------
// Desktop (AppImage / DMG) shared steps
// ---------------------------------------------------------------------------