Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.
TypeScript agentaiai-agentsall-in-onecrmcrm-systememaillinearmcpmessagingnotesnotionofficerustslackslack-alternativestartupstartup-toolstypescriptworkspace
chore(ci): migrate the deploy pipeline family to cargo x workflows (#4529)
teof90d8b713be2
diff --git a/.github/actionlint.yaml b/.github/actionlint.yamlindex 579545a746..0f45ea3d1c 100644--- a/.github/actionlint.yaml+++ b/.github/actionlint.yaml@@ -1,5 +1,6 @@self-hosted-runner:labels:+ - db-migrator- linux-extra-beefy- linux-latest-middy- namespace-profile-linux-middiff --git a/.github/workflows/cancel_stuck_cloud_storage_deploys.yml b/.github/workflows/cancel_stuck_cloud_storage_deploys.ymlindex 4a2a13c901..dea6411f06 100644--- a/.github/workflows/cancel_stuck_cloud_storage_deploys.yml+++ b/.github/workflows/cancel_stuck_cloud_storage_deploys.yml@@ -24,7 +24,7 @@ jobs:name: Cancel runs stuck queued or in_progress past thresholdruns-on: namespace-profile-linux-tiny-no-cacheenv:- WORKFLOW_FILE: deploy-cloud-storage-on-push.yml+ WORKFLOW_FILE: deploy_cloud_storage_on_push.ymlMAX_AGE_MINUTES: ${{ inputs.max-age-minutes || '90' }}DRY_RUN: ${{ inputs.dry-run || 'false' }}GH_TOKEN: ${{ github.token }}diff --git a/.github/workflows/deploy-all-services.yml b/.github/workflows/deploy-all-services.ymldeleted file mode 100644index b02e030046..0000000000--- a/.github/workflows/deploy-all-services.yml+++ /dev/null@@ -1,437 +0,0 @@-name: Deploy All Services-on:- workflow_dispatch:- inputs:- environment:- type: choice- required: true- default: 'dev'- options:- - dev- - prod- description: The environment we are deploying to.- workflow_call:- inputs:- environment:- type: string- required: true- description: The environment we are deploying to.- secrets:- AWS_ACCESS_KEY:- required: true- AWS_SECRET_ACCESS_KEY:- required: true- PULUMI_ACCESS_TOKEN:- required: true- DD_APP_KEY:- required: true- DD_API_KEY:- required: true- CACHIX_AUTH_TOKEN:- required: true--# Only run 1 deploy-all workflow at a time per environment. Literal prefix:-# for workflow_call runs `github.workflow` expands to the *caller's* name,-# which would split dispatches and callers into separate groups and let them-# race the same Pulumi stacks. Groups are repo-global strings, so the on-push-# caller shares this group by using the same literal.-concurrency:- group: deploy-all-services-${{ inputs.environment }}- cancel-in-progress: false--jobs:- setup:- name: Setup Deployment Matrix- runs-on: ubuntu-latest- outputs:- matrix: ${{ steps.set-matrix.outputs.matrix }}- binaries: ${{ steps.set-matrix.outputs.binaries }}- lambdas: ${{ steps.set-matrix.outputs.lambdas }}- steps:- - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd- - name: Set deployment matrix- id: set-matrix- run: |- set -euo pipefail- cfg=.github/services-config.json- # Full list drives deploy-services; the filtered lists keep each build- # matrix to only the services that actually produce that artifact, so- # no build job spins up Nix + a cache volume for nothing.- services=$(jq -c '.services | keys' "$cfg")- binaries=$(jq -c '[.services | to_entries[] | select((.value.deploy_binaries // []) | length > 0) | .key]' "$cfg")- lambdas=$(jq -c '[.services | to_entries[] | select((.value.deploy_lambdas // []) | length > 0) | .key]' "$cfg")- echo "matrix=${services}" >> "$GITHUB_OUTPUT"- echo "binaries=${binaries}" >> "$GITHUB_OUTPUT"- echo "lambdas=${lambdas}" >> "$GITHUB_OUTPUT"- echo "All services: ${services}"- echo "With binaries: ${binaries}"- echo "With lambdas: ${lambdas}"-- # Warm the two shared dep closures in parallel. Each build matrix depends- # only on its own warm job, so a slow lambda-closure warm never blocks- # binary builds and vice versa. The /nix cache volume is an optimization;- # Cachix is the consistency backstop either way.- warm-binaries:- name: Warm shared binary deps onto sticky disk- needs: setup- if: ${{ needs.setup.outputs.binaries != '[]' }}- runs-on: namespace-profile-linux-mid- steps:- - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd-- # Namespace cache volume backing /nix. Purely an optimization: if the- # profile has no cache volumes (or the mount fails), setup-nix performs- # a cold install and Cachix substitutes the warm layers — slower, never- # wrong. Hence continue-on-error.- - name: Mount /nix cache volume- continue-on-error: true- uses: namespacelabs/nscloud-cache-action@15799a6b54e5765f85b2aac25b3f0df43ed571c0 # v1.4.3- with:- path: /nix-- - name: Setup Nix- uses: ./.github/actions/setup-nix-- - name: Configure Cachix fallback- uses: ./.github/actions/setup-cachix- with:- cachix-auth-token: ${{ secrets.CACHIX_AUTH_TOKEN }}-- - name: Warm shared release deps- shell: bash- run: |- set -euo pipefail- cachix_pid=- if command -v cachix >/dev/null 2>&1 && [ -n "${CACHIX_CACHE_NAME:-}" ]; then- cachix watch-store "$CACHIX_CACHE_NAME" >/tmp/cachix-watch-store.log 2>&1 &- cachix_pid=$!- trap 'if [ -n "${cachix_pid:-}" ]; then kill "$cachix_pid" 2>/dev/null || true; wait "$cachix_pid" 2>/dev/null || true; fi' EXIT- fi- nix build --print-build-logs ".#deployCargoArtifacts"- echo "Shared release deps realised into /nix/store; committed on job exit."-- - name: Teardown Nix (free cache volume for commit)- if: always()- uses: ./.github/actions/teardown-nix-- warm-lambdas:- name: Warm shared lambda deps onto sticky disk- needs: setup- if: ${{ needs.setup.outputs.lambdas != '[]' }}- runs-on: namespace-profile-linux-mid- steps:- - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd-- # Namespace cache volume backing /nix. Purely an optimization: if the- # profile has no cache volumes (or the mount fails), setup-nix performs- # a cold install and Cachix substitutes the warm layers — slower, never- # wrong. Hence continue-on-error.- - name: Mount /nix cache volume- continue-on-error: true- uses: namespacelabs/nscloud-cache-action@15799a6b54e5765f85b2aac25b3f0df43ed571c0 # v1.4.3- with:- path: /nix-- - name: Setup Nix- uses: ./.github/actions/setup-nix-- - name: Configure Cachix fallback- uses: ./.github/actions/setup-cachix- with:- cachix-auth-token: ${{ secrets.CACHIX_AUTH_TOKEN }}-- - name: Warm shared lambda dep closure- shell: bash- run: |- set -euo pipefail- cachix_pid=- if command -v cachix >/dev/null 2>&1 && [ -n "${CACHIX_CACHE_NAME:-}" ]; then- cachix watch-store "$CACHIX_CACHE_NAME" >/tmp/cachix-watch-store.log 2>&1 &- cachix_pid=$!- trap 'if [ -n "${cachix_pid:-}" ]; then kill "$cachix_pid" 2>/dev/null || true; wait "$cachix_pid" 2>/dev/null || true; fi' EXIT- fi- nix build --print-build-logs ".#lambdaDeployCargoArtifacts" ".#callRecordingPreviewFfmpegLayer"- echo "Shared lambda deps realised into /nix/store; committed on job exit."-- - name: Teardown Nix (free cache volume for commit)- if: always()- uses: ./.github/actions/teardown-nix-- build-service-binaries:- name: Build ${{ matrix.service }} binaries- needs: [setup, warm-binaries]- if: ${{ needs.setup.outputs.binaries != '[]' }}- # Clones the warm /nix cache volume populated by warm-binaries so only- # each service's own crate compiles (cold volume -> Cachix substitution).- runs-on: namespace-profile-linux-mid- strategy:- fail-fast: false- # No max-parallel: all binary services build concurrently (they only clone- # the warm /nix disk and compile their own crate), instead of batching 8.- matrix:- service: ${{ fromJson(needs.setup.outputs.binaries) }}- steps:- - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd- with:- clean: false-- # Namespace cache volume backing /nix. Purely an optimization: if the- # profile has no cache volumes (or the mount fails), setup-nix performs- # a cold install and Cachix substitutes the warm layers — slower, never- # wrong. Hence continue-on-error.- - name: Mount /nix cache volume- continue-on-error: true- uses: namespacelabs/nscloud-cache-action@15799a6b54e5765f85b2aac25b3f0df43ed571c0 # v1.4.3- with:- path: /nix-- - name: Setup Nix- uses: ./.github/actions/setup-nix-- - name: Configure Cachix fallback- uses: ./.github/actions/setup-cachix- with:- cachix-auth-token: ${{ secrets.CACHIX_AUTH_TOKEN }}-- - name: Build prebuilt binaries- shell: bash- env:- SERVICE: ${{ matrix.service }}- run: |- set -euo pipefail- mkdir -p prebuilt-- binaries=$(jq -r --arg service "$SERVICE" '.services[$service].deploy_binaries // [] | @tsv' .github/services-config.json)- if [[ -n "$binaries" ]]; then- cachix_pid=- if command -v cachix >/dev/null 2>&1 && [ -n "${CACHIX_CACHE_NAME:-}" ]; then- cachix watch-store "$CACHIX_CACHE_NAME" >/tmp/cachix-watch-store.log 2>&1 &- cachix_pid=$!- trap 'if [ -n "${cachix_pid:-}" ]; then kill "$cachix_pid" 2>/dev/null || true; wait "$cachix_pid" 2>/dev/null || true; fi' EXIT- fi- nix build --print-build-logs ".#deploy-service-binaries-${SERVICE}"- cp -r result/bin/* prebuilt/- mkdir -p prebuilt/nix-store- while IFS= read -r store_path; do- cp -a "$store_path" prebuilt/nix-store/- done < <(nix-store -qR result)- else- echo "No deploy binaries configured for $SERVICE; uploading an empty artifact"- fi-- touch prebuilt/.keep- tar -C prebuilt -czf prebuilt-binaries.tar.gz .- # Receipt: the deploy job logs the same hash on read.- echo "handoff receipt: $(sha256sum prebuilt-binaries.tar.gz | cut -d' ' -f1) ($(stat -c%s prebuilt-binaries.tar.gz) bytes)"-- # Handoff to the matching deploy job via Namespace artifact storage:- # strongly consistent object storage (no snapshot-visibility races),- # rides Namespace's network rather than the GitHub artifacts API.- # Attempt-scoped path so re-runs never collide with stale uploads.- - name: Upload handoff artifact- shell: bash- env:- DEST: handoff/${{ github.run_id }}-${{ github.run_attempt }}/${{ matrix.service }}/prebuilt-binaries.tar.gz- run: nsc artifact upload prebuilt-binaries.tar.gz "$DEST" --expires_in=24h-- - name: Teardown Nix (free cache volume for commit)- if: always()- uses: ./.github/actions/teardown-nix-- build-lambda-artifacts:- name: Build ${{ matrix.service }} Lambda artifacts- needs: [setup, warm-lambdas]- if: ${{ needs.setup.outputs.lambdas != '[]' }}- # Each handler is a crane + cargo-zigbuild nix package. This job builds all- # of a service's handlers via `nix build .#deploy-lambda-<name>`, cloning the- # warm lambda /nix disk committed by warm-lambdas. Unchanged handlers are- # pure cache hits (substituted from the disk/Cachix, no recompile).- runs-on: namespace-profile-linux-mid- strategy:- fail-fast: false- # No max-parallel: runners autoscale, so all lambda services- # build concurrently instead of batching 8.- matrix:- service: ${{ fromJson(needs.setup.outputs.lambdas) }}- steps:- - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd- with:- clean: false-- # Namespace cache volume backing /nix. Purely an optimization: if the- # profile has no cache volumes (or the mount fails), setup-nix performs- # a cold install and Cachix substitutes the warm layers — slower, never- # wrong. Hence continue-on-error.- - name: Mount /nix cache volume- continue-on-error: true- uses: namespacelabs/nscloud-cache-action@15799a6b54e5765f85b2aac25b3f0df43ed571c0 # v1.4.3- with:- path: /nix-- - name: Setup Nix- uses: ./.github/actions/setup-nix-- - name: Configure Cachix fallback- uses: ./.github/actions/setup-cachix- with:- cachix-auth-token: ${{ secrets.CACHIX_AUTH_TOKEN }}-- - name: Build Lambda artifacts- env:- SERVICE: ${{ matrix.service }}- run: .github/scripts/build-cloud-storage-lambdas-nix.sh-- - name: Log handoff receipt- shell: bash- run: |- set -euo pipefail- # The build script writes lambda-artifacts.tar.gz to the workspace- # root; the deploy job logs the same hash on read.- echo "handoff receipt: $(sha256sum lambda-artifacts.tar.gz | cut -d' ' -f1) ($(stat -c%s lambda-artifacts.tar.gz) bytes)"-- # Handoff via Namespace artifact storage (see build-service-binaries).- - name: Upload handoff artifact- shell: bash- env:- DEST: handoff/${{ github.run_id }}-${{ github.run_attempt }}/${{ matrix.service }}/lambda-artifacts.tar.gz- run: nsc artifact upload lambda-artifacts.tar.gz "$DEST" --expires_in=24h-- - name: Teardown Nix (free cache volume for commit)- if: always()- uses: ./.github/actions/teardown-nix-- migrate-db:- name: Run Database Migrations- runs-on: db-migrator- needs: [build-service-binaries, build-lambda-artifacts]- steps:- - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd- with:- sparse-checkout: .github/- - name: Run migrations- uses: ./.github/actions/migrate-cloud-storage-db- with:- environment: ${{ inputs.environment }}-- deploy-services:- name: Deploy ${{ matrix.service }}- needs: [setup, build-service-binaries, build-lambda-artifacts, migrate-db]- if: ${{ needs.setup.outputs.matrix != '[]' }}- # Deploys via Pulumi + Docker; AWS auth is via explicit static keys- # (configure-aws-credentials).- runs-on: namespace-profile-linux-small- # Pin PULUMI_HOME to a fixed path (outside the workspace, which the deploy- # action's checkout cleans) so the plugins subdir can be backed by a sticky- # disk. Propagates into the composite action's pulumi step via the job env.- env:- PULUMI_HOME: /pulumi- strategy:- fail-fast: false- # No max-parallel: all services deploy concurrently.- matrix:- service: ${{ fromJson(needs.setup.outputs.matrix) }}- steps:- - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd-- - name: Get project name- id: project-name- uses: ./.github/actions/get-project-name- with:- service-name: ${{ matrix.service }}-- - name: Check artifact config- id: check-artifacts- env:- SERVICE: ${{ matrix.service }}- run: |- has_binaries=$(jq -r --arg service "$SERVICE" '((.services[$service].deploy_binaries // []) | length) > 0' .github/services-config.json)- has_lambdas=$(jq -r --arg service "$SERVICE" '((.services[$service].deploy_lambdas // []) | length) > 0' .github/services-config.json)- echo "has_binaries=$has_binaries" >> "$GITHUB_OUTPUT"- echo "has_lambdas=$has_lambdas" >> "$GITHUB_OUTPUT"-- # Pull the handoff tars from Namespace artifact storage into runner.temp- # (outside the workspace, which the composite action's checkout cleans).- # The composite's tar-path branch handles receipts + the extract guard.- - name: Download handoff artifacts- if: ${{ steps.check-artifacts.outputs.has_binaries == 'true' || steps.check-artifacts.outputs.has_lambdas == 'true' }}- shell: bash- env:- HAS_BINARIES: ${{ steps.check-artifacts.outputs.has_binaries }}- HAS_LAMBDAS: ${{ steps.check-artifacts.outputs.has_lambdas }}- BASE: handoff/${{ github.run_id }}-${{ github.run_attempt }}/${{ matrix.service }}- run: |- set -euo pipefail- if ! command -v nsc >/dev/null 2>&1; then- echo "::error::nsc CLI not found — this job expects a Namespace runner (or add namespacelabs/nscloud-setup)"- exit 1- fi- mkdir -p "$RUNNER_TEMP/handoff"- if [[ "$HAS_BINARIES" == "true" ]]; then- nsc artifact download "$BASE/prebuilt-binaries.tar.gz" "$RUNNER_TEMP/handoff/prebuilt-binaries.tar.gz"- fi- if [[ "$HAS_LAMBDAS" == "true" ]]; then- nsc artifact download "$BASE/lambda-artifacts.tar.gz" "$RUNNER_TEMP/handoff/lambda-artifacts.tar.gz"- fi-- # Cache Pulumi's provider plugins ($PULUMI_HOME/plugins) on a Namespace- # cache volume. Plugins are version-pinned by infra/ and identical across- # services. Optimization-only: a cold volume just re-downloads (~45s).- - name: Cache Pulumi plugins- continue-on-error: true- uses: namespacelabs/nscloud-cache-action@15799a6b54e5765f85b2aac25b3f0df43ed571c0 # v1.4.3- with:- path: /pulumi/plugins-- # PULUMI_HOME (/pulumi) and its mounted plugins subdir come up root-owned;- # make them writable so pulumi can write credentials + plugins as the- # runner. No-op when the runner already runs as root.- - name: Ensure Pulumi home is writable- shell: bash- run: |- set -euo pipefail- if [ "$(id -u)" -ne 0 ]; then sudo chown -R "$(id -u):$(id -g)" /pulumi; fi-- - name: Deploy ${{ matrix.service }}- uses: ./.github/actions/deploy-cloud-storage-pulumi- with:- environment: ${{ inputs.environment }}- aws-access-key: ${{ secrets.AWS_ACCESS_KEY }}- aws-secret-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}- pulumi-access-token: ${{ secrets.PULUMI_ACCESS_TOKEN }}- pulumi-service-name: ${{ steps.project-name.outputs.project-name }}- use-namespace-builder: 'true'- prebuilt-binaries-tar: ${{ steps.check-artifacts.outputs.has_binaries == 'true' && format('{0}/handoff/prebuilt-binaries.tar.gz', runner.temp) || '' }}- lambda-artifacts-tar: ${{ steps.check-artifacts.outputs.has_lambdas == 'true' && format('{0}/handoff/lambda-artifacts.tar.gz', runner.temp) || '' }}- dd-app-key: ${{ secrets.DD_APP_KEY }}- dd-api-key: ${{ secrets.DD_API_KEY }}-- deployment-summary:- name: Deployment Summary- runs-on: ubuntu-latest- needs: [setup, warm-binaries, warm-lambdas, build-service-binaries, build-lambda-artifacts, deploy-services]- if: always()- steps:- - name: Check deployment results- run: |- if [[ "${{ needs.setup.result }}" == "failure" ]]; then- echo "❌ Deployment setup failed"- exit 1- elif [[ "${{ needs.setup.result }}" == "skipped" ]]; then- echo "⏭️ Deployment setup was skipped"- elif [[ "${{ needs.warm-binaries.result }}" == "failure" || "${{ needs.warm-lambdas.result }}" == "failure" ]]; then- echo "❌ Warming shared deps onto a sticky disk failed"- exit 1- elif [[ "${{ needs.build-service-binaries.result }}" == "failure" ]]; then- echo "❌ One or more service binary builds failed"- exit 1- elif [[ "${{ needs.build-lambda-artifacts.result }}" == "failure" ]]; then- echo "❌ One or more Lambda artifact builds failed"- exit 1- elif [[ "${{ needs.deploy-services.result }}" == "failure" ]]; then- echo "❌ One or more service deployments failed"- exit 1- elif [[ "${{ needs.deploy-services.result }}" == "skipped" ]]; then- echo "⏭️ No services to deploy"- else- echo "✅ All service deployments completed successfully for ${{ inputs.environment }} environment"- fidiff --git a/.github/workflows/deploy-cloud-storage-on-push.yml b/.github/workflows/deploy-cloud-storage-on-push.ymldeleted file mode 100644index 8cf5339557..0000000000--- a/.github/workflows/deploy-cloud-storage-on-push.yml+++ /dev/null@@ -1,63 +0,0 @@-# Thin trigger: every push to main touching the Rust services (or the deploy-# machinery) deploys all services to dev via the shared deploy-all-services-# pipeline — the same warm sticky-disk nix builds, crane/zigbuild lambdas, and-# handoff topology used by manual dispatches and prod releases. This replaced-# a standalone pipeline that built binaries on Cachix-only runners and lambdas-# via per-handler cargo-lambda jobs.-name: Deploy Cloud Storage Services on Push-on:- push:- branches: [ 'main' ]- paths:- - 'Cargo.toml'- - 'Cargo.lock'- - 'Cross.toml'- - 'clippy.toml'- - 'deny.toml'- - 'rust-toolchain.toml'- - '.cargo/**'- - '.config/**'- - '.sqlx/**'- - 'crates/**'- - 'services/**'- - 'tooling/xtask/**'- - 'tooling/just/**'- - 'tooling/scripts/**'- - 'static_assets/**'- - 'docker/**'- - 'nix/**'- - 'nix-support/**'- - 'infra/**'- - '.github/workflows/deploy-cloud-storage-on-push.yml'- - '.github/workflows/deploy-all-services.yml'- - '.github/actions/deploy-cloud-storage-pulumi/**'- - '.github/actions/setup-nix/**'- - '.github/actions/teardown-nix/**'- - '.github/actions/setup-cachix/**'- - '.github/actions/migrate-cloud-storage-db/**'- - '.github/scripts/build-cloud-storage-lambdas-nix.sh'- - '.github/services-config.json'- - '.github/workspace-dep-closures.json'- - 'flake.nix'- - 'flake.lock'--# NOTE: deliberately no concurrency block here. The called workflow's own-# top-level group (deploy-all-services-${{ inputs.environment }}) IS honored-# for workflow_call runs and is what serializes push deploys against manual-# dev dispatches (running deploys finish; queued pushes coalesce to the-# newest). Declaring the same group in this wrapper makes GitHub detect a-# self-deadlock ("between a top level workflow and ...") and cancel the run.--jobs:- deploy-all:- name: Deploy Cloud Storage Services- uses: ./.github/workflows/deploy-all-services.yml- with:- environment: dev- secrets:- AWS_ACCESS_KEY: ${{ secrets.AWS_ACCESS_KEY }}- AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}- PULUMI_ACCESS_TOKEN: ${{ secrets.PULUMI_ACCESS_TOKEN }}- DD_APP_KEY: ${{ secrets.DD_APP_KEY }}- DD_API_KEY: ${{ secrets.DD_API_KEY }}- CACHIX_AUTH_TOKEN: ${{ secrets.CACHIX_AUTH_TOKEN }}diff --git a/.github/workflows/deploy-service-generic.yml b/.github/workflows/deploy-service-generic.ymlindex 9366f58d16..ff5ab7c678 100644--- a/.github/workflows/deploy-service-generic.yml+++ b/.github/workflows/deploy-service-generic.yml@@ -52,7 +52,7 @@ concurrency:jobs:deploy:name: Deploy ${{ inputs.service }}- uses: ./.github/workflows/reusable-deploy-service.yml+ uses: ./.github/workflows/reusable_deploy_service.ymlwith:environment: ${{ inputs.environment }}service-name: ${{ inputs.service }}diff --git a/.github/workflows/deploy_all_services.yml b/.github/workflows/deploy_all_services.ymlnew file mode 100644index 0000000000..965ad02b94--- /dev/null+++ b/.github/workflows/deploy_all_services.yml@@ -0,0 +1,390 @@+# DO NOT EDIT — regenerate with `cargo x workflows` (from the repository root).+# Source: tooling/xtask/crates/xtask_workflows/src/workflows/deploy_all_services.rs+name: Deploy All Services+on:+ workflow_call:+ inputs:+ environment:+ type: string+ required: true+ description: The environment we are deploying to.+ secrets:+ AWS_ACCESS_KEY:+ required: true+ AWS_SECRET_ACCESS_KEY:+ required: true+ PULUMI_ACCESS_TOKEN:+ required: true+ DD_APP_KEY:+ required: true+ DD_API_KEY:+ required: true+ CACHIX_AUTH_TOKEN:+ required: true+ workflow_dispatch:+ inputs:+ environment:+ type: choice+ required: true+ default: dev+ options:+ - dev+ - prod+ description: The environment we are deploying to.+jobs:+ setup:+ name: Setup Deployment Matrix+ runs-on: namespace-profile-linux-tiny-no-cache+ outputs:+ matrix: ${{ steps.set-matrix.outputs.matrix }}+ binaries: ${{ steps.set-matrix.outputs.binaries }}+ lambdas: ${{ steps.set-matrix.outputs.lambdas }}+ steps:+ - name: Checkout Repo+ uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd+ with:+ persist-credentials: 'false'+ - id: set-matrix+ name: Set deployment matrix+ run: |+ set -euo pipefail+ cfg=.github/services-config.json+ # Full list drives deploy-services; the filtered lists keep each build+ # matrix to only the services that actually produce that artifact, so+ # no build job spins up Nix + a cache volume for nothing.+ services=$(jq -c '.services | keys' "$cfg")+ binaries=$(jq -c '[.services | to_entries[] | select((.value.deploy_binaries // []) | length > 0) | .key]' "$cfg")+ lambdas=$(jq -c '[.services | to_entries[] | select((.value.deploy_lambdas // []) | length > 0) | .key]' "$cfg")+ echo "matrix=${services}" >> "$GITHUB_OUTPUT"+ echo "binaries=${binaries}" >> "$GITHUB_OUTPUT"+ echo "lambdas=${lambdas}" >> "$GITHUB_OUTPUT"+ echo "All services: ${services}"+ echo "With binaries: ${binaries}"+ echo "With lambdas: ${lambdas}"+ warm-binaries:+ needs:+ - setup+ if: ${{ needs.setup.outputs.binaries != '[]' }}+ name: Warm shared binary deps onto sticky disk+ runs-on: namespace-profile-linux-mid+ steps:+ - name: Checkout Repo+ uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd+ with:+ persist-credentials: 'false'+ - name: Mount /nix cache volume+ uses: namespacelabs/nscloud-cache-action@15799a6b54e5765f85b2aac25b3f0df43ed571c0+ with:+ path: /nix+ continue-on-error: true+ - name: Setup Nix+ uses: ./.github/actions/setup-nix+ - name: Configure Cachix fallback+ uses: ./.github/actions/setup-cachix+ with:+ cachix-auth-token: ${{ secrets.CACHIX_AUTH_TOKEN }}+ - name: Warm shared release deps+ run: |+ set -euo pipefail+ cachix_pid=+ if command -v cachix >/dev/null 2>&1 && [ -n "${CACHIX_CACHE_NAME:-}" ]; then+ cachix watch-store "$CACHIX_CACHE_NAME" >/tmp/cachix-watch-store.log 2>&1 &+ cachix_pid=$!+ trap 'if [ -n "${cachix_pid:-}" ]; then kill "$cachix_pid" 2>/dev/null || true; wait "$cachix_pid" 2>/dev/null || true; fi' EXIT+ fi+ nix build --print-build-logs ".#deployCargoArtifacts"+ echo "Shared release deps realised into /nix/store; committed on job exit."+ shell: bash+ - name: Teardown Nix+ if: always()+ uses: ./.github/actions/teardown-nix+ warm-lambdas:+ needs:+ - setup+ if: ${{ needs.setup.outputs.lambdas != '[]' }}+ name: Warm shared lambda deps onto sticky disk+ runs-on: namespace-profile-linux-mid+ steps:+ - name: Checkout Repo+ uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd+ with:+ persist-credentials: 'false'+ - name: Mount /nix cache volume+ uses: namespacelabs/nscloud-cache-action@15799a6b54e5765f85b2aac25b3f0df43ed571c0+ with:+ path: /nix+ continue-on-error: true+ - name: Setup Nix+ uses: ./.github/actions/setup-nix+ - name: Configure Cachix fallback+ uses: ./.github/actions/setup-cachix+ with:+ cachix-auth-token: ${{ secrets.CACHIX_AUTH_TOKEN }}+ - name: Warm shared lambda dep closure+ run: |+ set -euo pipefail+ cachix_pid=+ if command -v cachix >/dev/null 2>&1 && [ -n "${CACHIX_CACHE_NAME:-}" ]; then+ cachix watch-store "$CACHIX_CACHE_NAME" >/tmp/cachix-watch-store.log 2>&1 &+ cachix_pid=$!+ trap 'if [ -n "${cachix_pid:-}" ]; then kill "$cachix_pid" 2>/dev/null || true; wait "$cachix_pid" 2>/dev/null || true; fi' EXIT+ fi+ nix build --print-build-logs ".#lambdaDeployCargoArtifacts" ".#callRecordingPreviewFfmpegLayer"+ echo "Shared lambda deps realised into /nix/store; committed on job exit."+ shell: bash+ - name: Teardown Nix+ if: always()+ uses: ./.github/actions/teardown-nix+ build-service-binaries:+ needs:+ - setup+ - warm-binaries+ if: ${{ needs.setup.outputs.binaries != '[]' }}+ name: Build ${{ matrix.service }} binaries+ runs-on: namespace-profile-linux-mid+ strategy:+ matrix:+ service: ${{ fromJson(needs.setup.outputs.binaries) }}+ fail-fast: false+ steps:+ - name: Checkout Repo+ uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd+ with:+ persist-credentials: 'false'+ clean: 'false'+ - name: Mount /nix cache volume+ uses: namespacelabs/nscloud-cache-action@15799a6b54e5765f85b2aac25b3f0df43ed571c0+ with:+ path: /nix+ continue-on-error: true+ - name: Setup Nix+ uses: ./.github/actions/setup-nix+ - name: Configure Cachix fallback+ uses: ./.github/actions/setup-cachix+ with:+ cachix-auth-token: ${{ secrets.CACHIX_AUTH_TOKEN }}+ - name: Build prebuilt binaries+ run: |++ set -euo pipefail+ cachix_pid=+ if command -v cachix >/dev/null 2>&1 && [ -n "${CACHIX_CACHE_NAME:-}" ]; then+ cachix watch-store "$CACHIX_CACHE_NAME" >/tmp/cachix-watch-store.log 2>&1 &+ cachix_pid=$!+ trap 'if [ -n "${cachix_pid:-}" ]; then kill "$cachix_pid" 2>/dev/null || true; wait "$cachix_pid" 2>/dev/null || true; fi' EXIT+ fi+ mkdir -p prebuilt+ nix build --print-build-logs ".#deploy-service-binaries-${SERVICE}"+ cp -r result/bin/* prebuilt/+ mkdir -p prebuilt/nix-store+ while IFS= read -r store_path; do+ cp -a "$store_path" prebuilt/nix-store/+ done < <(nix-store -qR result)+ touch prebuilt/.keep+ tar -C prebuilt -czf prebuilt-binaries.tar.gz .+ # Receipt: the deploy job logs the same hash on read.+ echo "handoff receipt: $(sha256sum prebuilt-binaries.tar.gz | cut -d' ' -f1) ($(stat -c%s prebuilt-binaries.tar.gz) bytes)"++ shell: bash+ env:+ SERVICE: ${{ matrix.service }}+ - name: Upload handoff artifact+ run: nsc artifact upload prebuilt-binaries.tar.gz "$DEST" --expires_in=24h+ shell: bash+ env:+ DEST: handoff/${{ github.run_id }}-${{ github.run_attempt }}/${{ matrix.service }}/prebuilt-binaries.tar.gz+ - name: Teardown Nix+ if: always()+ uses: ./.github/actions/teardown-nix+ build-lambda-artifacts:+ needs:+ - setup+ - warm-lambdas+ if: ${{ needs.setup.outputs.lambdas != '[]' }}+ name: Build ${{ matrix.service }} Lambda artifacts+ runs-on: namespace-profile-linux-mid+ strategy:+ matrix:+ service: ${{ fromJson(needs.setup.outputs.lambdas) }}+ fail-fast: false+ steps:+ - name: Checkout Repo+ uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd+ with:+ persist-credentials: 'false'+ clean: 'false'+ - name: Mount /nix cache volume+ uses: namespacelabs/nscloud-cache-action@15799a6b54e5765f85b2aac25b3f0df43ed571c0+ with:+ path: /nix+ continue-on-error: true+ - name: Setup Nix+ uses: ./.github/actions/setup-nix+ - name: Configure Cachix fallback+ uses: ./.github/actions/setup-cachix+ with:+ cachix-auth-token: ${{ secrets.CACHIX_AUTH_TOKEN }}+ - name: Build Lambda artifacts+ run: .github/scripts/build-cloud-storage-lambdas-nix.sh+ env:+ SERVICE: ${{ matrix.service }}+ - name: Log handoff receipt+ run: |+ set -euo pipefail+ # The build script writes lambda-artifacts.tar.gz to the workspace+ # root; the deploy job logs the same hash on read.+ echo "handoff receipt: $(sha256sum lambda-artifacts.tar.gz | cut -d' ' -f1) ($(stat -c%s lambda-artifacts.tar.gz) bytes)"+ shell: bash+ - name: Upload handoff artifact+ run: nsc artifact upload lambda-artifacts.tar.gz "$DEST" --expires_in=24h+ shell: bash+ env:+ DEST: handoff/${{ github.run_id }}-${{ github.run_attempt }}/${{ matrix.service }}/lambda-artifacts.tar.gz+ - name: Teardown Nix+ if: always()+ uses: ./.github/actions/teardown-nix+ migrate-db:+ needs:+ - setup+ - warm-binaries+ - warm-lambdas+ - build-service-binaries+ - build-lambda-artifacts+ if: ${{ !cancelled() && needs.setup.outputs.matrix != '[]' && !contains(needs.*.result, 'failure') && !contains(needs.*.result, 'cancelled') }}+ name: Run Database Migrations+ runs-on: db-migrator+ steps:+ - name: Checkout Repo+ uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd+ with:+ persist-credentials: 'false'+ sparse-checkout: .github/+ - name: Run migrations+ uses: ./.github/actions/migrate-cloud-storage-db+ with:+ environment: ${{ inputs.environment }}+ deploy-services:+ needs:+ - setup+ - warm-binaries+ - warm-lambdas+ - build-service-binaries+ - build-lambda-artifacts+ - migrate-db+ if: ${{ !cancelled() && needs.setup.outputs.matrix != '[]' && needs.migrate-db.result == 'success' && !contains(needs.*.result, 'failure') && !contains(needs.*.result, 'cancelled') }}+ name: Deploy ${{ matrix.service }}+ runs-on: namespace-profile-linux-small+ env:+ PULUMI_HOME: /pulumi+ strategy:+ matrix:+ service: ${{ fromJson(needs.setup.outputs.matrix) }}+ fail-fast: false+ steps:+ - name: Checkout Repo+ uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd+ with:+ persist-credentials: 'false'+ - id: project-name+ name: Get project name+ uses: ./.github/actions/get-project-name+ with:+ service-name: ${{ matrix.service }}+ - id: check-artifacts+ name: Check artifact config+ run: |+ has_binaries=$(jq -r --arg service "$SERVICE" '((.services[$service].deploy_binaries // []) | length) > 0' .github/services-config.json)+ has_lambdas=$(jq -r --arg service "$SERVICE" '((.services[$service].deploy_lambdas // []) | length) > 0' .github/services-config.json)+ echo "has_binaries=$has_binaries" >> "$GITHUB_OUTPUT"+ echo "has_lambdas=$has_lambdas" >> "$GITHUB_OUTPUT"+ env:+ SERVICE: ${{ matrix.service }}+ - name: Download handoff artifacts+ if: ${{ steps.check-artifacts.outputs.has_binaries == 'true' || steps.check-artifacts.outputs.has_lambdas == 'true' }}+ run: |+ set -euo pipefail+ if ! command -v nsc >/dev/null 2>&1; then+ echo "::error::nsc CLI not found — this job expects a Namespace runner (or add namespacelabs/nscloud-setup)"+ exit 1+ fi+ mkdir -p "$RUNNER_TEMP/handoff"+ if [[ "$HAS_BINARIES" == "true" ]]; then+ nsc artifact download "$BASE/prebuilt-binaries.tar.gz" "$RUNNER_TEMP/handoff/prebuilt-binaries.tar.gz"+ fi+ if [[ "$HAS_LAMBDAS" == "true" ]]; then+ nsc artifact download "$BASE/lambda-artifacts.tar.gz" "$RUNNER_TEMP/handoff/lambda-artifacts.tar.gz"+ fi+ shell: bash+ env:+ HAS_BINARIES: ${{ steps.check-artifacts.outputs.has_binaries }}+ HAS_LAMBDAS: ${{ steps.check-artifacts.outputs.has_lambdas }}+ BASE: handoff/${{ github.run_id }}-${{ github.run_attempt }}/${{ matrix.service }}+ - name: Cache Pulumi plugins+ uses: namespacelabs/nscloud-cache-action@15799a6b54e5765f85b2aac25b3f0df43ed571c0+ with:+ path: /pulumi/plugins+ continue-on-error: true+ - name: Ensure Pulumi home is writable+ run: |+ set -euo pipefail+ sudo mkdir -p /pulumi+ if [ "$(id -u)" -ne 0 ]; then sudo chown -R "$(id -u):$(id -g)" /pulumi; fi+ shell: bash+ - name: Deploy ${{ matrix.service }}+ uses: ./.github/actions/deploy-cloud-storage-pulumi+ with:+ environment: ${{ inputs.environment }}+ aws-access-key: ${{ secrets.AWS_ACCESS_KEY }}+ aws-secret-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}+ pulumi-access-token: ${{ secrets.PULUMI_ACCESS_TOKEN }}+ pulumi-service-name: ${{ steps.project-name.outputs.project-name }}+ use-namespace-builder: 'true'+ prebuilt-binaries-tar: ${{ steps.check-artifacts.outputs.has_binaries == 'true' && format('{0}/handoff/prebuilt-binaries.tar.gz', runner.temp) || '' }}+ lambda-artifacts-tar: ${{ steps.check-artifacts.outputs.has_lambdas == 'true' && format('{0}/handoff/lambda-artifacts.tar.gz', runner.temp) || '' }}+ dd-app-key: ${{ secrets.DD_APP_KEY }}+ dd-api-key: ${{ secrets.DD_API_KEY }}+ deployment-summary:+ needs:+ - setup+ - warm-binaries+ - warm-lambdas+ - build-service-binaries+ - build-lambda-artifacts+ - migrate-db+ - deploy-services+ if: always()+ name: Deployment Summary+ runs-on: namespace-profile-linux-tiny-no-cache+ steps:+ - name: Check deployment results+ run: |+ if [[ "${{ needs.setup.result }}" == "failure" ]]; then+ echo "❌ Deployment setup failed"+ exit 1+ elif [[ "${{ needs.setup.result }}" == "skipped" ]]; then+ echo "⏭️ Deployment setup was skipped"+ elif [[ "${{ needs.warm-binaries.result }}" == "failure" || "${{ needs.warm-lambdas.result }}" == "failure" ]]; then+ echo "❌ Warming shared deps onto a sticky disk failed"+ exit 1+ elif [[ "${{ needs.build-service-binaries.result }}" == "failure" ]]; then+ echo "❌ One or more service binary builds failed"+ exit 1+ elif [[ "${{ needs.build-lambda-artifacts.result }}" == "failure" ]]; then+ echo "❌ One or more Lambda artifact builds failed"+ exit 1+ elif [[ "${{ needs.migrate-db.result }}" == "failure" ]]; then+ echo "❌ Database migrations failed"+ exit 1+ elif [[ "${{ needs.deploy-services.result }}" == "failure" ]]; then+ echo "❌ One or more service deployments failed"+ exit 1+ elif [[ "${{ needs.deploy-services.result }}" == "skipped" ]]; then+ echo "⏭️ No services to deploy"+ else+ echo "✅ All service deployments completed successfully for $ENVIRONMENT environment"+ fi+ shell: bash+ env:+ ENVIRONMENT: ${{ inputs.environment }}+concurrency:+ group: deploy-all-services-${{ inputs.environment }}+ cancel-in-progress: falsediff --git a/.github/workflows/deploy_cloud_storage_on_push.yml b/.github/workflows/deploy_cloud_storage_on_push.ymlnew file mode 100644index 0000000000..001b943852--- /dev/null+++ b/.github/workflows/deploy_cloud_storage_on_push.yml@@ -0,0 +1,52 @@+# DO NOT EDIT — regenerate with `cargo x workflows` (from the repository root).+# Source: tooling/xtask/crates/xtask_workflows/src/workflows/deploy_cloud_storage_on_push.rs+name: Deploy Cloud Storage Services on Push+on:+ push:+ branches:+ - main+ paths:+ - Cargo.toml+ - Cargo.lock+ - Cross.toml+ - clippy.toml+ - deny.toml+ - rust-toolchain.toml+ - .cargo/**+ - .config/**+ - .sqlx/**+ - crates/**+ - services/**+ - tooling/xtask/**+ - tooling/just/**+ - tooling/scripts/**+ - static_assets/**+ - docker/**+ - nix/**+ - nix-support/**+ - infra/**+ - .github/workflows/deploy_cloud_storage_on_push.yml+ - .github/workflows/deploy_all_services.yml+ - .github/actions/deploy-cloud-storage-pulumi/**+ - .github/actions/setup-nix/**+ - .github/actions/teardown-nix/**+ - .github/actions/setup-cachix/**+ - .github/actions/migrate-cloud-storage-db/**+ - .github/scripts/build-cloud-storage-lambdas-nix.sh+ - .github/services-config.json+ - .github/workspace-dep-closures.json+ - flake.nix+ - flake.lock+jobs:+ deploy-all:+ name: Deploy Cloud Storage Services+ uses: ./.github/workflows/deploy_all_services.yml+ with:+ environment: dev+ secrets:+ AWS_ACCESS_KEY: ${{ secrets.AWS_ACCESS_KEY }}+ AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}+ PULUMI_ACCESS_TOKEN: ${{ secrets.PULUMI_ACCESS_TOKEN }}+ DD_APP_KEY: ${{ secrets.DD_APP_KEY }}+ DD_API_KEY: ${{ secrets.DD_API_KEY }}+ CACHIX_AUTH_TOKEN: ${{ secrets.CACHIX_AUTH_TOKEN }}diff --git a/.github/workflows/release-production.yml b/.github/workflows/release-production.ymlindex b0e1074860..8e5de91f1b 100644--- a/.github/workflows/release-production.yml+++ b/.github/workflows/release-production.yml@@ -13,7 +13,7 @@ env:jobs:deploy-cloud-storage:name: Deploy Cloud Storage Services- uses: ./.github/workflows/deploy-all-services.yml+ uses: ./.github/workflows/deploy_all_services.ymlwith:environment: prodsecrets:diff --git a/.github/workflows/reusable-deploy-service.yml b/.github/workflows/reusable-deploy-service.ymldeleted file mode 100644index 21b382ea2c..0000000000--- a/.github/workflows/reusable-deploy-service.yml+++ /dev/null@@ -1,282 +0,0 @@-# Single-service deploy on the shared namespace + nix + crane pipeline — the-# same warm sticky-disk nix builds, crane/cargo-zigbuild lambdas, nsc artifact-# handoff, and Namespace Docker builder used by deploy-all-services, scoped to-# one service. This replaced a pipeline that built binaries on Cachix-only-# linux-extra-beefy runners, built lambdas per-handler, and handed off via the-# GitHub artifacts API.-#-# deploy-all-services warms a shared dep closure once and fans a build matrix-# out across it; with a single service there is no fan-out to amortise, so the-# per-service build realises its own closure directly (the /nix sticky disk +-# Cachix still substitute everything unchanged from prior runs).-name: Reusable Deploy Service-on:- workflow_call:- inputs:- environment:- required: true- type: string- description: The environment to deploy to- service-name:- required: true- type: string- description: The name of the service to deploy- pulumi-stack-name:- required: false- type: string- description: Override pulumi stack name (defaults to service-name)- use-docker:- required: false- type: boolean- default: true- description: Whether to setup docker- use-lfs:- required: false- type: boolean- default: false- description: Whether to checkout LFS content- secrets:- AWS_ACCESS_KEY:- required: true- AWS_SECRET_ACCESS_KEY:- required: true- PULUMI_ACCESS_TOKEN:- required: true- DD_APP_KEY:- required: true- DD_API_KEY:- required: true- CACHIX_AUTH_TOKEN:- required: true--jobs:- # Resolve which artifact kinds this service produces so the heavy nix build- # jobs only spin up a Namespace runner + /nix volume when there's something to- # build (mirrors deploy-all-services' matrix filtering, for one service).- setup:- name: Check ${{ inputs.service-name }} artifacts- runs-on: ubuntu-latest- outputs:- has_binaries: ${{ steps.check.outputs.has_binaries }}- has_lambdas: ${{ steps.check.outputs.has_lambdas }}- steps:- # Pinned actions/checkout v6: https://github.com/actions/checkout/commit/de0fac2e4500dabe0009e67214ff5f5447ce83dd- - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd- with:- sparse-checkout: .github/- - name: Check artifact config- id: check- env:- SERVICE: ${{ inputs.service-name }}- run: |- set -euo pipefail- cfg=.github/services-config.json- has_binaries=$(jq -r --arg service "$SERVICE" '((.services[$service].deploy_binaries // []) | length) > 0' "$cfg")- has_lambdas=$(jq -r --arg service "$SERVICE" '((.services[$service].deploy_lambdas // []) | length) > 0' "$cfg")- echo "has_binaries=$has_binaries" >> "$GITHUB_OUTPUT"- echo "has_lambdas=$has_lambdas" >> "$GITHUB_OUTPUT"- echo "service=$SERVICE binaries=$has_binaries lambdas=$has_lambdas"-- build-service-binaries:- name: Build ${{ inputs.service-name }} binaries- needs: setup- if: ${{ needs.setup.outputs.has_binaries == 'true' }}- runs-on: namespace-profile-linux-mid- steps:- - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd- with:- clean: false-- # Namespace cache volume backing /nix. Purely an optimization: if the- # profile has no cache volumes (or the mount fails), setup-nix performs- # a cold install and Cachix substitutes the warm layers — slower, never- # wrong. Hence continue-on-error.- - name: Mount /nix cache volume- continue-on-error: true- uses: namespacelabs/nscloud-cache-action@15799a6b54e5765f85b2aac25b3f0df43ed571c0 # v1.4.3- with:- path: /nix-- - name: Setup Nix- uses: ./.github/actions/setup-nix-- - name: Configure Cachix fallback- uses: ./.github/actions/setup-cachix- with:- cachix-auth-token: ${{ secrets.CACHIX_AUTH_TOKEN }}-- - name: Build prebuilt binaries- shell: bash- env:- SERVICE: ${{ inputs.service-name }}- run: |- set -euo pipefail- mkdir -p prebuilt- cachix_pid=- if command -v cachix >/dev/null 2>&1 && [ -n "${CACHIX_CACHE_NAME:-}" ]; then- cachix watch-store "$CACHIX_CACHE_NAME" >/tmp/cachix-watch-store.log 2>&1 &- cachix_pid=$!- trap 'if [ -n "${cachix_pid:-}" ]; then kill "$cachix_pid" 2>/dev/null || true; wait "$cachix_pid" 2>/dev/null || true; fi' EXIT- fi- nix build --print-build-logs ".#deploy-service-binaries-${SERVICE}"- cp -r result/bin/* prebuilt/- mkdir -p prebuilt/nix-store- while IFS= read -r store_path; do- cp -a "$store_path" prebuilt/nix-store/- done < <(nix-store -qR result)- touch prebuilt/.keep- tar -C prebuilt -czf prebuilt-binaries.tar.gz .- # Receipt: the deploy job logs the same hash on read.- echo "handoff receipt: $(sha256sum prebuilt-binaries.tar.gz | cut -d' ' -f1) ($(stat -c%s prebuilt-binaries.tar.gz) bytes)"-- # Handoff to the deploy job via Namespace artifact storage: strongly- # consistent object storage (no snapshot-visibility races), rides- # Namespace's network rather than the GitHub artifacts API. Attempt-scoped- # path so re-runs never collide with stale uploads.- - name: Upload handoff artifact- shell: bash- env:- DEST: handoff/${{ github.run_id }}-${{ github.run_attempt }}/${{ inputs.service-name }}/prebuilt-binaries.tar.gz- run: nsc artifact upload prebuilt-binaries.tar.gz "$DEST" --expires_in=24h-- - name: Teardown Nix (free cache volume for commit)- if: always()- uses: ./.github/actions/teardown-nix-- build-lambda-artifacts:- name: Build ${{ inputs.service-name }} Lambda artifacts- needs: setup- if: ${{ needs.setup.outputs.has_lambdas == 'true' }}- # Each handler is a crane + cargo-zigbuild nix package; the build script runs- # `nix build .#deploy-lambda-<name>` for the service's handlers. Unchanged- # handlers are pure cache hits (substituted from the /nix disk or Cachix).- runs-on: namespace-profile-linux-mid- steps:- - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd- with:- clean: false-- # See note on the binary build's volume mount.- - name: Mount /nix cache volume- continue-on-error: true- uses: namespacelabs/nscloud-cache-action@15799a6b54e5765f85b2aac25b3f0df43ed571c0 # v1.4.3- with:- path: /nix-- - name: Setup Nix- uses: ./.github/actions/setup-nix-- - name: Configure Cachix fallback- uses: ./.github/actions/setup-cachix- with:- cachix-auth-token: ${{ secrets.CACHIX_AUTH_TOKEN }}-- - name: Build Lambda artifacts- env:- SERVICE: ${{ inputs.service-name }}- run: .github/scripts/build-cloud-storage-lambdas-nix.sh-- - name: Log handoff receipt- shell: bash- run: |- set -euo pipefail- # The build script writes lambda-artifacts.tar.gz to the workspace- # root; the deploy job logs the same hash on read.- echo "handoff receipt: $(sha256sum lambda-artifacts.tar.gz | cut -d' ' -f1) ($(stat -c%s lambda-artifacts.tar.gz) bytes)"-- # Handoff via Namespace artifact storage (see build-service-binaries).- - name: Upload handoff artifact- shell: bash- env:- DEST: handoff/${{ github.run_id }}-${{ github.run_attempt }}/${{ inputs.service-name }}/lambda-artifacts.tar.gz- run: nsc artifact upload lambda-artifacts.tar.gz "$DEST" --expires_in=24h-- - name: Teardown Nix (free cache volume for commit)- if: always()- uses: ./.github/actions/teardown-nix-- deploy:- name: Deploy ${{ inputs.service-name }}- needs: [setup, build-service-binaries, build-lambda-artifacts]- # A build job is skipped when the service has no binaries/lambdas (and a- # service may legitimately have neither). Proceed as long as nothing- # actually failed or was cancelled — skipped needs are expected here.- if: ${{ !failure() && !cancelled() }}- # Deploys via Pulumi + Docker; AWS auth is via explicit static keys- # (configure-aws-credentials).- runs-on: namespace-profile-linux-small- # Pin PULUMI_HOME to a fixed path (outside the workspace, which the deploy- # action's checkout cleans) so the plugins subdir can be backed by a sticky- # disk. Propagates into the composite action's pulumi step via the job env.- env:- PULUMI_HOME: /pulumi- steps:- - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd-- - name: Get project name- id: project-name- uses: ./.github/actions/get-project-name- with:- service-name: ${{ inputs.service-name }}-- # Pull the handoff tars from Namespace artifact storage into runner.temp- # (outside the workspace, which the composite action's checkout cleans).- # The composite's tar-path branch handles receipts + the extract guard.- - name: Download handoff artifacts- if: ${{ needs.setup.outputs.has_binaries == 'true' || needs.setup.outputs.has_lambdas == 'true' }}- shell: bash- env:- HAS_BINARIES: ${{ needs.setup.outputs.has_binaries }}- HAS_LAMBDAS: ${{ needs.setup.outputs.has_lambdas }}- BASE: handoff/${{ github.run_id }}-${{ github.run_attempt }}/${{ inputs.service-name }}- run: |- set -euo pipefail- if ! command -v nsc >/dev/null 2>&1; then- echo "::error::nsc CLI not found — this job expects a Namespace runner (or add namespacelabs/nscloud-setup)"- exit 1- fi- mkdir -p "$RUNNER_TEMP/handoff"- if [[ "$HAS_BINARIES" == "true" ]]; then- nsc artifact download "$BASE/prebuilt-binaries.tar.gz" "$RUNNER_TEMP/handoff/prebuilt-binaries.tar.gz"- fi- if [[ "$HAS_LAMBDAS" == "true" ]]; then- nsc artifact download "$BASE/lambda-artifacts.tar.gz" "$RUNNER_TEMP/handoff/lambda-artifacts.tar.gz"- fi-- # Cache Pulumi's provider plugins ($PULUMI_HOME/plugins) on a Namespace- # cache volume. Plugins are version-pinned by infra/ and identical across- # services. Optimization-only: a cold volume just re-downloads (~45s).- - name: Cache Pulumi plugins- continue-on-error: true- uses: namespacelabs/nscloud-cache-action@15799a6b54e5765f85b2aac25b3f0df43ed571c0 # v1.4.3- with:- path: /pulumi/plugins-- # PULUMI_HOME (/pulumi) and its mounted plugins subdir come up root-owned;- # make them writable so pulumi can write credentials + plugins as the- # runner. No-op when the runner already runs as root. mkdir -p first: the- # plugin cache mount is continue-on-error, so on a mount failure /pulumi- # may not exist yet — create it so the cold-cache fallback works instead- # of the chown hard-failing the deploy.- - name: Ensure Pulumi home is writable- shell: bash- run: |- set -euo pipefail- sudo mkdir -p /pulumi- if [ "$(id -u)" -ne 0 ]; then sudo chown -R "$(id -u):$(id -g)" /pulumi; fi-- - name: Deploy ${{ inputs.service-name }}- uses: ./.github/actions/deploy-cloud-storage-pulumi- with:- environment: ${{ inputs.environment }}- aws-access-key: ${{ secrets.AWS_ACCESS_KEY }}- aws-secret-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}- pulumi-access-token: ${{ secrets.PULUMI_ACCESS_TOKEN }}- pulumi-service-name: ${{ inputs.pulumi-stack-name || steps.project-name.outputs.project-name }}- use-namespace-builder: 'true'- use-docker: ${{ inputs.use-docker }}- use-lfs: ${{ inputs.use-lfs }}- prebuilt-binaries-tar: ${{ needs.setup.outputs.has_binaries == 'true' && format('{0}/handoff/prebuilt-binaries.tar.gz', runner.temp) || '' }}- lambda-artifacts-tar: ${{ needs.setup.outputs.has_lambdas == 'true' && format('{0}/handoff/lambda-artifacts.tar.gz', runner.temp) || '' }}- dd-app-key: ${{ secrets.DD_APP_KEY }}- dd-api-key: ${{ secrets.DD_API_KEY }}diff --git a/.github/workflows/reusable_deploy_service.yml b/.github/workflows/reusable_deploy_service.ymlnew file mode 100644index 0000000000..868004f40f--- /dev/null+++ b/.github/workflows/reusable_deploy_service.yml@@ -0,0 +1,229 @@+# DO NOT EDIT — regenerate with `cargo x workflows` (from the repository root).+# Source: tooling/xtask/crates/xtask_workflows/src/workflows/reusable_deploy_service.rs+name: Reusable Deploy Service+on:+ workflow_call:+ inputs:+ environment:+ required: true+ type: string+ description: The environment to deploy to+ service-name:+ required: true+ type: string+ description: The name of the service to deploy+ pulumi-stack-name:+ required: false+ type: string+ description: Override pulumi stack name (defaults to service-name)+ use-docker:+ required: false+ type: boolean+ default: true+ description: Whether to setup docker+ use-lfs:+ required: false+ type: boolean+ default: false+ description: Whether to checkout LFS content+ secrets:+ AWS_ACCESS_KEY:+ required: true+ AWS_SECRET_ACCESS_KEY:+ required: true+ PULUMI_ACCESS_TOKEN:+ required: true+ DD_APP_KEY:+ required: true+ DD_API_KEY:+ required: true+ CACHIX_AUTH_TOKEN:+ required: true+jobs:+ setup:+ name: Check ${{ inputs.service-name }} artifacts+ runs-on: namespace-profile-linux-tiny-no-cache+ outputs:+ has_binaries: ${{ steps.check.outputs.has_binaries }}+ has_lambdas: ${{ steps.check.outputs.has_lambdas }}+ steps:+ - name: Checkout Repo+ uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd+ with:+ persist-credentials: 'false'+ sparse-checkout: .github/+ - id: check+ name: Check artifact config+ run: |+ set -euo pipefail+ cfg=.github/services-config.json+ has_binaries=$(jq -r --arg service "$SERVICE" '((.services[$service].deploy_binaries // []) | length) > 0' "$cfg")+ has_lambdas=$(jq -r --arg service "$SERVICE" '((.services[$service].deploy_lambdas // []) | length) > 0' "$cfg")+ echo "has_binaries=$has_binaries" >> "$GITHUB_OUTPUT"+ echo "has_lambdas=$has_lambdas" >> "$GITHUB_OUTPUT"+ echo "service=$SERVICE binaries=$has_binaries lambdas=$has_lambdas"+ env:+ SERVICE: ${{ inputs.service-name }}+ build-service-binaries:+ needs:+ - setup+ if: ${{ needs.setup.outputs.has_binaries == 'true' }}+ name: Build ${{ inputs.service-name }} binaries+ runs-on: namespace-profile-linux-mid+ steps:+ - name: Checkout Repo+ uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd+ with:+ persist-credentials: 'false'+ clean: 'false'+ - name: Mount /nix cache volume+ uses: namespacelabs/nscloud-cache-action@15799a6b54e5765f85b2aac25b3f0df43ed571c0+ with:+ path: /nix+ continue-on-error: true+ - name: Setup Nix+ uses: ./.github/actions/setup-nix+ - name: Configure Cachix fallback+ uses: ./.github/actions/setup-cachix+ with:+ cachix-auth-token: ${{ secrets.CACHIX_AUTH_TOKEN }}+ - name: Build prebuilt binaries+ run: |++ set -euo pipefail+ cachix_pid=+ if command -v cachix >/dev/null 2>&1 && [ -n "${CACHIX_CACHE_NAME:-}" ]; then+ cachix watch-store "$CACHIX_CACHE_NAME" >/tmp/cachix-watch-store.log 2>&1 &+ cachix_pid=$!+ trap 'if [ -n "${cachix_pid:-}" ]; then kill "$cachix_pid" 2>/dev/null || true; wait "$cachix_pid" 2>/dev/null || true; fi' EXIT+ fi+ mkdir -p prebuilt+ nix build --print-build-logs ".#deploy-service-binaries-${SERVICE}"+ cp -r result/bin/* prebuilt/+ mkdir -p prebuilt/nix-store+ while IFS= read -r store_path; do+ cp -a "$store_path" prebuilt/nix-store/+ done < <(nix-store -qR result)+ touch prebuilt/.keep+ tar -C prebuilt -czf prebuilt-binaries.tar.gz .+ # Receipt: the deploy job logs the same hash on read.+ echo "handoff receipt: $(sha256sum prebuilt-binaries.tar.gz | cut -d' ' -f1) ($(stat -c%s prebuilt-binaries.tar.gz) bytes)"++ shell: bash+ env:+ SERVICE: ${{ inputs.service-name }}+ - name: Upload handoff artifact+ run: nsc artifact upload prebuilt-binaries.tar.gz "$DEST" --expires_in=24h+ shell: bash+ env:+ DEST: handoff/${{ github.run_id }}-${{ github.run_attempt }}/${{ inputs.service-name }}/prebuilt-binaries.tar.gz+ - name: Teardown Nix+ if: always()+ uses: ./.github/actions/teardown-nix+ build-lambda-artifacts:+ needs:+ - setup+ if: ${{ needs.setup.outputs.has_lambdas == 'true' }}+ name: Build ${{ inputs.service-name }} Lambda artifacts+ runs-on: namespace-profile-linux-mid+ steps:+ - name: Checkout Repo+ uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd+ with:+ persist-credentials: 'false'+ clean: 'false'+ - name: Mount /nix cache volume+ uses: namespacelabs/nscloud-cache-action@15799a6b54e5765f85b2aac25b3f0df43ed571c0+ with:+ path: /nix+ continue-on-error: true+ - name: Setup Nix+ uses: ./.github/actions/setup-nix+ - name: Configure Cachix fallback+ uses: ./.github/actions/setup-cachix+ with:+ cachix-auth-token: ${{ secrets.CACHIX_AUTH_TOKEN }}+ - name: Build Lambda artifacts+ run: .github/scripts/build-cloud-storage-lambdas-nix.sh+ env:+ SERVICE: ${{ inputs.service-name }}+ - name: Log handoff receipt+ run: |+ set -euo pipefail+ # The build script writes lambda-artifacts.tar.gz to the workspace+ # root; the deploy job logs the same hash on read.+ echo "handoff receipt: $(sha256sum lambda-artifacts.tar.gz | cut -d' ' -f1) ($(stat -c%s lambda-artifacts.tar.gz) bytes)"+ shell: bash+ - name: Upload handoff artifact+ run: nsc artifact upload lambda-artifacts.tar.gz "$DEST" --expires_in=24h+ shell: bash+ env:+ DEST: handoff/${{ github.run_id }}-${{ github.run_attempt }}/${{ inputs.service-name }}/lambda-artifacts.tar.gz+ - name: Teardown Nix+ if: always()+ uses: ./.github/actions/teardown-nix+ deploy:+ needs:+ - setup+ - build-service-binaries+ - build-lambda-artifacts+ if: ${{ !failure() && !cancelled() }}+ name: Deploy ${{ inputs.service-name }}+ runs-on: namespace-profile-linux-small+ env:+ PULUMI_HOME: /pulumi+ steps:+ - name: Checkout Repo+ uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd+ with:+ persist-credentials: 'false'+ - id: project-name+ name: Get project name+ uses: ./.github/actions/get-project-name+ with:+ service-name: ${{ inputs.service-name }}+ - name: Download handoff artifacts+ if: ${{ needs.setup.outputs.has_binaries == 'true' || needs.setup.outputs.has_lambdas == 'true' }}+ run: |+ set -euo pipefail+ if ! command -v nsc >/dev/null 2>&1; then+ echo "::error::nsc CLI not found — this job expects a Namespace runner (or add namespacelabs/nscloud-setup)"+ exit 1+ fi+ mkdir -p "$RUNNER_TEMP/handoff"+ if [[ "$HAS_BINARIES" == "true" ]]; then+ nsc artifact download "$BASE/prebuilt-binaries.tar.gz" "$RUNNER_TEMP/handoff/prebuilt-binaries.tar.gz"+ fi+ if [[ "$HAS_LAMBDAS" == "true" ]]; then+ nsc artifact download "$BASE/lambda-artifacts.tar.gz" "$RUNNER_TEMP/handoff/lambda-artifacts.tar.gz"+ fi+ shell: bash+ env:+ HAS_BINARIES: ${{ needs.setup.outputs.has_binaries }}+ HAS_LAMBDAS: ${{ needs.setup.outputs.has_lambdas }}+ BASE: handoff/${{ github.run_id }}-${{ github.run_attempt }}/${{ inputs.service-name }}+ - name: Cache Pulumi plugins+ uses: namespacelabs/nscloud-cache-action@15799a6b54e5765f85b2aac25b3f0df43ed571c0+ with:+ path: /pulumi/plugins+ continue-on-error: true+ - name: Ensure Pulumi home is writable+ run: |+ set -euo pipefail+ sudo mkdir -p /pulumi+ if [ "$(id -u)" -ne 0 ]; then sudo chown -R "$(id -u):$(id -g)" /pulumi; fi+ shell: bash+ - name: Deploy ${{ inputs.service-name }}+ uses: ./.github/actions/deploy-cloud-storage-pulumi+ with:+ environment: ${{ inputs.environment }}+ aws-access-key: ${{ secrets.AWS_ACCESS_KEY }}+ aws-secret-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}+ pulumi-access-token: ${{ secrets.PULUMI_ACCESS_TOKEN }}+ pulumi-service-name: ${{ inputs.pulumi-stack-name || steps.project-name.outputs.project-name }}+ use-namespace-builder: 'true'+ use-docker: ${{ inputs.use-docker }}+ use-lfs: ${{ inputs.use-lfs }}+ prebuilt-binaries-tar: ${{ needs.setup.outputs.has_binaries == 'true' && format('{0}/handoff/prebuilt-binaries.tar.gz', runner.temp) || '' }}+ lambda-artifacts-tar: ${{ needs.setup.outputs.has_lambdas == 'true' && format('{0}/handoff/lambda-artifacts.tar.gz', runner.temp) || '' }}+ dd-app-key: ${{ secrets.DD_APP_KEY }}+ dd-api-key: ${{ secrets.DD_API_KEY }}diff --git a/tooling/xtask/crates/xtask_workflows/src/workflows/cancel_stuck_cloud_storage_deploys.rs b/tooling/xtask/crates/xtask_workflows/src/workflows/cancel_stuck_cloud_storage_deploys.rsindex eada9e9956..f9b6fb3955 100644--- a/tooling/xtask/crates/xtask_workflows/src/workflows/cancel_stuck_cloud_storage_deploys.rs+++ b/tooling/xtask/crates/xtask_workflows/src/workflows/cancel_stuck_cloud_storage_deploys.rs@@ -56,9 +56,9 @@ fn cancel_stuck() -> Job {.name("Cancel runs stuck queued or in_progress past threshold").runs_on(runners::Runner::TinyNoCache.to_string()).timeout_minutes(5u32)- // Keep in sync with the deploy trigger's generated filename when the- // deploy family is ported.- .add_env(("WORKFLOW_FILE", "deploy-cloud-storage-on-push.yml"))+ // Keep in sync with the deploy trigger's generated filename+ // (see deploy_cloud_storage_on_push.rs).+ .add_env(("WORKFLOW_FILE", "deploy_cloud_storage_on_push.yml")).add_env(("MAX_AGE_MINUTES", "${{ inputs.max-age-minutes || '90' }}")).add_env(("DRY_RUN", "${{ inputs.dry-run || 'false' }}")).add_env(("GH_TOKEN", "${{ github.token }}"))diff --git a/tooling/xtask/crates/xtask_workflows/src/workflows/deploy_all_services.rs b/tooling/xtask/crates/xtask_workflows/src/workflows/deploy_all_services.rsnew file mode 100644index 0000000000..5859abe8fa--- /dev/null+++ b/tooling/xtask/crates/xtask_workflows/src/workflows/deploy_all_services.rs@@ -0,0 +1,463 @@+//! `Deploy All Services` — the shared cloud-storage deploy pipeline: warm the+//! two shared dep closures onto sticky disks, fan a build matrix out per+//! service (binaries via crane, lambdas via crane + cargo-zigbuild), hand the+//! artifacts off through Namespace artifact storage, run DB migrations, then+//! deploy every service via Pulumi. Called by `deploy_cloud_storage_on_push`+//! (dev) and `release-production` (prod), and manually dispatchable.+//! Generated into `deploy_all_services.yml` (replaces the hand-written+//! `deploy-all-services.yml`).++use anyhow::Result;+use gh_workflow::{+ Concurrency, Env, Event, Expression, Job, Run, Step, Strategy, Use, Workflow, WorkflowCall,+ WorkflowDispatch,+};++use crate::workflows::{runners, steps, vars};++/// The in-VPC self-hosted runner with network access to the databases. Stays+/// off Namespace deliberately — migrations need to reach RDS.+const DB_MIGRATOR_RUNNER: &str = "db-migrator";++/// Build the workflow. The dispatch/call input blocks are filled in by+/// [`patch`].+pub fn deploy_all_services() -> Workflow {+ Workflow::new("Deploy All Services")+ .on(Event::default()+ .workflow_dispatch(WorkflowDispatch::default())+ .workflow_call(WorkflowCall::default()))+ .concurrency(+ // Only run 1 deploy-all workflow at a time per environment.+ // Literal prefix: for workflow_call runs `github.workflow` expands+ // to the *caller's* name, which would split dispatches and callers+ // into separate groups and let them race the same Pulumi stacks.+ // Groups are repo-global strings, so the on-push caller shares+ // this group by using the same literal.+ Concurrency::new(Expression::new(+ "deploy-all-services-${{ inputs.environment }}",+ ))+ .cancel_in_progress(false),+ )+ .add_job("setup", setup())+ .add_job(+ "warm-binaries",+ warm_job(+ "Warm shared binary deps onto sticky disk",+ "binaries",+ "Warm shared release deps",+ "\".#deployCargoArtifacts\"",+ "Shared release deps realised into /nix/store; committed on job exit.",+ ),+ )+ .add_job(+ "warm-lambdas",+ warm_job(+ "Warm shared lambda deps onto sticky disk",+ "lambdas",+ "Warm shared lambda dep closure",+ "\".#lambdaDeployCargoArtifacts\" \".#callRecordingPreviewFfmpegLayer\"",+ "Shared lambda deps realised into /nix/store; committed on job exit.",+ ),+ )+ .add_job("build-service-binaries", build_service_binaries())+ .add_job("build-lambda-artifacts", build_lambda_artifacts())+ .add_job("migrate-db", migrate_db())+ .add_job("deploy-services", deploy_services())+ .add_job("deployment-summary", deployment_summary())+}++/// Fill in the ordered dispatch/call input blocks.+pub fn patch(root: &mut serde_yaml::Value) -> Result<()> {+ let on = root+ .get_mut("on")+ .and_then(serde_yaml::Value::as_mapping_mut)+ .ok_or_else(|| anyhow::anyhow!("rendered workflow has no `on` mapping"))?;+ on.insert(+ "workflow_dispatch".into(),+ crate::workflows::yaml_fragment(indoc::indoc! {r#"+ inputs:+ environment:+ type: choice+ required: true+ default: 'dev'+ options:+ - dev+ - prod+ description: The environment we are deploying to.+ "#})?,+ );+ on.insert(+ "workflow_call".into(),+ crate::workflows::yaml_fragment(indoc::indoc! {r#"+ inputs:+ environment:+ type: string+ required: true+ description: The environment we are deploying to.+ secrets:+ AWS_ACCESS_KEY:+ required: true+ AWS_SECRET_ACCESS_KEY:+ required: true+ PULUMI_ACCESS_TOKEN:+ required: true+ DD_APP_KEY:+ required: true+ DD_API_KEY:+ required: true+ CACHIX_AUTH_TOKEN:+ required: true+ "#})?,+ );+ Ok(())+}++fn setup() -> Job {+ Job::default()+ .name("Setup Deployment Matrix")+ .runs_on(runners::Runner::TinyNoCache.to_string())+ .add_output("matrix", "${{ steps.set-matrix.outputs.matrix }}")+ .add_output("binaries", "${{ steps.set-matrix.outputs.binaries }}")+ .add_output("lambdas", "${{ steps.set-matrix.outputs.lambdas }}")+ .add_step(steps::checkout_v4())+ .add_step(set_matrix())+}++fn set_matrix() -> Step<Run> {+ Step::new("Set deployment matrix")+ .run(indoc::indoc! {r#"+ set -euo pipefail+ cfg=.github/services-config.json+ # Full list drives deploy-services; the filtered lists keep each build+ # matrix to only the services that actually produce that artifact, so+ # no build job spins up Nix + a cache volume for nothing.+ services=$(jq -c '.services | keys' "$cfg")+ binaries=$(jq -c '[.services | to_entries[] | select((.value.deploy_binaries // []) | length > 0) | .key]' "$cfg")+ lambdas=$(jq -c '[.services | to_entries[] | select((.value.deploy_lambdas // []) | length > 0) | .key]' "$cfg")+ echo "matrix=${services}" >> "$GITHUB_OUTPUT"+ echo "binaries=${binaries}" >> "$GITHUB_OUTPUT"+ echo "lambdas=${lambdas}" >> "$GITHUB_OUTPUT"+ echo "All services: ${services}"+ echo "With binaries: ${binaries}"+ echo "With lambdas: ${lambdas}"+ "#})+ .id("set-matrix")+}++/// Warm one shared dep closure onto the /nix sticky disk. The two warm jobs+/// run in parallel and each build matrix depends only on its own warm job, so+/// a slow lambda-closure warm never blocks binary builds and vice versa. The+/// /nix cache volume is an optimization; Cachix is the consistency backstop+/// either way.+fn warm_job(+ name: &str,+ gate_output: &str,+ build_step_name: &str,+ targets: &str,+ done_msg: &str,+) -> Job {+ Job::default()+ .name(name)+ .needs(vec!["setup".to_string()])+ .cond(Expression::new(format!(+ "${{{{ needs.setup.outputs.{gate_output} != '[]' }}}}"+ )))+ .runs_on(runners::Runner::Mid.to_string())+ .add_step(steps::checkout_v4())+ .add_step(steps::mount_nix_cache_volume())+ .add_step(steps::setup_nix())+ .add_step(steps::setup_cachix())+ .add_step(steps::nix_build_watched(build_step_name, targets, done_msg))+ .add_step(steps::teardown_nix())+}++/// Base for the per-service build matrix jobs: clones the warm /nix cache+/// volume populated by the matching warm job, so only each service's own+/// crate compiles (cold volume -> Cachix substitution). No max-parallel:+/// runners autoscale, so all services build concurrently.+fn build_job(name: &str, warm_job_id: &str, gate_output: &str) -> Job {+ Job::default()+ .name(name)+ .needs(vec!["setup".to_string(), warm_job_id.to_string()])+ .cond(Expression::new(format!(+ "${{{{ needs.setup.outputs.{gate_output} != '[]' }}}}"+ )))+ .runs_on(runners::Runner::Mid.to_string())+ .strategy(Strategy {+ fail_fast: Some(false),+ matrix: Some(serde_json::json!({+ "service": format!("${{{{ fromJson(needs.setup.outputs.{gate_output}) }}}}"),+ })),+ max_parallel: None,+ })+ .add_step(steps::checkout_v4().add_with(("clean", false)))+ .add_step(steps::mount_nix_cache_volume())+ .add_step(steps::setup_nix())+ .add_step(steps::setup_cachix())+}++fn build_service_binaries() -> Job {+ build_job(+ "Build ${{ matrix.service }} binaries",+ "warm-binaries",+ "binaries",+ )+ .add_step(build_prebuilt_binaries())+ .add_step(steps::upload_handoff_artifact(+ "prebuilt-binaries.tar.gz",+ "${{ matrix.service }}",+ ))+ .add_step(steps::teardown_nix())+}++fn build_prebuilt_binaries() -> Step<Run> {+ let script = steps::with_cachix_watch(indoc::indoc! {r#"+ mkdir -p prebuilt+ nix build --print-build-logs ".#deploy-service-binaries-${SERVICE}"+ cp -r result/bin/* prebuilt/+ mkdir -p prebuilt/nix-store+ while IFS= read -r store_path; do+ cp -a "$store_path" prebuilt/nix-store/+ done < <(nix-store -qR result)+ touch prebuilt/.keep+ tar -C prebuilt -czf prebuilt-binaries.tar.gz .+ # Receipt: the deploy job logs the same hash on read.+ echo "handoff receipt: $(sha256sum prebuilt-binaries.tar.gz | cut -d' ' -f1) ($(stat -c%s prebuilt-binaries.tar.gz) bytes)"+ "#});+ Step::new("Build prebuilt binaries")+ .run(script)+ .shell("bash")+ .add_env(Env::new("SERVICE", "${{ matrix.service }}"))+}++/// Each handler is a crane + cargo-zigbuild nix package. This job builds all+/// of a service's handlers via `nix build .#deploy-lambda-<name>`, cloning the+/// warm lambda /nix disk committed by warm-lambdas. Unchanged handlers are+/// pure cache hits (substituted from the disk/Cachix, no recompile).+fn build_lambda_artifacts() -> Job {+ build_job(+ "Build ${{ matrix.service }} Lambda artifacts",+ "warm-lambdas",+ "lambdas",+ )+ .add_step(build_lambdas())+ .add_step(log_lambda_receipt())+ .add_step(steps::upload_handoff_artifact(+ "lambda-artifacts.tar.gz",+ "${{ matrix.service }}",+ ))+ .add_step(steps::teardown_nix())+}++fn build_lambdas() -> Step<Run> {+ Step::new("Build Lambda artifacts")+ .run(".github/scripts/build-cloud-storage-lambdas-nix.sh")+ .add_env(Env::new("SERVICE", "${{ matrix.service }}"))+}++fn log_lambda_receipt() -> Step<Run> {+ Step::new("Log handoff receipt")+ .run(indoc::indoc! {r#"+ set -euo pipefail+ # The build script writes lambda-artifacts.tar.gz to the workspace+ # root; the deploy job logs the same hash on read.+ echo "handoff receipt: $(sha256sum lambda-artifacts.tar.gz | cut -d' ' -f1) ($(stat -c%s lambda-artifacts.tar.gz) bytes)"+ "#})+ .shell("bash")+}++fn migrate_db() -> Job {+ Job::default()+ .name("Run Database Migrations")+ .needs(vec![+ "setup".to_string(),+ "warm-binaries".to_string(),+ "warm-lambdas".to_string(),+ "build-service-binaries".to_string(),+ "build-lambda-artifacts".to_string(),+ ])+ .cond(Expression::new(+ "${{ !cancelled() && needs.setup.outputs.matrix != '[]' && !contains(needs.*.result, 'failure') && !contains(needs.*.result, 'cancelled') }}",+ ))+ .runs_on(DB_MIGRATOR_RUNNER)+ .add_step(steps::checkout_v4().add_with(("sparse-checkout", ".github/")))+ .add_step(run_migrations())+}++fn run_migrations() -> Step<Use> {+ steps::uses_local(+ "Run migrations",+ xtask_paths::repo_dir!(".github/actions/migrate-cloud-storage-db"),+ )+ .add_with(("environment", "${{ inputs.environment }}"))+}++/// Deploys via Pulumi + Docker; AWS auth is via explicit static keys+/// (configure-aws-credentials). PULUMI_HOME is pinned to a fixed path (outside+/// the workspace, which the deploy action's checkout cleans) so the plugins+/// subdir can be backed by a sticky disk; it propagates into the composite+/// action's pulumi step via the job env.+fn deploy_services() -> Job {+ Job::default()+ .name("Deploy ${{ matrix.service }}")+ .needs(vec![+ "setup".to_string(),+ "warm-binaries".to_string(),+ "warm-lambdas".to_string(),+ "build-service-binaries".to_string(),+ "build-lambda-artifacts".to_string(),+ "migrate-db".to_string(),+ ])+ .cond(Expression::new(+ "${{ !cancelled() && needs.setup.outputs.matrix != '[]' && needs.migrate-db.result == 'success' && !contains(needs.*.result, 'failure') && !contains(needs.*.result, 'cancelled') }}",+ ))+ .runs_on(runners::Runner::Small.to_string())+ .add_env(("PULUMI_HOME", "/pulumi"))+ .strategy(Strategy {+ fail_fast: Some(false),+ matrix: Some(serde_json::json!({+ "service": "${{ fromJson(needs.setup.outputs.matrix) }}",+ })),+ max_parallel: None,+ })+ .add_step(steps::checkout_v4())+ .add_step(get_project_name())+ .add_step(check_artifact_config())+ .add_step(download_handoff_artifacts())+ .add_step(steps::cache_pulumi_plugins())+ .add_step(steps::ensure_pulumi_home_writable())+ .add_step(deploy_service())+}++fn get_project_name() -> Step<Use> {+ steps::uses_local(+ "Get project name",+ xtask_paths::repo_dir!(".github/actions/get-project-name"),+ )+ .id("project-name")+ .add_with(("service-name", "${{ matrix.service }}"))+}++fn check_artifact_config() -> Step<Run> {+ Step::new("Check artifact config")+ .run(indoc::indoc! {r#"+ has_binaries=$(jq -r --arg service "$SERVICE" '((.services[$service].deploy_binaries // []) | length) > 0' .github/services-config.json)+ has_lambdas=$(jq -r --arg service "$SERVICE" '((.services[$service].deploy_lambdas // []) | length) > 0' .github/services-config.json)+ echo "has_binaries=$has_binaries" >> "$GITHUB_OUTPUT"+ echo "has_lambdas=$has_lambdas" >> "$GITHUB_OUTPUT"+ "#})+ .id("check-artifacts")+ .add_env(Env::new("SERVICE", "${{ matrix.service }}"))+}++/// Pull the handoff tars from Namespace artifact storage into runner.temp+/// (outside the workspace, which the composite action's checkout cleans).+/// The composite's tar-path branch handles receipts + the extract guard.+fn download_handoff_artifacts() -> Step<Run> {+ Step::new("Download handoff artifacts")+ .run(indoc::indoc! {r#"+ set -euo pipefail+ if ! command -v nsc >/dev/null 2>&1; then+ echo "::error::nsc CLI not found — this job expects a Namespace runner (or add namespacelabs/nscloud-setup)"+ exit 1+ fi+ mkdir -p "$RUNNER_TEMP/handoff"+ if [[ "$HAS_BINARIES" == "true" ]]; then+ nsc artifact download "$BASE/prebuilt-binaries.tar.gz" "$RUNNER_TEMP/handoff/prebuilt-binaries.tar.gz"+ fi+ if [[ "$HAS_LAMBDAS" == "true" ]]; then+ nsc artifact download "$BASE/lambda-artifacts.tar.gz" "$RUNNER_TEMP/handoff/lambda-artifacts.tar.gz"+ fi+ "#})+ .if_condition(Expression::new(+ "${{ steps.check-artifacts.outputs.has_binaries == 'true' || steps.check-artifacts.outputs.has_lambdas == 'true' }}",+ ))+ .shell("bash")+ .add_env(Env::new(+ "HAS_BINARIES",+ "${{ steps.check-artifacts.outputs.has_binaries }}",+ ))+ .add_env(Env::new(+ "HAS_LAMBDAS",+ "${{ steps.check-artifacts.outputs.has_lambdas }}",+ ))+ .add_env(Env::new(+ "BASE",+ "handoff/${{ github.run_id }}-${{ github.run_attempt }}/${{ matrix.service }}",+ ))+}++fn deploy_service() -> Step<Use> {+ steps::uses_local(+ "Deploy ${{ matrix.service }}",+ xtask_paths::repo_dir!(".github/actions/deploy-cloud-storage-pulumi"),+ )+ .add_with(("environment", "${{ inputs.environment }}"))+ .add_with(("aws-access-key", vars::AWS_ACCESS_KEY))+ .add_with(("aws-secret-key", vars::AWS_SECRET_ACCESS_KEY))+ .add_with(("pulumi-access-token", vars::PULUMI_ACCESS_TOKEN))+ .add_with((+ "pulumi-service-name",+ "${{ steps.project-name.outputs.project-name }}",+ ))+ .add_with(("use-namespace-builder", "true"))+ .add_with((+ "prebuilt-binaries-tar",+ "${{ steps.check-artifacts.outputs.has_binaries == 'true' && format('{0}/handoff/prebuilt-binaries.tar.gz', runner.temp) || '' }}",+ ))+ .add_with((+ "lambda-artifacts-tar",+ "${{ steps.check-artifacts.outputs.has_lambdas == 'true' && format('{0}/handoff/lambda-artifacts.tar.gz', runner.temp) || '' }}",+ ))+ .add_with(("dd-app-key", vars::DD_APP_KEY))+ .add_with(("dd-api-key", vars::DD_API_KEY))+}++fn deployment_summary() -> Job {+ Job::default()+ .name("Deployment Summary")+ .runs_on(runners::Runner::TinyNoCache.to_string())+ .needs(vec![+ "setup".to_string(),+ "warm-binaries".to_string(),+ "warm-lambdas".to_string(),+ "build-service-binaries".to_string(),+ "build-lambda-artifacts".to_string(),+ "migrate-db".to_string(),+ "deploy-services".to_string(),+ ])+ .cond(Expression::new("always()"))+ .add_step(check_deployment_results())+}++fn check_deployment_results() -> Step<Run> {+ Step::new("Check deployment results").run(indoc::indoc! {r#"+ if [[ "${{ needs.setup.result }}" == "failure" ]]; then+ echo "❌ Deployment setup failed"+ exit 1+ elif [[ "${{ needs.setup.result }}" == "skipped" ]]; then+ echo "⏭️ Deployment setup was skipped"+ elif [[ "${{ needs.warm-binaries.result }}" == "failure" || "${{ needs.warm-lambdas.result }}" == "failure" ]]; then+ echo "❌ Warming shared deps onto a sticky disk failed"+ exit 1+ elif [[ "${{ needs.build-service-binaries.result }}" == "failure" ]]; then+ echo "❌ One or more service binary builds failed"+ exit 1+ elif [[ "${{ needs.build-lambda-artifacts.result }}" == "failure" ]]; then+ echo "❌ One or more Lambda artifact builds failed"+ exit 1+ elif [[ "${{ needs.migrate-db.result }}" == "failure" ]]; then+ echo "❌ Database migrations failed"+ exit 1+ elif [[ "${{ needs.deploy-services.result }}" == "failure" ]]; then+ echo "❌ One or more service deployments failed"+ exit 1+ elif [[ "${{ needs.deploy-services.result }}" == "skipped" ]]; then+ echo "⏭️ No services to deploy"+ else+ echo "✅ All service deployments completed successfully for $ENVIRONMENT environment"+ fi+ "#})+ .shell("bash")+ .add_env(Env::new("ENVIRONMENT", "${{ inputs.environment }}"))+}diff --git a/tooling/xtask/crates/xtask_workflows/src/workflows/deploy_cloud_storage_on_push.rs b/tooling/xtask/crates/xtask_workflows/src/workflows/deploy_cloud_storage_on_push.rsnew file mode 100644index 0000000000..b1b07518a0--- /dev/null+++ b/tooling/xtask/crates/xtask_workflows/src/workflows/deploy_cloud_storage_on_push.rs@@ -0,0 +1,103 @@+//! `Deploy Cloud Storage Services on Push` — thin trigger: every push to main+//! touching the Rust services (or the deploy machinery) deploys all services+//! to dev via the shared [`crate::workflows::deploy_all_services`] pipeline —+//! the same warm sticky-disk nix builds, crane/zigbuild lambdas, and handoff+//! topology used by manual dispatches and prod releases. Generated into+//! `deploy_cloud_storage_on_push.yml` (replaces the hand-written+//! `deploy-cloud-storage-on-push.yml`; keep+//! [`crate::workflows::cancel_stuck_cloud_storage_deploys`]'s `WORKFLOW_FILE`+//! in sync with this filename).+//!+//! NOTE: deliberately no concurrency block here. The called workflow's own+//! top-level group (`deploy-all-services-${{ inputs.environment }}`) IS+//! honored for workflow_call runs and is what serializes push deploys against+//! manual dev dispatches (running deploys finish; queued pushes coalesce to+//! the newest). Declaring the same group in this wrapper makes GitHub detect+//! a self-deadlock ("between a top level workflow and ...") and cancel the+//! run.++use anyhow::Result;+use gh_workflow::{Event, Job, Push, Workflow};++/// Build the workflow. The caller job's `with:`/`secrets:` are filled in by+/// [`patch`].+pub fn deploy_cloud_storage_on_push() -> Workflow {+ Workflow::new("Deploy Cloud Storage Services on Push")+ .on(Event::default().push(+ Push::default()+ .add_branch("main")+ .add_path(xtask_paths::repo_glob!("Cargo.toml"))+ .add_path(xtask_paths::repo_glob!("Cargo.lock"))+ .add_path(xtask_paths::repo_glob!("Cross.toml"))+ .add_path(xtask_paths::repo_glob!("clippy.toml"))+ .add_path(xtask_paths::repo_glob!("deny.toml"))+ .add_path(xtask_paths::repo_glob!("rust-toolchain.toml"))+ .add_path(xtask_paths::repo_glob!(".cargo/**"))+ .add_path(xtask_paths::repo_glob!(".config/**"))+ .add_path(xtask_paths::repo_glob!(".sqlx/**"))+ .add_path(xtask_paths::repo_glob!("crates/**"))+ .add_path(xtask_paths::repo_glob!("services/**"))+ .add_path(xtask_paths::repo_glob!("tooling/xtask/**"))+ .add_path(xtask_paths::repo_glob!("tooling/just/**"))+ .add_path(xtask_paths::repo_glob!("tooling/scripts/**"))+ .add_path(xtask_paths::repo_glob!("static_assets/**"))+ .add_path(xtask_paths::repo_glob!("docker/**"))+ .add_path(xtask_paths::repo_glob!("nix/**"))+ .add_path(xtask_paths::repo_glob!("nix-support/**"))+ .add_path(xtask_paths::repo_glob!("infra/**"))+ .add_path(xtask_paths::repo_glob!(+ ".github/workflows/deploy_cloud_storage_on_push.yml"+ ))+ .add_path(xtask_paths::repo_glob!(+ ".github/workflows/deploy_all_services.yml"+ ))+ .add_path(xtask_paths::repo_glob!(+ ".github/actions/deploy-cloud-storage-pulumi/**"+ ))+ .add_path(xtask_paths::repo_glob!(".github/actions/setup-nix/**"))+ .add_path(xtask_paths::repo_glob!(".github/actions/teardown-nix/**"))+ .add_path(xtask_paths::repo_glob!(".github/actions/setup-cachix/**"))+ .add_path(xtask_paths::repo_glob!(+ ".github/actions/migrate-cloud-storage-db/**"+ ))+ .add_path(xtask_paths::repo_glob!(+ ".github/scripts/build-cloud-storage-lambdas-nix.sh"+ ))+ .add_path(xtask_paths::repo_glob!(".github/services-config.json"))+ .add_path(xtask_paths::repo_glob!(+ ".github/workspace-dep-closures.json"+ ))+ .add_path(xtask_paths::repo_glob!("flake.nix"))+ .add_path(xtask_paths::repo_glob!("flake.lock")),+ ))+ .add_job("deploy-all", deploy_all())+}++/// Add the caller job's `with:` and explicit `secrets:` map, and drop the+/// `runs-on` that `Job::default()` injects (invalid alongside `uses:`).+pub fn patch(root: &mut serde_yaml::Value) -> Result<()> {+ let job = crate::workflows::job_mut(root, "deploy-all")?;+ job.remove("runs-on");+ job.insert(+ "with".into(),+ crate::workflows::yaml_fragment("environment: dev")?,+ );+ job.insert(+ "secrets".into(),+ crate::workflows::yaml_fragment(indoc::indoc! {r#"+ AWS_ACCESS_KEY: ${{ secrets.AWS_ACCESS_KEY }}+ AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}+ PULUMI_ACCESS_TOKEN: ${{ secrets.PULUMI_ACCESS_TOKEN }}+ DD_APP_KEY: ${{ secrets.DD_APP_KEY }}+ DD_API_KEY: ${{ secrets.DD_API_KEY }}+ CACHIX_AUTH_TOKEN: ${{ secrets.CACHIX_AUTH_TOKEN }}+ "#})?,+ );+ Ok(())+}++fn deploy_all() -> Job {+ Job::default()+ .name("Deploy Cloud Storage Services")+ .uses("./.github/workflows/deploy_all_services.yml")+}diff --git a/tooling/xtask/crates/xtask_workflows/src/workflows/mod.rs b/tooling/xtask/crates/xtask_workflows/src/workflows/mod.rsindex f37a0694ce..baedc155f6 100644--- a/tooling/xtask/crates/xtask_workflows/src/workflows/mod.rs+++ b/tooling/xtask/crates/xtask_workflows/src/workflows/mod.rs@@ -24,12 +24,15 @@ mod code_check_cloud_storage;mod code_check_conventions;mod code_check_infra;mod deploy_ai_editing_worker;+mod deploy_all_services;+mod deploy_cloud_storage_on_push;mod deploy_preview;mod deploy_web_app;mod deploy_web_app_dev_push;mod path_validation;mod preview_fly;mod pulumi_preview_pr;+mod reusable_deploy_service;mod reusable_preview_service;mod runners;mod steps;@@ -163,6 +166,26 @@ const WORKFLOWS: &[WorkflowFile] = &[file_name: "deploy_ai_editing_worker.yml",render_yaml: || render_gh_workflow(deploy_ai_editing_worker::deploy_ai_editing_worker)(),},+ WorkflowFile {+ slug: "deploy_all_services",+ file_name: "deploy_all_services.yml",+ render_yaml: || {+ render_patched(+ deploy_all_services::deploy_all_services,+ deploy_all_services::patch,+ )+ },+ },+ WorkflowFile {+ slug: "deploy_cloud_storage_on_push",+ file_name: "deploy_cloud_storage_on_push.yml",+ render_yaml: || {+ render_patched(+ deploy_cloud_storage_on_push::deploy_cloud_storage_on_push,+ deploy_cloud_storage_on_push::patch,+ )+ },+ },WorkflowFile {slug: "deploy_preview",file_name: "deploy_preview.yml",@@ -193,6 +216,16 @@ const WORKFLOWS: &[WorkflowFile] = &[)},},+ WorkflowFile {+ slug: "reusable_deploy_service",+ file_name: "reusable_deploy_service.yml",+ render_yaml: || {+ render_patched(+ reusable_deploy_service::reusable_deploy_service,+ reusable_deploy_service::patch,+ )+ },+ },WorkflowFile {slug: "reusable_preview_service",file_name: "reusable_preview_service.yml",diff --git a/tooling/xtask/crates/xtask_workflows/src/workflows/reusable_deploy_service.rs b/tooling/xtask/crates/xtask_workflows/src/workflows/reusable_deploy_service.rsnew file mode 100644index 0000000000..518e0f1625--- /dev/null+++ b/tooling/xtask/crates/xtask_workflows/src/workflows/reusable_deploy_service.rs@@ -0,0 +1,283 @@+//! `Reusable Deploy Service` — single-service deploy on the shared+//! namespace + nix + crane pipeline: the same warm sticky-disk nix builds,+//! crane/cargo-zigbuild lambdas, nsc artifact handoff, and Namespace Docker+//! builder used by [`crate::workflows::deploy_all_services`], scoped to one+//! service. Generated into `reusable_deploy_service.yml` (replaces the+//! hand-written `reusable-deploy-service.yml`).+//!+//! deploy-all-services warms a shared dep closure once and fans a build+//! matrix out across it; with a single service there is no fan-out to+//! amortise, so the per-service build realises its own closure directly (the+//! /nix sticky disk + Cachix still substitute everything unchanged from+//! prior runs).++use anyhow::Result;+use gh_workflow::{Env, Event, Expression, Job, Run, Step, Use, Workflow, WorkflowCall};++use crate::workflows::{runners, steps, vars};++/// Build the workflow. The `workflow_call` input/secret block is filled in by+/// [`patch`].+pub fn reusable_deploy_service() -> Workflow {+ Workflow::new("Reusable Deploy Service")+ .on(Event::default().workflow_call(WorkflowCall::default()))+ .add_job("setup", setup())+ .add_job("build-service-binaries", build_service_binaries())+ .add_job("build-lambda-artifacts", build_lambda_artifacts())+ .add_job("deploy", deploy())+}++/// Fill in the ordered `workflow_call` inputs/secrets block.+pub fn patch(root: &mut serde_yaml::Value) -> Result<()> {+ let on = root+ .get_mut("on")+ .and_then(serde_yaml::Value::as_mapping_mut)+ .ok_or_else(|| anyhow::anyhow!("rendered workflow has no `on` mapping"))?;+ on.insert(+ "workflow_call".into(),+ crate::workflows::yaml_fragment(indoc::indoc! {r#"+ inputs:+ environment:+ required: true+ type: string+ description: The environment to deploy to+ service-name:+ required: true+ type: string+ description: The name of the service to deploy+ pulumi-stack-name:+ required: false+ type: string+ description: Override pulumi stack name (defaults to service-name)+ use-docker:+ required: false+ type: boolean+ default: true+ description: Whether to setup docker+ use-lfs:+ required: false+ type: boolean+ default: false+ description: Whether to checkout LFS content+ secrets:+ AWS_ACCESS_KEY:+ required: true+ AWS_SECRET_ACCESS_KEY:+ required: true+ PULUMI_ACCESS_TOKEN:+ required: true+ DD_APP_KEY:+ required: true+ DD_API_KEY:+ required: true+ CACHIX_AUTH_TOKEN:+ required: true+ "#})?,+ );+ Ok(())+}++/// Resolve which artifact kinds this service produces so the heavy nix build+/// jobs only spin up a Namespace runner + /nix volume when there's something+/// to build (mirrors deploy-all-services' matrix filtering, for one service).+fn setup() -> Job {+ Job::default()+ .name("Check ${{ inputs.service-name }} artifacts")+ .runs_on(runners::Runner::TinyNoCache.to_string())+ .add_output("has_binaries", "${{ steps.check.outputs.has_binaries }}")+ .add_output("has_lambdas", "${{ steps.check.outputs.has_lambdas }}")+ .add_step(steps::checkout_v4().add_with(("sparse-checkout", ".github/")))+ .add_step(check_artifact_config())+}++fn check_artifact_config() -> Step<Run> {+ Step::new("Check artifact config")+ .run(indoc::indoc! {r#"+ set -euo pipefail+ cfg=.github/services-config.json+ has_binaries=$(jq -r --arg service "$SERVICE" '((.services[$service].deploy_binaries // []) | length) > 0' "$cfg")+ has_lambdas=$(jq -r --arg service "$SERVICE" '((.services[$service].deploy_lambdas // []) | length) > 0' "$cfg")+ echo "has_binaries=$has_binaries" >> "$GITHUB_OUTPUT"+ echo "has_lambdas=$has_lambdas" >> "$GITHUB_OUTPUT"+ echo "service=$SERVICE binaries=$has_binaries lambdas=$has_lambdas"+ "#})+ .id("check")+ .add_env(Env::new("SERVICE", "${{ inputs.service-name }}"))+}++/// Base for the two nix build jobs: warm /nix volume + Nix + Cachix fallback.+fn build_job(name: &str, gate_output: &str) -> Job {+ Job::default()+ .name(name)+ .needs(vec!["setup".to_string()])+ .cond(Expression::new(format!(+ "${{{{ needs.setup.outputs.{gate_output} == 'true' }}}}"+ )))+ .runs_on(runners::Runner::Mid.to_string())+ .add_step(steps::checkout_v4().add_with(("clean", false)))+ .add_step(steps::mount_nix_cache_volume())+ .add_step(steps::setup_nix())+ .add_step(steps::setup_cachix())+}++fn build_service_binaries() -> Job {+ build_job("Build ${{ inputs.service-name }} binaries", "has_binaries")+ .add_step(build_prebuilt_binaries())+ .add_step(steps::upload_handoff_artifact(+ "prebuilt-binaries.tar.gz",+ "${{ inputs.service-name }}",+ ))+ .add_step(steps::teardown_nix())+}++fn build_prebuilt_binaries() -> Step<Run> {+ let script = steps::with_cachix_watch(indoc::indoc! {r#"+ mkdir -p prebuilt+ nix build --print-build-logs ".#deploy-service-binaries-${SERVICE}"+ cp -r result/bin/* prebuilt/+ mkdir -p prebuilt/nix-store+ while IFS= read -r store_path; do+ cp -a "$store_path" prebuilt/nix-store/+ done < <(nix-store -qR result)+ touch prebuilt/.keep+ tar -C prebuilt -czf prebuilt-binaries.tar.gz .+ # Receipt: the deploy job logs the same hash on read.+ echo "handoff receipt: $(sha256sum prebuilt-binaries.tar.gz | cut -d' ' -f1) ($(stat -c%s prebuilt-binaries.tar.gz) bytes)"+ "#});+ Step::new("Build prebuilt binaries")+ .run(script)+ .shell("bash")+ .add_env(Env::new("SERVICE", "${{ inputs.service-name }}"))+}++/// Each handler is a crane + cargo-zigbuild nix package; the build script+/// runs `nix build .#deploy-lambda-<name>` for the service's handlers.+/// Unchanged handlers are pure cache hits (substituted from the /nix disk or+/// Cachix).+fn build_lambda_artifacts() -> Job {+ build_job(+ "Build ${{ inputs.service-name }} Lambda artifacts",+ "has_lambdas",+ )+ .add_step(build_lambdas())+ .add_step(log_lambda_receipt())+ .add_step(steps::upload_handoff_artifact(+ "lambda-artifacts.tar.gz",+ "${{ inputs.service-name }}",+ ))+ .add_step(steps::teardown_nix())+}++fn build_lambdas() -> Step<Run> {+ Step::new("Build Lambda artifacts")+ .run(".github/scripts/build-cloud-storage-lambdas-nix.sh")+ .add_env(Env::new("SERVICE", "${{ inputs.service-name }}"))+}++fn log_lambda_receipt() -> Step<Run> {+ Step::new("Log handoff receipt")+ .run(indoc::indoc! {r#"+ set -euo pipefail+ # The build script writes lambda-artifacts.tar.gz to the workspace+ # root; the deploy job logs the same hash on read.+ echo "handoff receipt: $(sha256sum lambda-artifacts.tar.gz | cut -d' ' -f1) ($(stat -c%s lambda-artifacts.tar.gz) bytes)"+ "#})+ .shell("bash")+}++/// A build job is skipped when the service has no binaries/lambdas (and a+/// service may legitimately have neither). Proceed as long as nothing+/// actually failed or was cancelled — skipped needs are expected here.+/// Deploys via Pulumi + Docker; AWS auth is via explicit static keys.+fn deploy() -> Job {+ Job::default()+ .name("Deploy ${{ inputs.service-name }}")+ .needs(vec![+ "setup".to_string(),+ "build-service-binaries".to_string(),+ "build-lambda-artifacts".to_string(),+ ])+ .cond(Expression::new("${{ !failure() && !cancelled() }}"))+ .runs_on(runners::Runner::Small.to_string())+ .add_env(("PULUMI_HOME", "/pulumi"))+ .add_step(steps::checkout_v4())+ .add_step(get_project_name())+ .add_step(download_handoff_artifacts())+ .add_step(steps::cache_pulumi_plugins())+ .add_step(steps::ensure_pulumi_home_writable())+ .add_step(deploy_service())+}++fn get_project_name() -> Step<Use> {+ steps::uses_local(+ "Get project name",+ xtask_paths::repo_dir!(".github/actions/get-project-name"),+ )+ .id("project-name")+ .add_with(("service-name", "${{ inputs.service-name }}"))+}++/// Pull the handoff tars from Namespace artifact storage into runner.temp+/// (outside the workspace, which the composite action's checkout cleans).+/// The composite's tar-path branch handles receipts + the extract guard.+fn download_handoff_artifacts() -> Step<Run> {+ Step::new("Download handoff artifacts")+ .run(indoc::indoc! {r#"+ set -euo pipefail+ if ! command -v nsc >/dev/null 2>&1; then+ echo "::error::nsc CLI not found — this job expects a Namespace runner (or add namespacelabs/nscloud-setup)"+ exit 1+ fi+ mkdir -p "$RUNNER_TEMP/handoff"+ if [[ "$HAS_BINARIES" == "true" ]]; then+ nsc artifact download "$BASE/prebuilt-binaries.tar.gz" "$RUNNER_TEMP/handoff/prebuilt-binaries.tar.gz"+ fi+ if [[ "$HAS_LAMBDAS" == "true" ]]; then+ nsc artifact download "$BASE/lambda-artifacts.tar.gz" "$RUNNER_TEMP/handoff/lambda-artifacts.tar.gz"+ fi+ "#})+ .if_condition(Expression::new(+ "${{ needs.setup.outputs.has_binaries == 'true' || needs.setup.outputs.has_lambdas == 'true' }}",+ ))+ .shell("bash")+ .add_env(Env::new(+ "HAS_BINARIES",+ "${{ needs.setup.outputs.has_binaries }}",+ ))+ .add_env(Env::new(+ "HAS_LAMBDAS",+ "${{ needs.setup.outputs.has_lambdas }}",+ ))+ .add_env(Env::new(+ "BASE",+ "handoff/${{ github.run_id }}-${{ github.run_attempt }}/${{ inputs.service-name }}",+ ))+}++fn deploy_service() -> Step<Use> {+ steps::uses_local(+ "Deploy ${{ inputs.service-name }}",+ xtask_paths::repo_dir!(".github/actions/deploy-cloud-storage-pulumi"),+ )+ .add_with(("environment", "${{ inputs.environment }}"))+ .add_with(("aws-access-key", vars::AWS_ACCESS_KEY))+ .add_with(("aws-secret-key", vars::AWS_SECRET_ACCESS_KEY))+ .add_with(("pulumi-access-token", vars::PULUMI_ACCESS_TOKEN))+ .add_with((+ "pulumi-service-name",+ "${{ inputs.pulumi-stack-name || steps.project-name.outputs.project-name }}",+ ))+ .add_with(("use-namespace-builder", "true"))+ .add_with(("use-docker", "${{ inputs.use-docker }}"))+ .add_with(("use-lfs", "${{ inputs.use-lfs }}"))+ .add_with((+ "prebuilt-binaries-tar",+ "${{ needs.setup.outputs.has_binaries == 'true' && format('{0}/handoff/prebuilt-binaries.tar.gz', runner.temp) || '' }}",+ ))+ .add_with((+ "lambda-artifacts-tar",+ "${{ needs.setup.outputs.has_lambdas == 'true' && format('{0}/handoff/lambda-artifacts.tar.gz', runner.temp) || '' }}",+ ))+ .add_with(("dd-app-key", vars::DD_APP_KEY))+ .add_with(("dd-api-key", vars::DD_API_KEY))+}diff --git a/tooling/xtask/crates/xtask_workflows/src/workflows/steps.rs b/tooling/xtask/crates/xtask_workflows/src/workflows/steps.rsindex 02f8a1f61e..96b3ac9383 100644--- a/tooling/xtask/crates/xtask_workflows/src/workflows/steps.rs+++ b/tooling/xtask/crates/xtask_workflows/src/workflows/steps.rs@@ -4,7 +4,7 @@//! Third-party actions are pinned to a SHA with the human-readable version in a//! trailing comment, matching the rest of the repo's workflows.-use gh_workflow::{Expression, Job, Run, Step, Use};+use gh_workflow::{Env, Expression, Job, Run, Step, Use};use xtask_paths::{RepoDir, RuntimePath};use crate::workflows::vars;@@ -211,6 +211,94 @@ pub fn gated_job() -> Job {))}+// ---------------------------------------------------------------------------+// Deploy family (deploy_all_services / reusable_deploy_service) shared steps+// ---------------------------------------------------------------------------++/// `actions/checkout` pinned to the v4 SHA the deploy pipelines use. Compose+/// options at the call site (`clean: false` on jobs reusing a mounted /nix+/// volume, `sparse-checkout` for action-only jobs).+pub fn checkout_v4() -> Step<Use> {+ Step::new("Checkout Repo")+ .uses(+ "actions",+ "checkout",+ "de0fac2e4500dabe0009e67214ff5f5447ce83dd",+ ) // v4+ .add_with(("persist-credentials", false))+}++/// Wrap a shell fragment in the standard Cachix watch-store lifecycle.+///+/// The watcher is optional and always cleaned up, so Nix builds still succeed+/// when Cachix is unavailable while every deploy-family caller shares the same+/// process and trap semantics.+pub fn with_cachix_watch(inner: &str) -> String {+ indoc::formatdoc! {r#"+ set -euo pipefail+ cachix_pid=+ if command -v cachix >/dev/null 2>&1 && [ -n "${{CACHIX_CACHE_NAME:-}}" ]; then+ cachix watch-store "$CACHIX_CACHE_NAME" >/tmp/cachix-watch-store.log 2>&1 &+ cachix_pid=$!+ trap 'if [ -n "${{cachix_pid:-}}" ]; then kill "$cachix_pid" 2>/dev/null || true; wait "$cachix_pid" 2>/dev/null || true; fi' EXIT+ fi+ {inner}+ "#}+}++/// `nix build` wrapped in `cachix watch-store`, so realised store paths are+/// pushed to Cachix as they build — the consistency backstop when the /nix+/// volume is cold or evicted.+pub fn nix_build_watched(name: &str, targets: &str, done_msg: &str) -> Step<Run> {+ let script = with_cachix_watch(&format!(+ "nix build --print-build-logs {targets}\necho \"{done_msg}\""+ ));+ Step::new(name).run(script).shell("bash")+}++/// Upload a build's handoff tarball to Namespace artifact storage: strongly+/// consistent object storage that rides Namespace's network rather than the+/// GitHub artifacts API. Attempt-scoped path so re-runs never collide with+/// stale uploads; the deploy job logs the same hash on read.+pub fn upload_handoff_artifact(file: &str, service_expr: &str) -> Step<Run> {+ Step::new("Upload handoff artifact")+ .run(format!("nsc artifact upload {file} \"$DEST\" --expires_in=24h"))+ .shell("bash")+ .add_env(Env::new(+ "DEST",+ format!("handoff/${{{{ github.run_id }}}}-${{{{ github.run_attempt }}}}/{service_expr}/{file}"),+ ))+}++/// Mount Pulumi's provider-plugin dir on a Namespace cache volume. Plugins are+/// version-pinned by infra/ and identical across services; a cold volume just+/// re-downloads (~45s). Requires the job to pin `PULUMI_HOME: /pulumi`.+pub fn cache_pulumi_plugins() -> Step<Use> {+ Step::new("Cache Pulumi plugins")+ .uses(+ "namespacelabs",+ "nscloud-cache-action",+ "15799a6b54e5765f85b2aac25b3f0df43ed571c0", // v1.4.3+ )+ .add_with(("path", "/pulumi/plugins"))+ .continue_on_error(true)+}++/// Make PULUMI_HOME (/pulumi) and its mounted plugins subdir writable so+/// pulumi can write credentials + plugins as the runner (no-op when already+/// root). `mkdir -p` first: the plugin cache mount is continue-on-error, so on+/// a mount failure /pulumi may not exist yet — create it so the cold-cache+/// fallback works instead of the chown hard-failing the deploy.+pub fn ensure_pulumi_home_writable() -> Step<Run> {+ Step::new("Ensure Pulumi home is writable")+ .run(indoc::indoc! {r#"+ set -euo pipefail+ sudo mkdir -p /pulumi+ if [ "$(id -u)" -ne 0 ]; then sudo chown -R "$(id -u):$(id -g)" /pulumi; fi+ "#})+ .shell("bash")+}+// ---------------------------------------------------------------------------// Desktop (AppImage / DMG) shared steps// ---------------------------------------------------------------------------