Skip to content

Where agent teams gather. Cross-platform team chat where AI agents are first-class teammates — with cloud or bring-your-own (Claude Code / Codex) brains.

TypeScript

fix(deploy): guard schema-compatible deployment recovery

OKNOTOK3597f8ecb769
diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml
index 759ee26..989e280 100644
--- a/.github/workflows/deploy.yml
+++ b/.github/workflows/deploy.yml
@@ -1,11 +1,7 @@
-# Roll the cumora-server Deployment to a specific image tag.
-#
-# Production has one ignition path: an explicit workflow_dispatch. Build and
-# deploy are intentionally separate so green CI produces a candidate, not a
-# production mutation. GitHub's protected `production` environment supplies
-# the independent approval step; this workflow then pins the selected tag to
-# a digest, captures the previous revision, rolls out, runs authenticated
-# smoke, and automatically rolls back if the evidence fails.
+# Roll cumora-server to an immutable image digest with guarded recovery.
+# The executable orchestration lives in scripts/deploy-release.mjs so the same
+# capture, migration, CAS, timeout, smoke, and recovery path can be tested with
+# a fake kubectl. A recovered candidate still leaves this workflow failed.
name: Deploy
@@ -13,7 +9,7 @@ on:
workflow_dispatch:
inputs:
tag:
- description: 'Image tag in AR (e.g. d1389e2, latest). Resolved to digest before set-image.'
+ description: 'Image tag in AR (e.g. d1389e2, latest). Resolved to digest before touching GKE.'
required: false
default: 'latest'
include_agent:
@@ -21,49 +17,72 @@ on:
required: false
default: 'Y'
repair_0002:
- description: >-
- Let migration 0002 repair its own precondition (off | archive-detach).
- archive-detach copies every conversations.members id that has no
- participant in that conversation's tenant into
- conversation_members_detached_0002 (with its ordinal, so it is
- reversible) and removes it from the array. Messages are untouched.
- Runs inside 0002's transaction. Leave off unless you have read the
- precheck report from a failed run.
+ description: 'Migration 0002 repair mode (off | archive-detach)'
required: false
default: 'off'
type: choice
options: ['off', 'archive-detach']
+ recovery_mode:
+ description: 'Baseline mode (normal requires known-good baseline; forward-only never auto-restores)'
+ required: false
+ default: 'normal'
+ type: choice
+ options: ['normal', 'forward-only']
permissions:
contents: read
id-token: write
+concurrency:
+ group: cumora-production-deploy
+ cancel-in-progress: false
+
env:
GCP_PROJECT: cumora
AR_REGION: us-west2
AR_REPO: cumora
CLUSTER: cumora-prod-z
CLUSTER_LOCATION: us-west2-a
+ DEPLOYMENT: cumora-server
jobs:
deploy:
name: Roll cumora-server
runs-on: ubuntu-latest
- timeout-minutes: 30
+ # The candidate rollout, bounded old-image verifier, restore rollout, and
+ # two authenticated smokes fit inside this job budget. A cancellation
+ # cannot be relied on to run cleanup, so recovery state remains protected.
+ timeout-minutes: 45
environment:
name: production
url: https://api.cumora.ai
+ env:
+ RECOVERY_SECRET: cumora-deploy-recovery-${{ github.run_id }}
+ RECEIPT_SECRET: cumora-deploy-recovery-${{ github.run_id }}-receipt
+
steps:
- uses: actions/checkout@v4
+ - name: Set up Node.js
+ uses: actions/setup-node@v4
+ with:
+ node-version: '24'
+
+ - name: Install recovery helper runtime
+ run: npm ci --ignore-scripts --no-audit --no-fund
+
- name: Resolve image tag
id: tag
+ env:
+ INPUT_TAG: ${{ inputs.tag }}
+ INPUT_INCLUDE_AGENT: ${{ inputs.include_agent || 'Y' }}
run: |
- TAG="${{ inputs.tag }}"
- INCLUDE_AGENT="${{ inputs.include_agent || 'Y' }}"
- echo "tag=$TAG" >> $GITHUB_OUTPUT
- echo "include_agent=$INCLUDE_AGENT" >> $GITHUB_OUTPUT
- echo "→ event=${{ github.event_name }} deploying tag: $TAG include_agent: $INCLUDE_AGENT"
+ set -euo pipefail
+ TAG="$INPUT_TAG"
+ INCLUDE_AGENT="$INPUT_INCLUDE_AGENT"
+ case "$INCLUDE_AGENT" in Y|N) ;; *) echo "::error::include_agent must be Y or N"; exit 1 ;; esac
+ echo "tag=$TAG" >> "$GITHUB_OUTPUT"
+ echo "include_agent=$INCLUDE_AGENT" >> "$GITHUB_OUTPUT"
- uses: google-github-actions/auth@v2
with:
@@ -76,305 +95,73 @@ jobs:
run: gcloud components install gke-gcloud-auth-plugin --quiet
- name: kubectl cluster-credentials
- run: gcloud container clusters get-credentials ${{ env.CLUSTER }} --location ${{ env.CLUSTER_LOCATION }}
+ run: gcloud container clusters get-credentials "$CLUSTER" --location "$CLUSTER_LOCATION"
- name: Preflight current production API
+ continue-on-error: ${{ inputs.recovery_mode == 'forward-only' }}
env:
CUMORA_SMOKE_TOKEN: ${{ secrets.CUMORA_SMOKE_TOKEN }}
CUMORA_SMOKE_COMPANY_ID: ${{ secrets.CUMORA_SMOKE_COMPANY_ID }}
CUMORA_SMOKE_BASE: https://api.cumora.ai
- # The currently deployed release may predate the candidate's new
- # Shipping endpoints. Preflight proves auth + core reads; the
- # post-rollout smoke below is the strict Shipping contract gate.
CUMORA_SMOKE_REQUIRE_SHIPPING: N
run: node scripts/release-smoke.mjs
- - name: Resolve image to digest
+ - name: Resolve candidate image digests
id: digest
+ env:
+ TAG: ${{ steps.tag.outputs.tag }}
+ INCLUDE_AGENT: ${{ steps.tag.outputs.include_agent }}
run: |
- TAG="${{ steps.tag.outputs.tag }}"
- AR="${{ env.AR_REGION }}-docker.pkg.dev/${{ env.GCP_PROJECT }}/${{ env.AR_REPO }}"
-
- # Resolve through the tag REFERENCE, never `list --filter`. The
- # filter's `tags:X` is a substring match over every tag in the repo,
- # so `tags:89e7022` also matched 5e60594, 706e708, 7e34456 and five
- # more; `head -1` then picked whichever digest sorted first, which on
- # 2026-08-31 was a June image. That deploy rolled a three-month-old
- # server into production and was only caught because the image
- # predated /api/shipping/overview and the smoke gate 404'd. `tags=X`
- # is no better — it fuzzy-matches too. `describe <image>:<tag>` is
- # exact, and leaves the digest empty when the tag does not exist.
- SERVER_DIGEST=$(gcloud artifacts docker images describe "$AR/server:$TAG" \
- --format='value(image_summary.digest)' 2>/dev/null)
+ set -euo pipefail
+ AR="${AR_REGION}-docker.pkg.dev/${GCP_PROJECT}/${AR_REPO}"
+ SERVER_DIGEST=$(gcloud artifacts docker images describe "$AR/server:$TAG" --format='value(image_summary.digest)' 2>/dev/null)
if [ -z "$SERVER_DIGEST" ]; then
- echo "::error::server tag $TAG not found in AR"
+ echo "::error::server tag $TAG not found in Artifact Registry"
exit 1
fi
- echo "server=$AR/server@$SERVER_DIGEST" >> $GITHUB_OUTPUT
- echo "→ server: $AR/server@$SERVER_DIGEST"
-
- if [ "${{ steps.tag.outputs.include_agent }}" = "Y" ]; then
- AGENT_DIGEST=$(gcloud artifacts docker images describe "$AR/agent-computer:$TAG" \
- --format='value(image_summary.digest)' 2>/dev/null)
+ echo "server=$AR/server@$SERVER_DIGEST" >> "$GITHUB_OUTPUT"
+ if [ "$INCLUDE_AGENT" = "Y" ]; then
+ AGENT_DIGEST=$(gcloud artifacts docker images describe "$AR/agent-computer:$TAG" --format='value(image_summary.digest)' 2>/dev/null)
if [ -z "$AGENT_DIGEST" ]; then
- echo "::error::agent-computer tag $TAG not found in AR while include_agent=Y"
- echo "::error::Refusing a server-only deploy because runtime/CLI shim changes require server and agent-computer to stay on the same SHA. For an intentional server-only rollback, rerun manually with include_agent=N."
+ echo "::error::agent-computer tag $TAG not found while include_agent=Y"
exit 1
- else
- echo "agent=$AR/agent-computer@$AGENT_DIGEST" >> $GITHUB_OUTPUT
- echo "→ agent: $AR/agent-computer@$AGENT_DIGEST"
- fi
- fi
-
- - name: Run candidate migrations once
- id: migrate
- env:
- SERVER_IMAGE: ${{ steps.digest.outputs.server }}
- REPAIR_0002: ${{ inputs.repair_0002 || 'off' }}
- run: |
- set -euo pipefail
- case "$REPAIR_0002" in
- off|archive-detach) ;;
- *) echo "::error::repair_0002 must be 'off' or 'archive-detach' (got '$REPAIR_0002')"; exit 1 ;;
- esac
- if [ "$REPAIR_0002" != off ]; then
- echo "::warning::migration 0002 will run repair mode '$REPAIR_0002' — it rewrites conversations.members and archives what it removes into conversation_members_detached_0002"
- fi
- JOB="cumora-migrate-${GITHUB_SHA:0:7}-${GITHUB_RUN_ATTEMPT}"
- DEPLOY_JSON="$RUNNER_TEMP/cumora-deployment.json"
- JOB_JSON="$RUNNER_TEMP/cumora-migration-job.json"
-
- # DATABASE_URL points at 127.0.0.1 because production reaches Cloud
- # SQL through the proxy. Copy the live proxy container and workload
- # identity settings into one candidate migration Job. A native
- # sidecar init container exits automatically when the Job completes.
- kubectl get deployment cumora-server -o json > "$DEPLOY_JSON"
- jq --arg job "$JOB" --arg server "$SERVER_IMAGE" --arg repair "$REPAIR_0002" '
- .spec.template.spec as $pod
- | ([ $pod.initContainers[]?, $pod.containers[]? ]
- | map(select(.name == "cloud-sql-proxy")) | first) as $proxy
- | if $proxy == null then
- error("cumora-server has no cloud-sql-proxy container")
- else
- {
- apiVersion: "batch/v1",
- kind: "Job",
- metadata: {
- name: $job,
- labels: { app: "cumora-schema-migration" }
- },
- spec: {
- backoffLimit: 0,
- activeDeadlineSeconds: 600,
- ttlSecondsAfterFinished: 600,
- template: {
- metadata: { labels: { app: "cumora-schema-migration" } },
- spec: {
- serviceAccountName: $pod.serviceAccountName,
- imagePullSecrets: ($pod.imagePullSecrets // []),
- restartPolicy: "Never",
- initContainers: [
- ($proxy
- | del(.readinessProbe, .livenessProbe, .lifecycle)
- | .restartPolicy = "Always"
- | .startupProbe = (.startupProbe // {
- httpGet: { path: "/readiness", port: 9090 },
- periodSeconds: 1,
- failureThreshold: 60
- }))
- ],
- containers: [
- {
- name: "migrate",
- image: $server,
- imagePullPolicy: "IfNotPresent",
- command: ["npm", "run", "migrate"],
- envFrom: [{ secretRef: { name: "cumora" } }],
- # After envFrom on purpose: an explicit `env` entry
- # wins over one supplied by envFrom, so the
- # operator choice for this one run cannot be
- # shadowed by whatever the Secret happens to hold.
- env: [{ name: "MIGRATION_0002_REPAIR", value: $repair }],
- resources: {
- requests: { cpu: "100m", memory: "256Mi" },
- limits: { cpu: "1000m", memory: "1Gi" }
- }
- }
- ]
- }
- }
- }
- }
- end
- ' "$DEPLOY_JSON" > "$JOB_JSON"
-
- kubectl delete job "$JOB" --ignore-not-found --wait=true
- kubectl apply -f "$JOB_JSON"
-
- SUCCEEDED=N
- for _ in $(seq 1 120); do
- STATUS=$(kubectl get job "$JOB" -o json)
- if [ "$(echo "$STATUS" | jq -r '.status.succeeded // 0')" -ge 1 ]; then
- SUCCEEDED=Y
- break
- fi
- if [ "$(echo "$STATUS" | jq -r '.status.failed // 0')" -ge 1 ]; then
- break
fi
- sleep 5
- done
-
- echo "::group::Migration output"
- kubectl logs "job/$JOB" -c migrate --tail=300 2>&1 || true
- echo "::endgroup::"
- if [ "$SUCCEEDED" != "Y" ]; then
- kubectl describe job "$JOB"
- kubectl get pods -l job-name="$JOB" -o wide
- echo "::error::Candidate database migration did not complete; deployment was not mutated."
- exit 1
+ echo "agent=$AR/agent-computer@$AGENT_DIGEST" >> "$GITHUB_OUTPUT"
fi
- kubectl delete job "$JOB" --wait=true
- echo "job=$JOB" >> "$GITHUB_OUTPUT"
-
- - name: Patch deployment
- id: patch
- run: |
- SERVER="${{ steps.digest.outputs.server }}"
-
- PREVIOUS_SERVER=$(kubectl get deployment cumora-server -o jsonpath='{.spec.template.spec.containers[?(@.name=="server")].image}')
- PREVIOUS_AGENT=$(kubectl get deployment cumora-server -o jsonpath='{.spec.template.spec.containers[?(@.name=="server")].env[?(@.name=="CUMORA_AGENT_COMPUTER_IMAGE")].value}')
- echo "previous_server=$PREVIOUS_SERVER" >> $GITHUB_OUTPUT
- echo "previous_agent=$PREVIOUS_AGENT" >> $GITHUB_OUTPUT
- echo "→ rollback baseline server: $PREVIOUS_SERVER"
- echo "→ rollback baseline agent: ${PREVIOUS_AGENT:-'(unset)'}"
-
- # Apply one atomic Pod-template patch: select the candidate server,
- # remove the retired per-replica migration init container, and
- # re-assert the startup/readiness/liveness probe contract. Startup
- # allows the server's bounded schema transport retry to finish;
- # liveness MUST hit /api/livez (a DB-free "process alive" check),
- # NOT /api/health (which runs SELECT 1). Readiness stays on
- # /api/health so a pod that cannot reach the DB is kept out of
- # rotation without being killed. This keeps the fix durable even if
- # the Deployment is recreated from an older base manifest.
- AGENT="${{ steps.digest.outputs.agent }}"
- PATCH=$(jq -nc --arg server "$SERVER" --arg agent "$AGENT" '
- {
- spec: { template: { spec: {
- initContainers: [
- { name: "migrate", "$patch": "delete" }
- ],
- containers: [
- ({
- name: "server",
- image: $server,
- startupProbe: {
- httpGet: { path: "/api/livez", port: "http" },
- periodSeconds: 5,
- timeoutSeconds: 2,
- failureThreshold: 60
- },
- readinessProbe: {
- httpGet: { path: "/api/health", port: "http" },
- periodSeconds: 5,
- timeoutSeconds: 2
- },
- livenessProbe: {
- httpGet: { path: "/api/livez", port: "http" },
- periodSeconds: 30,
- timeoutSeconds: 3
- }
- } + if $agent == "" then {} else {
- env: [{ name: "CUMORA_AGENT_COMPUTER_IMAGE", value: $agent }]
- } end)
- ]
- } } }
- }
- ')
- kubectl patch deployment cumora-server --type=strategic -p="$PATCH"
-
- - name: Wait for rollout
- id: rollout
- run: |
- # If rollout times out, fall through to the post-mortem
- # step below — we want the diagnostic dump to print BEFORE
- # the job fails, otherwise GHA just shows "Process completed
- # with exit code 1" and we have to manually kubectl-investigate.
- set +e
- kubectl rollout status deployment/cumora-server --timeout=10m
- RC=$?
- set -e
- echo "rc=$RC" >> $GITHUB_OUTPUT
- # Don't exit yet — let the post-mortem step run regardless of RC.
- # The "Fail if rollout failed" step at the end actually fails the job.
-
- - name: Post-mortem on rollout failure
- if: steps.rollout.outputs.rc != '0'
- run: |
- echo "::group::Pods"
- kubectl get pods -l app=cumora-server -o wide
- echo "::endgroup::"
- echo "::group::Pod descriptions (events)"
- # Per-pod describe; tail keeps the output focused on Events
- for POD in $(kubectl get pods -l app=cumora-server -o name); do
- echo "==== $POD ===="
- kubectl describe "$POD" | sed -n '/Events:/,$p' | head -40
- done
- echo "::endgroup::"
- echo "::group::Deployment status"
- kubectl describe deployment cumora-server | grep -A 10 'Replicas:\|Conditions:\|StrategyType:'
- echo "::endgroup::"
- echo "::group::Recent cluster events"
- kubectl get events --sort-by=.lastTimestamp 2>&1 | tail -30
- echo "::endgroup::"
- echo "::group::Server container logs (current run)"
- for POD in $(kubectl get pods -l app=cumora-server -o jsonpath='{.items[*].metadata.name}'); do
- echo "==== $POD ===="
- kubectl logs "$POD" -c server --tail=100 2>&1 || true
- done
- echo "::endgroup::"
- echo "::group::Server container logs (previous run, if crashloop)"
- for POD in $(kubectl get pods -l app=cumora-server -o jsonpath='{.items[*].metadata.name}'); do
- echo "==== $POD ===="
- kubectl logs "$POD" -c server --previous --tail=100 2>&1 || true
- done
- echo "::endgroup::"
-
- - name: Fail if rollout failed
- if: steps.rollout.outputs.rc != '0'
- run: |
- echo "::error::Rollout failed with rc=${{ steps.rollout.outputs.rc }} — see Post-mortem step above."
- exit 1
-
- - name: Authenticated smoke test
- id: smoke
- continue-on-error: true
+ # This is the only mutating production step. The helper captures the
+ # baseline before migration, creates immutable state records, preserves
+ # Job identity/proxy/volumes/env/securityContext, applies JSON Patch CAS,
+ # handles every rollout failure and smoke failure, and never calls
+ # rollout undo.
+ - name: Run deployment transaction with guarded recovery
+ id: deploy
env:
+ RECOVERY_WORKDIR: ${{ runner.temp }}/cumora-recovery-${{ github.run_id }}
+ DEPLOYMENT: ${{ env.DEPLOYMENT }}
+ INCLUDE_AGENT: ${{ steps.tag.outputs.include_agent }}
+ RECOVERY_MODE: ${{ inputs.recovery_mode || 'normal' }}
+ CANDIDATE_SERVER_IMAGE: ${{ steps.digest.outputs.server }}
+ CANDIDATE_AGENT_IMAGE: ${{ steps.digest.outputs.agent }}
+ MIGRATION_REPAIR: ${{ inputs.repair_0002 || 'off' }}
+ MIGRATION_DEADLINE_SECONDS: '600'
+ MIGRATION_POLL_ATTEMPTS: '200'
+ VERIFIER_POLL_ATTEMPTS: '75'
+ RECOVERY_POLL_MS: '3000'
+ ROLLOUT_TIMEOUT_SECONDS: '600'
+ SMOKE_TIMEOUT_SECONDS: '120'
CUMORA_SMOKE_TOKEN: ${{ secrets.CUMORA_SMOKE_TOKEN }}
CUMORA_SMOKE_COMPANY_ID: ${{ secrets.CUMORA_SMOKE_COMPANY_ID }}
CUMORA_SMOKE_BASE: https://api.cumora.ai
- run: node scripts/release-smoke.mjs
-
- - name: Automatic rollback when smoke evidence fails
- if: steps.smoke.outcome == 'failure'
- run: |
- echo "::warning::Authenticated smoke failed; rolling deployment back to the previous revision."
- kubectl rollout undo deployment/cumora-server
- kubectl rollout status deployment/cumora-server --timeout=10m
- echo "::error::Candidate was rolled back. Previous server was ${{ steps.patch.outputs.previous_server }}."
-
- - name: Fail release after rollback
- if: steps.smoke.outcome == 'failure'
- run: exit 1
+ run: node --import tsx scripts/deploy-release.mjs run
- name: Summary
+ if: steps.deploy.outcome == 'success'
+ env:
+ SERVER_DIGEST: ${{ steps.digest.outputs.server }}
+ AGENT_DIGEST: ${{ steps.digest.outputs.agent || '(unchanged)' }}
run: |
- echo "### Deployed" >> $GITHUB_STEP_SUMMARY
- echo "- tag: \`${{ steps.tag.outputs.tag }}\`" >> $GITHUB_STEP_SUMMARY
- echo "- server: \`${{ steps.digest.outputs.server }}\`" >> $GITHUB_STEP_SUMMARY
- echo "- agent: \`${{ steps.digest.outputs.agent || '(unchanged)' }}\`" >> $GITHUB_STEP_SUMMARY
- echo "- previous server: \`${{ steps.patch.outputs.previous_server }}\`" >> $GITHUB_STEP_SUMMARY
- echo "- migration owner: one pre-deploy Job (\`${{ steps.migrate.outputs.job }}\`)" >> $GITHUB_STEP_SUMMARY
- echo "- authenticated smoke: passed" >> $GITHUB_STEP_SUMMARY
- echo "- cluster: \`${{ env.CLUSTER }}\` / \`${{ env.CLUSTER_LOCATION }}\`" >> $GITHUB_STEP_SUMMARY
+ printf '%s\n' '### Deployed' >> "$GITHUB_STEP_SUMMARY"
+ printf '%s\n' "- server digest: $SERVER_DIGEST" >> "$GITHUB_STEP_SUMMARY"
+ printf '%s\n' "- agent digest: $AGENT_DIGEST" >> "$GITHUB_STEP_SUMMARY"
+ printf '%s\n' '- candidate migration, rollout, and authenticated smoke: passed' >> "$GITHUB_STEP_SUMMARY"
diff --git a/docs/RELEASE.md b/docs/RELEASE.md
index 45d9c66..42a634f 100644
--- a/docs/RELEASE.md
+++ b/docs/RELEASE.md
@@ -48,18 +48,105 @@ To deploy a candidate:
precondition — see below.
4. Approve the protected `production` environment. The approver should not be
the person who built the feature for high-risk changes.
-5. Verify the workflow summary contains the selected digest, previous server
- image, completed rollout, and passed authenticated smoke.
+5. Verify the workflow summary contains the selected digest, a completed
+ rollout, and passed authenticated smoke. If the candidate fails, the
+ workflow remains red even when guarded recovery restores the previous
+ template successfully.
+
+The normal workflow requires the captured Deployment to have an observed,
+fully available rollout and a passing core baseline smoke. When that baseline
+is known to be unhealthy, an approver may explicitly choose
+`recovery_mode=forward-only` in the workflow dispatch. That mode still captures
+the UID/template, runs the candidate migration and CAS patch, and requires the
+candidate rollout plus strict Shipping smoke. It never runs automatic restore
+after a candidate rollout or smoke failure; the failed state is handled by a
+forward-compatible deployment plan.
Deploy first proves that the existing production API and smoke credential are
-healthy, runs one candidate-image migration Job and verifies its immutable
-ledger/index gates, records the current revision as the rollback baseline,
-updates the server (and optionally agent runtime) by digest, waits for GKE, then
-exercises real authenticated tenant paths: auth, conversations, and the
-Shipping overview/schema. A migration failure leaves the Deployment untouched;
-a failed post-deploy smoke automatically runs
-`kubectl rollout undo`, waits for the old revision to become ready, and fails
-the workflow.
+healthy, captures the complete Deployment Pod template and UID in an immutable
+create-only recovery Secret, runs one candidate-image migration Job, updates
+the server (and optionally agent runtime) by digest with an atomic
+compare-and-swap patch, waits for GKE, then exercises real authenticated tenant
+paths: auth, conversations, and the Shipping overview/schema. A migration
+failure leaves the Deployment untouched and retains the Job for inspection.
+The baseline and restored-template smoke checks cover the core authenticated
+paths; the candidate success check additionally requires Shipping overview and
+schema.
+
+The recovery path never uses `kubectl rollout undo`. Any failed rollout status
+or smoke failure first checks that the UID and exact candidate template still match the
+protected receipt. It then runs the captured old server image in an independent
+read-only schema verifier Job. The verifier imports its own `pool.ts`,
+`schema-version.ts`, and migration manifest, runs `BEGIN READ ONLY` with a
+30-second statement timeout, and must emit one valid compatibility marker. A
+missing module, timeout, malformed output, unknown schema, or template drift
+refuses recovery and leaves the candidate state for forward repair. Only a
+compatible result restores the exact captured template, waits for rollout, and
+runs authenticated smoke again. Recovery success is evidence for operators;
+the candidate workflow still fails.
+
+The baseline and candidate receipt Secrets contain Pod-template state and are
+immutable. Do not print, upload, or copy their contents. A retry reads and
+validates the original baseline instead of replacing it. Cancellation can skip
+cleanup, so inspect retained migration/verifier Jobs and the protected records
+before deleting anything.
+
+### Backend recovery runbook
+
+The release path classifies a failed candidate as follows:
+
+- **Same schema:** the old image's read-only verifier reports its current
+ ledger version inside that image's exact supported range. If the candidate
+ template has not drifted, the workflow restores the captured server/agent
+ images, probes, environment, mounts, and security context atomically, then
+ proves rollout and authenticated smoke.
+- **Newer or damaged schema:** the old verifier returns a known incompatible
+ result. Automatic restoration is refused. Keep the migration Job and
+ protected receipt, stop further deploy attempts, and use a forward-compatible
+ image or an explicit database repair plan. Lowering an image's MIN bound or
+ silently accepting an unknown migration is unsafe.
+- **Unknown compatibility:** missing old-image modules, a verifier timeout,
+ connection/cleanup error, a non-unique marker, invalid JSON, or any extra
+ unexpected output is treated as unknown. Do not restore automatically.
+- **Restore or smoke failure:** a failed JSON Patch CAS, rollout, or restored
+ smoke leaves the workflow failed and requires manual forward recovery. Check
+ the Deployment UID/template hash and controller events before a new run.
+
+The checked-in helper has an explicit recovery-only entry point. It does not
+capture a new baseline, run a migration, or patch a candidate before the
+compatibility check:
+
+```sh
+export KUBECTL=kubectl
+export DEPLOYMENT=cumora-server
+export DEPLOYMENT_NAMESPACE=default
+export RECOVERY_SECRET='cumora-deploy-recovery-<run-id>'
+export RECEIPT_SECRET='cumora-deploy-recovery-<run-id>-receipt'
+export RECOVERY_WORKDIR=/tmp/cumora-manual-recovery
+export VERIFIER_JOB_NAME="cumora-schema-verify-manual-$(date -u +%Y%m%d%H%M%S)"
+export CUMORA_SMOKE_TOKEN='(read from the protected operator environment)'
+export CUMORA_SMOKE_COMPANY_ID='(read from the protected operator environment)'
+export CUMORA_SMOKE_BASE=https://api.cumora.ai
+node --import tsx scripts/deploy-release.mjs recover
+```
+
+Use the exact protected Secret names from the failed run and run this from a
+checkout with the same helper. The command reads and validates the immutable
+baseline and candidate receipt, blocks on active migration Jobs, runs the old
+server image's read-only verifier under the fresh `VERIFIER_JOB_NAME`, then performs the UID/template CAS restore,
+rollout, and authenticated smoke. It never runs migration or accepts a
+mutable image. A missing baseline/receipt, active Job, schema incompatibility,
+unknown verifier result, or CAS drift returns a stable failure code and leaves
+the current Deployment unchanged. Inspect completed Jobs and their proxy state
+before retrying; do not delete an active Job to force a new result.
+
+If the receipt is missing or compatibility is unknown, use the explicit
+forward-only workflow mode: choose an image whose own manifest explicitly
+supports the current ledger, review any migration/expand-contract repair
+separately, and dispatch `Deploy` with `recovery_mode=forward-only`. Do not
+manufacture a baseline or lower the old image's supported minimum. Manual
+recovery success still belongs to the failed candidate incident; run a new
+reviewed forward deployment before marking the release successful.
#### When migration 0002 refuses to apply
diff --git a/docs/SHIPPING.md b/docs/SHIPPING.md
index b465c89..a3c66d5 100644
--- a/docs/SHIPPING.md
+++ b/docs/SHIPPING.md
@@ -71,9 +71,11 @@ the mute point without replaying the old backlog.
Backend deployment is intentionally separate from desktop tagging. See
[RELEASE.md](./RELEASE.md) for protected production approval, digest-pinned GKE
-rollout, authenticated smoke, automatic rollback, and scheduled readback.
+rollout, immutable recovery records, read-only schema compatibility checks,
+authenticated smoke, conditional exact-template recovery, and scheduled
+readback. A recovered candidate remains a failed release until a new reviewed
+forward deployment is verified.
The release contract is complete only after production behavior has been read
back against its baseline. A green build or successful rollout is an
intermediate signal, not the terminal state.
-
diff --git a/scripts/deploy-release.mjs b/scripts/deploy-release.mjs
new file mode 100644
index 0000000..4e5d134
--- /dev/null
+++ b/scripts/deploy-release.mjs
@@ -0,0 +1,490 @@
+import { spawn } from 'node:child_process'
+import { writeFile, mkdir } from 'node:fs/promises'
+import { join } from 'node:path'
+import { pathToFileURL } from 'node:url'
+
+const recovery = await import('../server/src/deploy/recovery.ts')
+
+const DEFAULT_ROLLOUT_TIMEOUT_SECONDS = 600
+const DEFAULT_JOB_TIMEOUT_SECONDS = 600
+const DEFAULT_POLL_SECONDS = 3
+const DEFAULT_POLL_ATTEMPTS = 30
+
+class CommandFailure extends Error {
+ constructor(code, details = {}) {
+ super(code)
+ this.code = code
+ this.details = details
+ }
+}
+
+function envRequired(name) {
+ const value = process.env[name]
+ if (!value) throw new CommandFailure(`missing_${name.toLowerCase()}`)
+ return value
+}
+
+function parseJson(raw, code) {
+ try {
+ return JSON.parse(raw)
+ } catch {
+ throw new CommandFailure(code)
+ }
+}
+
+function isComplete(job) {
+ return Number(job?.status?.succeeded ?? 0) > 0 || (Array.isArray(job?.status?.conditions) && job.status.conditions.some((condition) => condition?.type === 'Complete' && condition?.status === 'True'))
+}
+
+function isFailed(job) {
+ // A non-zero failed counter can coexist with active retries. Only the
+ // terminal Failed condition proves that a Job is no longer a writer.
+ return Array.isArray(job?.status?.conditions) && job.status.conditions.some((condition) => condition?.type === 'Failed' && condition?.status === 'True')
+}
+
+function hasActiveWriter(job) {
+ return Number(job?.status?.active ?? 0) > 0
+}
+
+function hasTerminalCondition(job) {
+ return isFailed(job) || (Array.isArray(job?.status?.conditions) && job.status.conditions.some((condition) => condition?.type === 'Complete' && condition?.status === 'True'))
+}
+
+function sleep(ms) {
+ return new Promise((resolve) => setTimeout(resolve, ms))
+}
+
+function runProcess(command, args, options = {}) {
+ const timeoutMs = options.timeoutMs ?? 120_000
+ return new Promise((resolve, reject) => {
+ const child = spawn(command, args, {
+ cwd: options.cwd,
+ env: options.env ?? process.env,
+ stdio: ['pipe', 'pipe', 'pipe'],
+ })
+ let stdout = ''
+ let stderr = ''
+ let timedOut = false
+ const timer = setTimeout(() => {
+ timedOut = true
+ child.kill('SIGTERM')
+ setTimeout(() => child.kill('SIGKILL'), 1_000).unref()
+ }, timeoutMs)
+ child.stdout.setEncoding('utf8')
+ child.stderr.setEncoding('utf8')
+ child.stdout.on('data', (chunk) => { stdout += chunk })
+ child.stderr.on('data', (chunk) => { stderr += chunk })
+ child.on('error', (error) => {
+ clearTimeout(timer)
+ reject(new CommandFailure('process_spawn_failed', { cause: error }))
+ })
+ child.on('close', (exitCode, signal) => {
+ clearTimeout(timer)
+ if (timedOut) {
+ reject(new CommandFailure('process_timeout', { stdout, stderr, signal }))
+ } else if (exitCode !== 0) {
+ reject(new CommandFailure('process_failed', { stdout, stderr, exitCode, signal }))
+ } else {
+ resolve({ stdout, stderr })
+ }
+ })
+ if (options.input) child.stdin.write(options.input)
+ child.stdin.end()
+ })
+}
+
+function namespaceArgs() {
+ const namespace = process.env.DEPLOYMENT_NAMESPACE
+ return namespace ? ['-n', namespace] : []
+}
+
+async function kubectl(args, options = {}) {
+ const command = process.env.KUBECTL || 'kubectl'
+ try {
+ return await runProcess(command, [...namespaceArgs(), ...args], options)
+ } catch (error) {
+ if (error instanceof CommandFailure) throw new CommandFailure(`kubectl_${error.code}`, error.details)
+ throw error
+ }
+}
+
+function workPath(name) {
+ const root = process.env.RECOVERY_WORKDIR || process.env.RUNNER_TEMP || '/tmp'
+ return join(root, `cumora-deploy-${process.env.GITHUB_RUN_ID || 'local'}-${name}`)
+}
+
+async function writePrivate(path, value) {
+ await writeFile(path, `${typeof value === 'string' ? value : JSON.stringify(value, null, 2)}\n`, { mode: 0o600 })
+}
+
+async function getDeployment(name) {
+ let result
+ try {
+ result = await kubectl(['get', 'deployment', name, '-o', 'json'])
+ } catch {
+ throw new CommandFailure('deployment_read_failed')
+ }
+ return parseJson(result.stdout, 'deployment_json_invalid')
+}
+
+async function getJob(name) {
+ const result = await kubectl(['get', 'job', name, '--ignore-not-found', '-o', 'json'])
+ if (!result.stdout.trim()) return null
+ return parseJson(result.stdout, 'job_json_invalid')
+}
+
+async function getSecret(name) {
+ try {
+ const result = await kubectl(['get', 'secret', name, '--ignore-not-found', '-o', 'json'])
+ if (!result.stdout.trim()) return null
+ return parseJson(result.stdout, 'secret_json_invalid')
+ } catch (_error) {
+ throw new CommandFailure('secret_read_failed')
+ }
+}
+
+async function getOptionalJob(name) {
+ try {
+ return await getJob(name)
+ } catch {
+ throw new CommandFailure('job_read_failed')
+ }
+}
+
+async function listResidualMigrationJobs() {
+ const items = []
+ for (const selector of ['app=cumora-schema-migration', 'app=cumora-schema-recovery']) {
+ let result
+ try {
+ result = await kubectl(['get', 'jobs', '--ignore-not-found', '-l', selector, '-o', 'json'])
+ } catch {
+ throw new CommandFailure('migration_jobs_read_failed')
+ }
+ if (!result.stdout.trim()) continue
+ const listing = parseJson(result.stdout, 'migration_jobs_json_invalid')
+ if (!listing || typeof listing !== 'object' || !Array.isArray(listing.items)) throw new CommandFailure('migration_jobs_json_invalid')
+ items.push(...listing.items)
+ }
+ return items
+}
+
+function secretData(secret, key) {
+ const encoded = secret?.data?.[key]
+ if (typeof encoded !== 'string') throw new CommandFailure('protected_state_missing')
+ try {
+ return parseJson(Buffer.from(encoded, 'base64').toString('utf8'), 'protected_state_invalid')
+ } catch {
+ throw new CommandFailure('protected_state_invalid')
+ }
+}
+
+async function createImmutableSecret(name, key, value, labels) {
+ const existing = await getSecret(name)
+ if (existing) return existing
+ const manifest = {
+ apiVersion: 'v1',
+ kind: 'Secret',
+ metadata: { name, labels },
+ immutable: true,
+ type: 'Opaque',
+ data: { [key]: Buffer.from(`${JSON.stringify(value)}\n`, 'utf8').toString('base64') },
+ }
+ try {
+ await kubectl(['create', '-f', '-'], { input: JSON.stringify(manifest) })
+ } catch (error) {
+ // A concurrent retry may have won the create-only race. Read and verify
+ // the protected state; never apply/update the original baseline Secret.
+ if (!/already.?exists|409/i.test(`${error.details?.stderr ?? ''} ${error.details?.stdout ?? ''}`)) {
+ throw new CommandFailure('protected_state_create_failed')
+ }
+ const raced = await getSecret(name)
+ if (!raced) throw new CommandFailure('protected_state_create_failed')
+ return raced
+ }
+ const created = await getSecret(name)
+ if (!created) throw new CommandFailure('protected_state_read_failed')
+ return created
+}
+
+async function captureBaseline(paths) {
+ const deployment = await getDeployment(paths.deployment)
+ let baseline = recovery.extractDeploymentSnapshot(deployment, {
+ expectedName: paths.deployment,
+ expectedNamespace: process.env.DEPLOYMENT_NAMESPACE,
+ })
+ const forwardOnly = process.env.RECOVERY_MODE === 'forward-only'
+ if (!forwardOnly) {
+ recovery.assertDeploymentHealthy(baseline)
+ if (!(await smoke(false))) throw new CommandFailure('baseline_smoke_failed')
+ }
+ baseline = { ...baseline, baselineSmokePassed: !forwardOnly, baselineVerifiedAt: new Date().toISOString() }
+ await writePrivate(paths.baseline, baseline)
+ const existing = await getSecret(paths.recoverySecret)
+ let protectedBaseline
+ if (existing) {
+ protectedBaseline = secretData(existing, 'baseline.json')
+ } else {
+ const created = await createImmutableSecret(paths.recoverySecret, 'baseline.json', baseline, {
+ app: 'cumora-deploy-recovery',
+ 'recovery.cumora.ai/run-id': process.env.GITHUB_RUN_ID || 'local',
+ })
+ protectedBaseline = secretData(created, 'baseline.json')
+ }
+ recovery.assertBaselineUnchanged(protectedBaseline, baseline)
+ return baseline
+}
+
+async function ensureBaselineUnchanged(paths, baseline) {
+ const current = recovery.extractDeploymentSnapshot(await getDeployment(paths.deployment), {
+ expectedName: baseline.name,
+ expectedNamespace: baseline.namespace,
+ })
+ recovery.assertBaselineUnchanged(baseline, current)
+ return current
+}
+
+async function runMigration(paths, baseline) {
+ await ensureBaselineUnchanged(paths, baseline)
+ const name = process.env.MIGRATION_JOB_NAME || `cumora-migrate-${(process.env.GITHUB_SHA || 'local').slice(0, 7)}-${process.env.GITHUB_RUN_ID || 'local'}`
+ const residual = await listResidualMigrationJobs()
+ if (residual.some((job) => hasActiveWriter(job) || !hasTerminalCondition(job))) throw new CommandFailure('migration_residual_job_present')
+ const existing = await getOptionalJob(name)
+ if (existing) throw new CommandFailure('migration_job_exists')
+ const job = recovery.buildMigrationJob(baseline, {
+ name,
+ image: envRequired('CANDIDATE_SERVER_IMAGE'),
+ repairMode: process.env.MIGRATION_REPAIR === 'archive-detach' ? 'archive-detach' : 'off',
+ activeDeadlineSeconds: Number(process.env.MIGRATION_DEADLINE_SECONDS || DEFAULT_JOB_TIMEOUT_SECONDS),
+ })
+ await kubectl(['apply', '-f', '-'], { input: JSON.stringify(job) })
+ await waitForJob(name, Number(process.env.MIGRATION_POLL_ATTEMPTS || DEFAULT_POLL_ATTEMPTS))
+ return name
+}
+
+async function waitForJob(name, attempts) {
+ const pollMs = Number(process.env.RECOVERY_POLL_MS || DEFAULT_POLL_SECONDS * 1_000)
+ for (let attempt = 0; attempt < attempts; attempt += 1) {
+ const job = await getJob(name)
+ if (!job) {
+ if (attempt + 1 < attempts) await sleep(pollMs)
+ continue
+ }
+ if (isComplete(job)) return job
+ if (isFailed(job)) throw new CommandFailure('job_failed')
+ if (attempt + 1 < attempts) await sleep(pollMs)
+ }
+ throw new CommandFailure('job_timeout')
+}
+
+async function applyCandidate(paths, baseline) {
+ const current = await ensureBaselineUnchanged(paths, baseline)
+ const images = { server: envRequired('CANDIDATE_SERVER_IMAGE') }
+ if (process.env.INCLUDE_AGENT !== 'N') {
+ const agent = envRequired('CANDIDATE_AGENT_IMAGE')
+ images.agent = agent
+ }
+ const patch = recovery.buildCandidatePatch(baseline, images)
+ await writePrivate(paths.candidatePatch, patch)
+ let response
+ try {
+ response = await kubectl(['patch', 'deployment', paths.deployment, '--type=json', '--patch-file', paths.candidatePatch, '-o', 'json'])
+ } catch {
+ throw new CommandFailure('candidate_patch_failed')
+ }
+ const receiptObject = parseJson(response.stdout, 'candidate_receipt_invalid')
+ const receipt = recovery.deploymentReceiptFromObject(receiptObject)
+ recovery.assertCandidateReceiptMatches(baseline, receipt, images)
+ await writePrivate(paths.receipt, receipt)
+ const existingReceipt = await getSecret(paths.receiptSecret)
+ let protectedReceipt
+ if (existingReceipt) {
+ protectedReceipt = secretData(existingReceipt, 'receipt.json')
+ } else {
+ const created = await createImmutableSecret(paths.receiptSecret, 'receipt.json', receipt, {
+ app: 'cumora-deploy-recovery',
+ 'recovery.cumora.ai/run-id': process.env.GITHUB_RUN_ID || 'local',
+ 'recovery.cumora.ai/receipt': 'true',
+ })
+ protectedReceipt = secretData(created, 'receipt.json')
+ }
+ if (protectedReceipt.uid !== receipt.uid || protectedReceipt.podTemplateHash !== receipt.podTemplateHash) {
+ throw new CommandFailure('candidate_receipt_conflict')
+ }
+ return { receipt, current }
+}
+
+async function rollout() {
+ try {
+ await kubectl(['rollout', 'status', `deployment/${process.env.DEPLOYMENT || 'cumora-server'}`, `--timeout=${Number(process.env.ROLLOUT_TIMEOUT_SECONDS || DEFAULT_ROLLOUT_TIMEOUT_SECONDS)}s`], {
+ timeoutMs: (Number(process.env.ROLLOUT_TIMEOUT_SECONDS || DEFAULT_ROLLOUT_TIMEOUT_SECONDS) + 5) * 1_000,
+ })
+ return { ok: true }
+ } catch {
+ return { ok: false }
+ }
+}
+
+async function smoke(requireShipping = true) {
+ try {
+ await runProcess(process.execPath, ['scripts/release-smoke.mjs'], {
+ cwd: process.env.GITHUB_WORKSPACE || process.cwd(),
+ timeoutMs: Number(process.env.SMOKE_TIMEOUT_SECONDS || 120) * 1_000,
+ env: { ...process.env, CUMORA_SMOKE_REQUIRE_SHIPPING: requireShipping ? 'Y' : 'N' },
+ })
+ return true
+ } catch {
+ return false
+ }
+}
+
+async function runSchemaVerifier(paths, baseline) {
+ const name = process.env.VERIFIER_JOB_NAME || `cumora-schema-verify-${(process.env.GITHUB_SHA || 'local').slice(0, 7)}-${process.env.GITHUB_RUN_ID || 'local'}`
+ const existing = await getOptionalJob(name)
+ if (existing) throw new CommandFailure('verifier_job_exists')
+ const job = recovery.buildSchemaVerifierJob(baseline, { name, image: baseline.serverImage })
+ await kubectl(['apply', '-f', '-'], { input: JSON.stringify(job) })
+ const status = await waitForJob(name, Number(process.env.VERIFIER_POLL_ATTEMPTS || DEFAULT_POLL_ATTEMPTS))
+ if (!isComplete(status)) throw new CommandFailure('verifier_failed')
+ let logs
+ try {
+ logs = await kubectl(['logs', `job/${name}`, '-c', 'migrate'])
+ } catch {
+ throw new CommandFailure('verifier_logs_failed')
+ }
+ // `kubectl logs` stdout is the verifier protocol. kubectl's own stderr is
+ // transport/diagnostic output and must not be mixed into the old image's
+ // strict single-marker stream.
+ const result = recovery.parseSchemaVerifierOutput(logs.stdout)
+ if (result.status !== 'compatible') throw new CommandFailure(result.status === 'incompatible' ? 'schema_incompatible' : 'schema_unknown')
+ return result
+}
+
+async function restore(paths, baseline, expectedHash) {
+ const currentObject = await getDeployment(paths.deployment)
+ const current = recovery.deploymentReceiptFromObject(currentObject)
+ const patch = recovery.buildRestorePatch(baseline, current, expectedHash)
+ await writePrivate(paths.restorePatch, patch)
+ try {
+ await kubectl(['patch', 'deployment', paths.deployment, '--type=json', '--patch-file', paths.restorePatch, '-o', 'json'])
+ } catch {
+ throw new CommandFailure('restore_patch_failed')
+ }
+ if (!(await rollout()).ok) throw new CommandFailure('restore_rollout_failed')
+ if (!(await smoke(false))) throw new CommandFailure('restore_smoke_failed')
+ const restored = recovery.extractDeploymentSnapshot(await getDeployment(paths.deployment), {
+ expectedName: baseline.name,
+ expectedNamespace: baseline.namespace,
+ })
+ if (restored.uid !== baseline.uid || restored.podTemplateHash !== baseline.podTemplateHash) {
+ throw new CommandFailure('restore_receipt_mismatch')
+ }
+ recovery.assertDeploymentHealthy(restored)
+}
+
+async function recover(paths, baseline) {
+ recovery.assertRecoveryEligible(baseline)
+ const protectedReceiptSecret = await getSecret(paths.receiptSecret)
+ if (!protectedReceiptSecret) throw new CommandFailure('protected_receipt_missing')
+ const protectedReceipt = secretData(protectedReceiptSecret, 'receipt.json')
+ recovery.assertReceiptShapeForRecovery(protectedReceipt)
+ recovery.assertRecoveryStateBinding(baseline, protectedReceipt)
+ // This first CAS check is deliberately before the old-image verifier. A
+ // controller/operator drift must never be hidden by a successful verifier.
+ const current = recovery.deploymentReceiptFromObject(await getDeployment(paths.deployment))
+ recovery.buildRestorePatch(baseline, current, protectedReceipt.podTemplateHash)
+ await runSchemaVerifier(paths, baseline)
+ await restore(paths, baseline, protectedReceipt.podTemplateHash)
+}
+
+async function recoverExistingDeployment() {
+ const state = paths()
+ await mkdir(state.root, { recursive: true, mode: 0o700 })
+ const baselineSecret = await getSecret(state.recoverySecret)
+ if (!baselineSecret) throw new CommandFailure('protected_baseline_missing')
+ const receiptSecret = await getSecret(state.receiptSecret)
+ if (!receiptSecret) throw new CommandFailure('protected_receipt_missing')
+ const baseline = secretData(baselineSecret, 'baseline.json')
+ const receipt = secretData(receiptSecret, 'receipt.json')
+ recovery.assertReceiptShapeForRecovery(receipt)
+ recovery.assertRecoveryStateBinding(baseline, receipt)
+ await writePrivate(state.baseline, baseline)
+ await writePrivate(state.receipt, receipt)
+ const residual = await listResidualMigrationJobs()
+ if (residual.some((job) => hasActiveWriter(job) || !hasTerminalCondition(job))) {
+ throw new CommandFailure('migration_residual_job_active')
+ }
+ const current = recovery.deploymentReceiptFromObject(await getDeployment(state.deployment))
+ recovery.buildRestorePatch(baseline, current, receipt.podTemplateHash)
+ recovery.assertRecoveryEligible(baseline)
+ await runSchemaVerifier(state, baseline)
+ await restore(state, baseline, receipt.podTemplateHash)
+ return { status: 'recovered', baselineHash: baseline.podTemplateHash }
+}
+
+function paths() {
+ const deployment = process.env.DEPLOYMENT || 'cumora-server'
+ return {
+ deployment,
+ recoverySecret: envRequired('RECOVERY_SECRET'),
+ receiptSecret: envRequired('RECEIPT_SECRET'),
+ root: process.env.RECOVERY_WORKDIR || process.env.RUNNER_TEMP || '/tmp',
+ baseline: workPath('baseline.json'),
+ candidatePatch: workPath('candidate-patch.json'),
+ receipt: workPath('receipt.json'),
+ restorePatch: workPath('restore-patch.json'),
+ }
+}
+
+export async function runDeploymentRelease() {
+ const state = paths()
+ await mkdir(state.root, { recursive: true, mode: 0o700 })
+ const baseline = await captureBaseline(state)
+ await runMigration(state, baseline)
+ await applyCandidate(state, baseline)
+ const rolledOut = await rollout()
+ if (!rolledOut.ok) {
+ if (process.env.RECOVERY_MODE === 'forward-only') throw new CommandFailure('candidate_failed_forward_only')
+ try {
+ await recover(state, baseline)
+ } catch (error) {
+ throw new CommandFailure(error instanceof CommandFailure ? error.code : 'recovery_failed')
+ }
+ throw new CommandFailure('candidate_failed_recovered')
+ }
+ if (!(await smoke(true))) {
+ if (process.env.RECOVERY_MODE === 'forward-only') throw new CommandFailure('candidate_failed_forward_only')
+ try {
+ await recover(state, baseline)
+ } catch (error) {
+ throw new CommandFailure(error instanceof CommandFailure ? error.code : 'recovery_failed')
+ }
+ throw new CommandFailure('candidate_failed_recovered')
+ }
+ const candidateObject = await getDeployment(state.deployment)
+ const candidateSnapshot = recovery.extractDeploymentSnapshot(candidateObject, {
+ expectedName: baseline.name,
+ expectedNamespace: baseline.namespace,
+ })
+ const candidate = recovery.deploymentReceiptFromObject(candidateObject)
+ const protectedReceipt = secretData(await getSecret(state.receiptSecret), 'receipt.json')
+ recovery.assertDeploymentReceiptMatches(baseline, candidate, protectedReceipt.podTemplateHash)
+ recovery.assertDeploymentHealthy(candidateSnapshot)
+ return { status: 'deployed', baselineHash: baseline.podTemplateHash }
+}
+
+export async function main(argv = process.argv.slice(2)) {
+ const command = argv[0] || 'run'
+ if (command === 'run') return runDeploymentRelease()
+ if (command === 'recover' || command === 'resume') return recoverExistingDeployment()
+ throw new CommandFailure('unknown_command')
+}
+
+if (import.meta.url === pathToFileURL(process.argv[1] ?? '').href) {
+ try {
+ const result = await main()
+ process.stdout.write(`${JSON.stringify(result)}\n`)
+ } catch (error) {
+ process.stderr.write(`deploy-release: ${error instanceof CommandFailure ? error.code : 'failed'}\n`)
+ process.exitCode = 1
+ }
+}
diff --git a/server/k8s/gke.md b/server/k8s/gke.md
index a4d6cb6..54a8423 100644
--- a/server/k8s/gke.md
+++ b/server/k8s/gke.md
@@ -219,7 +219,10 @@ After replacing `REPLACE-*` placeholders in
```sh
# For a manual installation, run the candidate image's migration command once
# against the same DATABASE_URL before starting application replicas. The
-# production Deploy workflow creates and verifies this one-shot Job for you.
+# production Deploy workflow creates and verifies this one-shot Job for you;
+# it captures a create-only Deployment recovery record before that Job and
+# restores an exact template only after the old image proves read-only schema
+# compatibility.
npm run migrate
kubectl apply -f server/k8s/cumora-server.gke.yaml
kubectl rollout status deployment/cumora-server
@@ -267,10 +270,17 @@ kubectl logs agent-<id>
- **Image upgrades** — tag both images with the same git sha;
redeploy by updating both the server Deployment image and
`CUMORA_AGENT_COMPUTER_IMAGE` to the same tag. The Deploy workflow runs one
- candidate migration Job before mutating the Deployment.
+ candidate migration Job before mutating the Deployment. Production resolves
+ both to immutable `@sha256:` digests and uses a JSON Patch UID/template CAS;
+ mutable tags, unknown image sources, and template drift are refused.
- **PG schema changes** — append an immutable version and checksum; never edit
an applied migration. Use expand/contract changes that remain compatible with
- the old server version still serving during the rolling-update window.
+ the old server version still serving during the rolling-update window. A
+ rollout timeout or smoke failure may recover only when the captured old image
+ reports the current ledger inside its own supported range through the
+ independent read-only verifier. Unknown or newer schema history requires a
+ forward-compatible image or reviewed repair; it is never treated as proof
+ that an arbitrary old image can start.
- **Idle scheduler** — runs in-process on EACH server replica.
That's fine because idle's tick currently publishes to
CH_MESSAGE_NEW which SETNX-dedups; only one replica handles
diff --git a/server/src/__integration__/deployment-schema-recovery.test.ts b/server/src/__integration__/deployment-schema-recovery.test.ts
new file mode 100644
index 0000000..cc626c0
--- /dev/null
+++ b/server/src/__integration__/deployment-schema-recovery.test.ts
@@ -0,0 +1,262 @@
+import assert from 'node:assert/strict'
+import { execFile as execFileCallback } from 'node:child_process'
+import { access, mkdtemp, mkdir, rm, symlink, unlink, writeFile } from 'node:fs/promises'
+import { fileURLToPath, pathToFileURL } from 'node:url'
+import { dirname, join, resolve } from 'node:path'
+import { tmpdir } from 'node:os'
+import { after, before, test } from 'node:test'
+import { promisify } from 'node:util'
+import { pool } from '../db/pool.js'
+import { MAX_SUPPORTED_SCHEMA_VERSION, MIN_SUPPORTED_SCHEMA_VERSION, SCHEMA_MIGRATIONS } from '../db/migrations/manifest.js'
+import { ensureSchemaOnce, teardownAll } from './_helpers.js'
+import {
+ buildSchemaVerifierScript,
+ parseSchemaVerifierOutput,
+} from '../deploy/recovery.js'
+
+const execFile = promisify(execFileCallback)
+const repoRoot = resolve(dirname(fileURLToPath(import.meta.url)), '../../..')
+const createdNodeModulesLinks = new Set<string>()
+
+interface SchemaVerifierRun {
+ status: string
+ [key: string]: unknown
+}
+
+function databaseUrl(): string {
+ const value = process.env.INTEGRATION_DATABASE_URL ?? process.env.DATABASE_URL
+ assert.ok(value, 'integration database URL is required')
+ return value
+}
+
+async function runVerifier(cwd = repoRoot, script = buildSchemaVerifierScript()): Promise<SchemaVerifierRun> {
+ await ensureNodeModules(cwd)
+ const env = {
+ ...process.env,
+ DATABASE_URL: databaseUrl(),
+ NODE_ENV: process.env.NODE_ENV ?? 'test',
+ OPENAI_API_KEY: process.env.OPENAI_API_KEY ?? 'integration-test-key',
+ NODE_PATH: process.env.NODE_PATH ?? join(repoRoot, 'node_modules'),
+ }
+ try {
+ const result = await execFile(
+ process.execPath,
+ ['--import', 'tsx', '--input-type=module', '-e', script],
+ { cwd, env, maxBuffer: 1024 * 1024, timeout: 15_000 },
+ )
+ return parseSchemaVerifierOutput(result.stdout, result.stderr) as unknown as SchemaVerifierRun
+ } catch (error) {
+ const failed = error as { stdout?: string; stderr?: string }
+ return parseSchemaVerifierOutput(failed.stdout ?? '', failed.stderr ?? '') as unknown as SchemaVerifierRun
+ }
+}
+
+async function ensureNodeModules(cwd: string): Promise<void> {
+ const target = join(cwd, 'node_modules')
+ try {
+ await access(target)
+ return
+ } catch {
+ // CI has a real install. The fallback keeps this test runnable from a
+ // clean worktree when the sibling checkout already owns the dependency
+ // tree; it is removed by after() and never enters git.
+ }
+ const fallbacks = [
+ process.env.CUMORA_TEST_NODE_MODULES,
+ join(repoRoot, 'node_modules'),
+ resolve(repoRoot, '..', 'cumora', 'node_modules'),
+ ].filter((value): value is string => Boolean(value))
+ for (const fallback of fallbacks) {
+ if (fallback === target) continue
+ try {
+ await access(fallback)
+ await symlink(fallback, target, 'dir')
+ createdNodeModulesLinks.add(target)
+ return
+ } catch {
+ // Try the next workspace-local dependency location.
+ }
+ }
+ throw new Error('node_modules is required for the old-image verifier fixture')
+}
+
+async function makeReadOnlyVerifierImageLikeDir(): Promise<string> {
+ const root = await mkdtemp(join(tmpdir(), 'cumora-old-image-schema-'))
+ await ensureNodeModules(root)
+ await writeFile(join(root, 'package.json'), '{"type":"module"}\n')
+ const dbDir = join(root, 'server', 'src', 'db')
+ const migrationsDir = join(dbDir, 'migrations')
+ await mkdir(migrationsDir, { recursive: true })
+ const realPool = pathToFileURL(resolve(repoRoot, 'server/src/db/pool.ts')).href
+ const realManifest = pathToFileURL(resolve(repoRoot, 'server/src/db/migrations/manifest.ts')).href
+ const realSchemaVersion = pathToFileURL(resolve(repoRoot, 'server/src/db/schema-version.ts')).href
+ await writeFile(join(dbDir, 'pool.ts'), `export { pool } from ${JSON.stringify(realPool)}\n`)
+ await writeFile(join(migrationsDir, 'manifest.ts'), `export * from ${JSON.stringify(realManifest)}\n`)
+ await writeFile(join(dbDir, 'schema-version.ts'), `
+import { verifySchemaCompatibility as verifyRealSchemaCompatibility } from ${JSON.stringify(realSchemaVersion)}
+
+export async function verifySchemaCompatibility(client) {
+ const { rows } = await client.query('SHOW transaction_read_only')
+ if (rows[0]?.transaction_read_only !== 'on') throw new Error('transaction is not read-only')
+ await client.query('SAVEPOINT read_only_probe')
+ try {
+ await client.query('CREATE TEMP TABLE cumora_verifier_read_only_probe (id integer)')
+ throw new Error('read-only transaction accepted DDL')
+ } catch (error) {
+ if (error?.code !== '25006') throw error
+ await client.query('ROLLBACK TO SAVEPOINT read_only_probe')
+ }
+ await client.query('RELEASE SAVEPOINT read_only_probe')
+ return verifyRealSchemaCompatibility(client)
+}
+`)
+ return root
+}
+
+async function withCommittedMutation<T>(
+ mutate: (client: import('pg').PoolClient) => Promise<void>,
+ restore: (client: import('pg').PoolClient) => Promise<void>,
+ check: () => Promise<T>,
+): Promise<T> {
+ const client = await pool.connect()
+ try {
+ await client.query('BEGIN')
+ await mutate(client)
+ await client.query('COMMIT')
+ try {
+ return await check()
+ } finally {
+ await client.query('BEGIN')
+ await restore(client)
+ await client.query('COMMIT')
+ }
+ } catch (error) {
+ await client.query('ROLLBACK').catch(() => {})
+ throw error
+ } finally {
+ client.release()
+ }
+}
+
+before(async () => { await ensureSchemaOnce() })
+after(async () => {
+ await teardownAll()
+ for (const link of createdNodeModulesLinks) await unlink(link).catch(() => {})
+ createdNodeModulesLinks.clear()
+})
+
+test('the actual inline verifier reads current PostgreSQL schema in read-only mode', async () => {
+ const beforeRows = await pool.query(
+ `SELECT version, name, checksum, execution_ms, applied_at
+ FROM schema_migrations ORDER BY version`,
+ )
+ const result = await runVerifier()
+ assert.equal(result.status, 'compatible', JSON.stringify(result))
+ assert.equal(result.currentVersion, SCHEMA_MIGRATIONS.at(-1)?.version)
+ assert.equal(result.minSupported, MIN_SUPPORTED_SCHEMA_VERSION)
+ assert.equal(result.maxSupported, MAX_SUPPORTED_SCHEMA_VERSION)
+ const afterRows = await pool.query(
+ `SELECT version, name, checksum, execution_ms, applied_at
+ FROM schema_migrations ORDER BY version`,
+ )
+ assert.deepEqual(afterRows.rows, beforeRows.rows, 'read-only verifier changed schema ledger')
+})
+
+test('the inline verifier enforces transaction READ ONLY through an old-image-like wrapper', async () => {
+ const imageLikeDir = await makeReadOnlyVerifierImageLikeDir()
+ try {
+ const beforeRows = await pool.query(
+ `SELECT version, name, checksum, execution_ms, applied_at
+ FROM schema_migrations ORDER BY version`,
+ )
+ const result = await runVerifier(imageLikeDir)
+ assert.equal(result.status, 'compatible', JSON.stringify(result))
+ assert.equal(result.currentVersion, SCHEMA_MIGRATIONS.at(-1)?.version)
+ const afterRows = await pool.query(
+ `SELECT version, name, checksum, execution_ms, applied_at
+ FROM schema_migrations ORDER BY version`,
+ )
+ assert.deepEqual(afterRows.rows, beforeRows.rows)
+ const { rows } = await pool.query<{ regclass: string | null }>(
+ `SELECT to_regclass('public.cumora_verifier_read_only_probe') AS regclass`,
+ )
+ assert.equal(rows[0]?.regclass, null, 'read-only verifier left a DDL probe table')
+ } finally {
+ await rm(imageLikeDir, { recursive: true, force: true })
+ }
+})
+
+test('an old image without verifier modules is an explicit unknown result', async () => {
+ const imageLikeDir = await mkdtemp(join(tmpdir(), 'cumora-old-image-missing-verifier-'))
+ await ensureNodeModules(imageLikeDir)
+ await writeFile(join(imageLikeDir, 'package.json'), '{"type":"module"}\n')
+ try {
+ const result = await runVerifier(imageLikeDir)
+ assert.equal(result.status, 'unknown', JSON.stringify(result))
+ assert.equal(result.reasonCode, 'module_unavailable')
+ } finally {
+ await rm(imageLikeDir, { recursive: true, force: true })
+ }
+})
+
+test('the actual verifier reports future schema as known incompatible', async () => {
+ const version = MAX_SUPPORTED_SCHEMA_VERSION + 1
+ const result = await withCommittedMutation(
+ async (client) => {
+ await client.query(
+ `INSERT INTO schema_migrations (version, name, checksum, execution_ms)
+ VALUES ($1, $2, $3, 0)`,
+ [version, `000${version}_test_future`, 'f'.repeat(64)],
+ )
+ },
+ (client) => client.query('DELETE FROM schema_migrations WHERE version = $1', [version]).then(() => undefined),
+ runVerifier,
+ )
+ assert.equal(result.status, 'incompatible', JSON.stringify(result))
+ assert.equal(result.code, 'schema_ahead')
+})
+
+test('the actual verifier rejects corrupted and gapped immutable history', async () => {
+ const original = await pool.query<{
+ version: number
+ name: string
+ checksum: string
+ execution_ms: number
+ applied_at: Date
+ }>(
+ `SELECT version, name, checksum, execution_ms, applied_at
+ FROM schema_migrations WHERE version = $1`,
+ [MAX_SUPPORTED_SCHEMA_VERSION - 1],
+ )
+ assert.equal(original.rows.length, 1)
+ const row = original.rows[0]!
+
+ const corrupted = await withCommittedMutation(
+ (client) => client.query(
+ `UPDATE schema_migrations SET checksum = $2 WHERE version = $1`,
+ [1, '0'.repeat(64)],
+ ).then(() => undefined),
+ (client) => client.query(
+ `UPDATE schema_migrations SET checksum = $2 WHERE version = $1`,
+ [1, SCHEMA_MIGRATIONS[0]!.checksum],
+ ).then(() => undefined),
+ runVerifier,
+ )
+ assert.equal(corrupted.status, 'incompatible', JSON.stringify(corrupted))
+ assert.equal(corrupted.code, 'migration_history_invalid')
+
+ const gapped = await withCommittedMutation(
+ (client) => client.query(
+ `DELETE FROM schema_migrations WHERE version = $1`,
+ [row.version],
+ ).then(() => undefined),
+ (client) => client.query(
+ `INSERT INTO schema_migrations (version, name, checksum, execution_ms, applied_at)
+ VALUES ($1, $2, $3, $4, $5)`,
+ [row.version, row.name, row.checksum, row.execution_ms, row.applied_at],
+ ).then(() => undefined),
+ runVerifier,
+ )
+ assert.equal(gapped.status, 'incompatible', JSON.stringify(gapped))
+ assert.equal(gapped.code, 'migration_history_invalid')
+})
diff --git a/server/src/__integration__/ws-doc-authorization.test.ts b/server/src/__integration__/ws-doc-authorization.test.ts
index 572a8f0..1024e31 100644
--- a/server/src/__integration__/ws-doc-authorization.test.ts
+++ b/server/src/__integration__/ws-doc-authorization.test.ts
@@ -89,11 +89,11 @@ async function waitForFrame(
predicate: (frame: Frame) => boolean,
label: string,
timeoutMs = FRAME_TIMEOUT_MS,
+ fromIndex = 0,
): Promise<Frame> {
- const start = harness.messages.length
const deadline = Date.now() + timeoutMs
while (Date.now() < deadline) {
- const found = harness.messages.slice(start).find(predicate)
+ const found = harness.messages.slice(fromIndex).find(predicate)
if (found) return found
await delay(5)
}
@@ -156,11 +156,14 @@ async function openSocket(userId: string): Promise<SocketHarness> {
async function subscribeSocket(harness: SocketHarness, documentId: string, replaySupported = true, omitCapability = false): Promise<void> {
const frame: Record<string, unknown> = { type: 'doc.subscribe', documentId }
if (!omitCapability) frame.replaySupported = replaySupported
+ const cursor = harness.messages.length
harness.socket.send(JSON.stringify(frame))
await waitForFrame(
harness,
(frame) => frame.type === 'doc.sync' && frame.documentId === documentId,
`doc.sync for ${documentId}`,
+ FRAME_TIMEOUT_MS,
+ cursor,
)
}
@@ -428,11 +431,18 @@ test('revocation committed before a retried authorization prevents the queued se
await beginUserLock(barrier, fixture.users[1]!)
const afterOrigin = `after-revoke-${randomUUID()}`
const targetBefore = target.messages.length
+ const peerCursor = peer.messages.length
const fanout = emitAwareness(fixture, afterOrigin)
await revokeMembership(fixture.users[1]!, fixture.companyId)
await barrier.query('ROLLBACK')
await fanout
- await waitForFrame(peer, (frame) => frame.type === 'doc.awareness' && frame.originId === afterOrigin, 'valid peer after revoke')
+ await waitForFrame(
+ peer,
+ (frame) => frame.type === 'doc.awareness' && frame.originId === afterOrigin,
+ 'valid peer after revoke',
+ FRAME_TIMEOUT_MS,
+ peerCursor,
+ )
await delay(100)
assert.equal(
target.messages.slice(targetBefore).some((frame) => frame.originId === afterOrigin),
@@ -534,11 +544,14 @@ test('close and unsubscribe during blocked hydration do not leave active subscri
// first sync may be dropped legitimately: FIFO can run the queued
// unsubscribe immediately after hydration, before the sync's outbound
// authorization batch gets its turn.
+ const syncCursor = unsubscribeSocket.messages.length
unsubscribeSocket.socket.send(JSON.stringify({ type: 'doc.subscribe', documentId: unsubscribeFixture.documentId }))
await waitForFrame(
unsubscribeSocket,
(frame) => frame.type === 'doc.sync' && frame.documentId === unsubscribeFixture.documentId,
'second sync after queued unsubscribe',
+ FRAME_TIMEOUT_MS,
+ syncCursor,
)
unsubscribeSocket.socket.send(JSON.stringify({ type: 'doc.unsubscribe', documentId: unsubscribeFixture.documentId }))
await delay(50)
diff --git a/server/src/__tests__/deployment-recovery.test.ts b/server/src/__tests__/deployment-recovery.test.ts
new file mode 100644
index 0000000..aa55b2f
--- /dev/null
+++ b/server/src/__tests__/deployment-recovery.test.ts
@@ -0,0 +1,278 @@
+import assert from 'node:assert/strict'
+import { test } from 'node:test'
+import {
+ DEPLOYMENT_PROBE_CONTRACT,
+ SCHEMA_VERIFIER_MARKER,
+ assertBaselineUnchanged,
+ buildCandidatePatch,
+ buildCandidateTemplate,
+ buildMigrationJob,
+ buildRestorePatch,
+ buildSchemaVerifierJob,
+ buildSchemaVerifierScript,
+ deploymentReceiptFromObject,
+ extractDeploymentSnapshot,
+ hashPodTemplate,
+ parseSchemaVerifierOutput,
+} from '../deploy/recovery.js'
+
+const DIGEST_OLD = 'a'.repeat(64)
+const DIGEST_NEW = 'b'.repeat(64)
+const DIGEST_CANDIDATE = 'c'.repeat(64)
+const DIGEST_PROXY = 'd'.repeat(64)
+const DIGEST_MIGRATE = 'e'.repeat(64)
+
+function deploymentFixture(): Record<string, unknown> {
+ return {
+ apiVersion: 'apps/v1',
+ kind: 'Deployment',
+ metadata: { name: 'cumora-server', uid: 'deployment-uid-1' },
+ spec: {
+ template: {
+ metadata: { labels: { app: 'cumora-server' }, annotations: { trace: 'fixture' } },
+ spec: {
+ serviceAccountName: 'cumora-server',
+ imagePullSecrets: [{ name: 'quay-pull' }],
+ securityContext: {
+ runAsNonRoot: true,
+ seccompProfile: { type: 'RuntimeDefault' },
+ },
+ automountServiceAccountToken: true,
+ volumes: [
+ { name: 'proxy-cert', secret: { secretName: 'cloud-sql-cert' } },
+ { name: 'scratch', emptyDir: {} },
+ ],
+ initContainers: [
+ {
+ name: 'migrate',
+ image: `old-migrate@sha256:${DIGEST_MIGRATE}`,
+ command: ['npm', 'run', 'migrate'],
+ },
+ ],
+ containers: [
+ {
+ name: 'cloud-sql-proxy',
+ image: `cloud-sql-proxy@sha256:${DIGEST_PROXY}`,
+ args: ['--port=5432'],
+ env: [{ name: 'PROXY_MODE', value: 'fixture' }],
+ volumeMounts: [{ name: 'proxy-cert', mountPath: '/var/run/proxy', readOnly: true }],
+ securityContext: {
+ runAsNonRoot: true,
+ allowPrivilegeEscalation: false,
+ capabilities: { drop: ['ALL'] },
+ },
+ readinessProbe: { httpGet: { path: '/readiness', port: 9090 } },
+ resources: { requests: { cpu: '50m', memory: '64Mi' } },
+ },
+ {
+ name: 'server',
+ image: `server@sha256:${DIGEST_OLD}`,
+ envFrom: [{ secretRef: { name: 'cumora' } }],
+ env: [{ name: 'CUMORA_AGENT_COMPUTER_IMAGE', value: `agent@sha256:${DIGEST_OLD}` }],
+ volumeMounts: [{ name: 'scratch', mountPath: '/tmp/cumora' }],
+ securityContext: { runAsUser: 1000, allowPrivilegeEscalation: false },
+ startupProbe: DEPLOYMENT_PROBE_CONTRACT.startupProbe,
+ readinessProbe: DEPLOYMENT_PROBE_CONTRACT.readinessProbe,
+ livenessProbe: DEPLOYMENT_PROBE_CONTRACT.livenessProbe,
+ resources: { requests: { cpu: '250m', memory: '512Mi' } },
+ },
+ ],
+ },
+ },
+ },
+ }
+}
+
+function snapshot() {
+ return extractDeploymentSnapshot(deploymentFixture(), { capturedAt: '2026-09-10T00:00:00.000Z' })
+}
+
+function templateSpec(job: Record<string, any>): Record<string, any> {
+ return job.spec.template.spec
+}
+
+test('captures immutable template and builds exact CAS candidate/restore patches', () => {
+ const baseline = snapshot()
+ const candidate = buildCandidateTemplate(baseline, {
+ server: `server@sha256:${DIGEST_NEW}`,
+ agent: `agent@sha256:${DIGEST_NEW}`,
+ })
+ const candidateRecord = candidate as any
+ const baselineRecord = baseline.podTemplate as any
+ const candidateServer = candidateRecord.spec.containers.find((container: any) => container.name === 'server')
+ assert.equal(candidateServer.image, `server@sha256:${DIGEST_NEW}`)
+ assert.equal(candidateServer.env.find((entry: any) => entry.name === 'CUMORA_AGENT_COMPUTER_IMAGE').value, `agent@sha256:${DIGEST_NEW}`)
+ assert.deepEqual(candidateRecord.spec.volumes, baselineRecord.spec.volumes)
+ assert.deepEqual(candidateRecord.spec.securityContext, baselineRecord.spec.securityContext)
+ assert.deepEqual(candidateRecord.metadata, (baseline.podTemplate as any).metadata)
+ assert.deepEqual(candidateRecord.spec.initContainers, [], 'retired migration init container must be removed')
+ assert.deepEqual(candidateServer.startupProbe, DEPLOYMENT_PROBE_CONTRACT.startupProbe)
+ assert.deepEqual(candidateServer.readinessProbe, DEPLOYMENT_PROBE_CONTRACT.readinessProbe)
+ assert.deepEqual(candidateServer.livenessProbe, DEPLOYMENT_PROBE_CONTRACT.livenessProbe)
+
+ const candidatePatch = buildCandidatePatch(baseline, { server: `server@sha256:${DIGEST_NEW}` })
+ assert.deepEqual(candidatePatch[0], { op: 'test', path: '/metadata/uid', value: baseline.uid })
+ assert.deepEqual(candidatePatch[1], { op: 'test', path: '/spec/template', value: baseline.podTemplate })
+ assert.equal((candidatePatch[2] as any)?.op, 'replace')
+
+ const candidateReceipt = deploymentReceiptFromObject({
+ ...deploymentFixture(),
+ spec: { template: candidate },
+ }, { observedAt: '2026-09-10T00:01:00.000Z' })
+ assert.equal(candidateReceipt.serverImage, `server@sha256:${DIGEST_NEW}`)
+ assert.equal(candidateReceipt.agentImage, `agent@sha256:${DIGEST_NEW}`)
+ assert.equal(candidateReceipt.podTemplateHash, hashPodTemplate(candidate))
+ const restorePatch = buildRestorePatch(baseline, candidateReceipt, candidateReceipt.podTemplateHash)
+ assert.deepEqual(restorePatch[0], { op: 'test', path: '/metadata/uid', value: baseline.uid })
+ assert.deepEqual(restorePatch[1], { op: 'test', path: '/spec/template', value: candidate })
+ assert.deepEqual(restorePatch[2], { op: 'replace', path: '/spec/template', value: baseline.podTemplate })
+ assert.equal(hashPodTemplate(baseline.podTemplate), baseline.podTemplateHash)
+
+ assertBaselineUnchanged(baseline, extractDeploymentSnapshot(deploymentFixture()))
+ assert.throws(
+ () => buildRestorePatch(baseline, { ...candidateReceipt, uid: 'other-uid' }, candidateReceipt.podTemplateHash),
+ /UID changed/,
+ )
+ assert.throws(
+ () => buildRestorePatch(baseline, candidateReceipt, 'wrong-candidate-hash'),
+ /template drifted/,
+ )
+
+ const mutableDeployment = deploymentFixture()
+ ;(mutableDeployment.spec as any).template.spec.containers[1].image = 'server:latest'
+ assert.throws(
+ () => extractDeploymentSnapshot(mutableDeployment),
+ /digest|immutable|automatic recovery/i,
+ )
+
+ const missingServerImage = deploymentFixture()
+ delete (missingServerImage.spec as any).template.spec.containers[1].image
+ assert.throws(() => extractDeploymentSnapshot(missingServerImage), /server container image/)
+
+ const valueFromServerImage = deploymentFixture()
+ ;(valueFromServerImage.spec as any).template.spec.containers[1].image = { valueFrom: { configMapKeyRef: { name: 'image' } } }
+ assert.throws(() => extractDeploymentSnapshot(valueFromServerImage), /server container image/)
+
+ const shortDigestServerImage = deploymentFixture()
+ ;(shortDigestServerImage.spec as any).template.spec.containers[1].image = 'server@sha256:shorthex'
+ assert.throws(() => extractDeploymentSnapshot(shortDigestServerImage), /sha256 digest/)
+
+ const mutableAgentImage = deploymentFixture()
+ ;(mutableAgentImage.spec as any).template.spec.containers[1].env = [
+ { name: 'CUMORA_AGENT_COMPUTER_IMAGE', value: 'agent:latest' },
+ ]
+ assert.throws(() => extractDeploymentSnapshot(mutableAgentImage), /agent container image|sha256 digest/)
+
+ const valueFromAgentImage = deploymentFixture()
+ ;(valueFromAgentImage.spec as any).template.spec.containers[1].env = [
+ { name: 'CUMORA_AGENT_COMPUTER_IMAGE', valueFrom: { secretKeyRef: { name: 'agent-image' } } },
+ ]
+ const valueFromAgentBaseline = extractDeploymentSnapshot(valueFromAgentImage)
+ const valueFromAgentReceipt = deploymentReceiptFromObject(valueFromAgentImage)
+ assert.throws(
+ () => buildRestorePatch(valueFromAgentBaseline, valueFromAgentReceipt, valueFromAgentReceipt.podTemplateHash),
+ /agent.*image|valueFrom|automatic recovery/i,
+ )
+})
+
+test('migration and verifier Jobs preserve pod identity, mounts, volumes, and security contexts', () => {
+ const baseline = snapshot()
+ const migration = buildMigrationJob(baseline, {
+ name: 'cumora-migrate-test',
+ image: `server@sha256:${DIGEST_CANDIDATE}`,
+ repairMode: 'off',
+ })
+ const verifier = buildSchemaVerifierJob(baseline, {
+ name: 'cumora-verify-test',
+ image: baseline.serverImage,
+ })
+ assert.throws(
+ () => buildSchemaVerifierJob(baseline, { name: 'cumora-verify-candidate', image: `server@sha256:${DIGEST_CANDIDATE}` }),
+ /baseline.*image|verifier.*image|old image/i,
+ )
+
+ for (const job of [migration, verifier]) {
+ const pod = templateSpec(job)
+ const baselineSpec = baseline.podTemplate as any
+ assert.equal(pod.serviceAccountName, 'cumora-server')
+ assert.deepEqual(pod.imagePullSecrets, baselineSpec.spec.imagePullSecrets)
+ assert.deepEqual(pod.securityContext, baselineSpec.spec.securityContext)
+ assert.deepEqual(pod.volumes, baselineSpec.spec.volumes)
+ assert.equal(pod.restartPolicy, 'Never')
+ const proxy = pod.initContainers.find((container: any) => container.name === 'cloud-sql-proxy')
+ assert.ok(proxy)
+ assert.deepEqual(proxy.volumeMounts, baselineSpec.spec.containers[0].volumeMounts)
+ assert.deepEqual(proxy.securityContext, baselineSpec.spec.containers[0].securityContext)
+ assert.equal(proxy.restartPolicy, 'Always')
+ assert.ok(proxy.startupProbe)
+ assert.ok(!pod.initContainers.some((container: any) => container.name === 'migrate'))
+ }
+
+ const migrationServer = templateSpec(migration).containers[0]
+ assert.equal(migrationServer.name, 'migrate')
+ assert.deepEqual(migrationServer.envFrom, [{ secretRef: { name: 'cumora' } }])
+ assert.deepEqual(migrationServer.command, ['npm', 'run', 'migrate'])
+ assert.equal(migrationServer.env.find((entry: any) => entry.name === 'MIGRATION_0002_REPAIR').value, 'off')
+
+ const verifierServer = templateSpec(verifier).containers[0]
+ assert.equal(verifierServer.name, 'migrate')
+ assert.deepEqual(verifierServer.envFrom, [{ secretRef: { name: 'cumora' } }])
+ assert.equal(verifierServer.workingDir, '/app')
+ assert.equal((verifier.spec as { activeDeadlineSeconds?: number }).activeDeadlineSeconds, 180)
+ assert.deepEqual(verifierServer.command.slice(0, 5), ['node', '--import', 'tsx', '--input-type=module', '-e'])
+ assert.match(verifierServer.args?.[0] ?? '', /BEGIN READ ONLY/)
+})
+
+test('verifier script is old-image-local, read-only, bounded, and uses strict protocol output', () => {
+ const script = buildSchemaVerifierScript()
+ assert.match(script, /\.\/server\/src\/db\/pool\.ts/)
+ assert.match(script, /\.\/server\/src\/db\/schema-version\.ts/)
+ assert.match(script, /BEGIN READ ONLY/)
+ assert.match(script, /COMMIT/)
+ assert.match(script, /pool\.end\(\)/)
+ assert.doesNotMatch(script, /\/app\/server\/src\/db/)
+
+ const compatible = parseSchemaVerifierOutput(
+ `${SCHEMA_VERIFIER_MARKER} ${JSON.stringify({
+ protocol: 1,
+ kind: 'schema-verifier',
+ status: 'compatible',
+ currentVersion: 7,
+ minSupported: 7,
+ maxSupported: 7,
+ })}\n`,
+ ) as unknown as Record<string, unknown>
+ assert.equal(compatible.status, 'compatible')
+ assert.equal(compatible.currentVersion, 7)
+ assert.equal(compatible.minSupported, 7)
+ assert.equal(compatible.maxSupported, 7)
+
+ const incompatible = parseSchemaVerifierOutput(
+ `${SCHEMA_VERIFIER_MARKER} ${JSON.stringify({
+ protocol: 1, kind: 'schema-verifier', status: 'incompatible', currentVersion: null,
+ minSupported: null, maxSupported: null, code: 'schema_ahead',
+ })}\n`,
+ ) as unknown as Record<string, unknown>
+ assert.equal(incompatible.status, 'incompatible')
+ assert.equal(incompatible.code, 'schema_ahead')
+
+ const unknown = parseSchemaVerifierOutput(
+ `${SCHEMA_VERIFIER_MARKER} ${JSON.stringify({
+ protocol: 1, kind: 'schema-verifier', status: 'unknown', currentVersion: null,
+ minSupported: null, maxSupported: null, reasonCode: 'connection_failed',
+ })}\n`,
+ ) as unknown as Record<string, unknown>
+ assert.equal(unknown.status, 'unknown')
+ assert.equal(unknown.reasonCode, 'connection_failed')
+
+ for (const output of [
+ '',
+ `${SCHEMA_VERIFIER_MARKER} {"protocol":1,"kind":"schema-verifier","status":"compatible","currentVersion":7}\nextra\n`,
+ `${SCHEMA_VERIFIER_MARKER} {"protocol":1,"kind":"schema-verifier","status":"compatible","currentVersion":0,"minSupported":7,"maxSupported":7}\n`,
+ `${SCHEMA_VERIFIER_MARKER} {"protocol":1,"kind":"schema-verifier","status":"unknown"}\n`,
+ `${SCHEMA_VERIFIER_MARKER} {bad-json}\n`,
+ `${SCHEMA_VERIFIER_MARKER} {"protocol":1,"kind":"schema-verifier","status":"compatible","currentVersion":7,"minSupported":7,"maxSupported":7}\n${SCHEMA_VERIFIER_MARKER} {"protocol":1,"kind":"schema-verifier","status":"compatible","currentVersion":7,"minSupported":7,"maxSupported":7}\n`,
+ ]) {
+ assert.equal(parseSchemaVerifierOutput(output).status, 'unknown')
+ }
+})
diff --git a/server/src/__tests__/deployment-release-flow.test.ts b/server/src/__tests__/deployment-release-flow.test.ts
new file mode 100644
index 0000000..d9a4d48
--- /dev/null
+++ b/server/src/__tests__/deployment-release-flow.test.ts
@@ -0,0 +1,841 @@
+import assert from 'node:assert/strict'
+import { createHash } from 'node:crypto'
+import { createServer, type Server } from 'node:http'
+import { chmod, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'
+import { once } from 'node:events'
+import { tmpdir } from 'node:os'
+import { join, dirname, resolve } from 'node:path'
+import { fileURLToPath } from 'node:url'
+import { spawn } from 'node:child_process'
+import { test } from 'node:test'
+
+const testFileDir = dirname(fileURLToPath(import.meta.url))
+const repoRoot = resolve(testFileDir, '../../..')
+const fakeKubectl = resolve(testFileDir, 'fixtures/deployment-fake-kubectl.mjs')
+
+const marker = 'FAKE_SECRET_MARKER'
+const companyId = 'company-flow-fixture'
+const serverOld = `server@sha256:${'1'.repeat(64)}`
+const serverNew = `server@sha256:${'2'.repeat(64)}`
+const agentOld = `agent@sha256:${'4'.repeat(64)}`
+const agentNew = `agent@sha256:${'5'.repeat(64)}`
+
+const startupProbe = {
+ httpGet: { path: '/api/livez', port: 'http' },
+ periodSeconds: 5,
+ timeoutSeconds: 2,
+ failureThreshold: 60,
+}
+const readinessProbe = {
+ httpGet: { path: '/api/health', port: 'http' },
+ periodSeconds: 5,
+ timeoutSeconds: 2,
+}
+const livenessProbe = {
+ httpGet: { path: '/api/livez', port: 'http' },
+ periodSeconds: 30,
+ timeoutSeconds: 3,
+}
+
+function clone<T>(value: T): T {
+ return JSON.parse(JSON.stringify(value)) as T
+}
+
+function baselineDeployment(): Record<string, unknown> {
+ return {
+ apiVersion: 'apps/v1',
+ kind: 'Deployment',
+ metadata: {
+ name: 'cumora-server',
+ namespace: 'default',
+ uid: 'deployment-flow-uid',
+ generation: 12,
+ annotations: { 'fixture.cumora.ai/secret': marker },
+ },
+ spec: {
+ replicas: 2,
+ template: {
+ metadata: {
+ labels: { app: 'cumora-server' },
+ annotations: { 'fixture.cumora.ai/template': marker },
+ },
+ spec: {
+ serviceAccountName: 'cumora-server',
+ automountServiceAccountToken: true,
+ imagePullSecrets: [{ name: 'fixture-pull' }],
+ securityContext: { runAsNonRoot: true, seccompProfile: { type: 'RuntimeDefault' } },
+ volumes: [
+ { name: 'proxy-cert', secret: { secretName: 'cloud-sql-cert' } },
+ { name: 'scratch', emptyDir: {} },
+ ],
+ initContainers: [{ name: 'migrate', image: `migrate@sha256:${'6'.repeat(64)}` }],
+ containers: [
+ {
+ name: 'cloud-sql-proxy',
+ image: `proxy@sha256:${'7'.repeat(64)}`,
+ args: ['--port=5432'],
+ env: [{ name: 'PROXY_MODE', value: 'fixture' }],
+ volumeMounts: [{ name: 'proxy-cert', mountPath: '/var/run/proxy', readOnly: true }],
+ securityContext: { runAsNonRoot: true, allowPrivilegeEscalation: false, capabilities: { drop: ['ALL'] } },
+ startupProbe: { httpGet: { path: '/readiness', port: 9090 }, periodSeconds: 1, failureThreshold: 60 },
+ readinessProbe: { httpGet: { path: '/readiness', port: 9090 } },
+ },
+ {
+ name: 'server',
+ image: serverOld,
+ envFrom: [{ secretRef: { name: 'cumora' } }],
+ env: [{ name: 'CUMORA_AGENT_COMPUTER_IMAGE', value: agentOld }],
+ volumeMounts: [{ name: 'scratch', mountPath: '/tmp/cumora' }],
+ securityContext: { runAsUser: 1000, allowPrivilegeEscalation: false },
+ startupProbe,
+ readinessProbe,
+ livenessProbe,
+ resources: { requests: { cpu: '250m', memory: '512Mi' } },
+ },
+ ],
+ },
+ },
+ },
+ status: {
+ observedGeneration: 12,
+ updatedReplicas: 2,
+ readyReplicas: 2,
+ availableReplicas: 2,
+ },
+ }
+}
+
+type ScenarioConfig = Record<string, unknown> & {
+ smoke?: Record<string, boolean>
+}
+
+function makeState(config: ScenarioConfig = {}, deployment = baselineDeployment()): Record<string, any> {
+ const state = {
+ config: {
+ baselineServerImage: serverOld,
+ secretMarker: marker,
+ emitSecretMarker: true,
+ ...config,
+ },
+ initialDeployment: clone(deployment),
+ deployment: clone(deployment),
+ phase: 'baseline',
+ secrets: {},
+ jobs: {},
+ history: { commands: [], patches: [], rollouts: [], appliedJobs: [] },
+ }
+ if (config.baselineUnhealthy) {
+ const deployment = state.deployment as any
+ deployment.status = {
+ ...deployment.status,
+ observedGeneration: deployment.metadata.generation - 1,
+ updatedReplicas: 0,
+ readyReplicas: 0,
+ availableReplicas: 0,
+ }
+ }
+ return state
+}
+
+function serverContainer(state: Record<string, any>) {
+ return state.deployment.spec.template.spec.containers.find((container: any) => container.name === 'server')
+}
+
+function stateHash(value: unknown): string {
+ return createHash('sha256').update(JSON.stringify(sortKeys(value))).digest('hex')
+}
+
+function sortKeys(value: unknown): unknown {
+ if (Array.isArray(value)) return value.map(sortKeys)
+ if (!value || typeof value !== 'object') return value
+ return Object.fromEntries(Object.entries(value).sort(([left], [right]) => left.localeCompare(right)).map(([key, child]) => [key, sortKeys(child)]))
+}
+
+function decodeSecret(state: Record<string, any>, secretName: string, key: string): Record<string, any> {
+ const encoded = state.secrets?.[secretName]?.data?.[key]
+ assert.equal(typeof encoded, 'string', `${secretName} must contain ${key}`)
+ return JSON.parse(Buffer.from(encoded, 'base64').toString('utf8')) as Record<string, any>
+}
+
+async function startSmokeServer(statePath: string, config: ScenarioConfig): Promise<{ server: Server; base: string }> {
+ let lastSmokePhase: string | null = null
+ let lastSmokeRequest = ''
+ const server = createServer(async (request, response) => {
+ const state = JSON.parse(await readFile(statePath, 'utf8')) as Record<string, any>
+ const phase = String(state.phase || 'baseline')
+ const pathname = (request.url || '/').split('?')[0]
+ const shouldPass = config.smoke?.[phase] !== false
+ const shippingRequired = String(config.shippingRequired ?? 'Y') !== 'N'
+ const terminalPath = shippingRequired ? '/api/shipping/overview' : '/api/conversations'
+ const shipping404Phases = Array.isArray(config.shipping404Phases) ? config.shipping404Phases.map(String) : []
+ const shipping404 = pathname === '/api/shipping/overview' && shipping404Phases.includes(phase)
+ if (shipping404) {
+ response.statusCode = 404
+ response.setHeader('content-type', 'text/plain')
+ response.end(`${marker}: shipping fixture unavailable during ${phase}`)
+ } else if (shouldPass) {
+ response.statusCode = 200
+ response.setHeader('content-type', 'application/json')
+ if (pathname === '/api/health') response.end(JSON.stringify({ ok: true }))
+ else if (pathname === '/api/auth/me') response.end(JSON.stringify({ companies: [{ id: companyId }] }))
+ else if (pathname === '/api/conversations') response.end(JSON.stringify([]))
+ else if (pathname === '/api/shipping/overview') response.end(JSON.stringify({ features: [], friction: [], dueReadbacks: [] }))
+ else response.end(JSON.stringify({ ok: true }))
+ } else {
+ response.statusCode = 503
+ response.setHeader('content-type', 'text/plain')
+ response.end(`${marker}: smoke fixture rejected ${phase}`)
+ }
+ lastSmokePhase = phase
+ lastSmokeRequest = pathname
+ if (config.driftAfterSmoke === phase && pathname === terminalPath && !state.driftAfterSmokeApplied) {
+ state.driftAfterSmokeApplied = true
+ state.deployment.spec.template.metadata.annotations = {
+ ...(state.deployment.spec.template.metadata.annotations || {}),
+ 'fake.cumora.ai/post-smoke-drift': marker,
+ }
+ await writeFile(statePath, `${JSON.stringify(state, null, 2)}\n`, { mode: 0o600 })
+ }
+ })
+ server.listen(0, '127.0.0.1')
+ await once(server, 'listening')
+ const address = server.address()
+ assert.ok(address && typeof address === 'object')
+ assert.equal(lastSmokePhase, null)
+ assert.equal(lastSmokeRequest, '')
+ return { server, base: `http://127.0.0.1:${address.port}` }
+}
+
+interface RunResult {
+ code: number | null
+ signal: NodeJS.Signals | null
+ stdout: string
+ stderr: string
+ state: Record<string, any>
+}
+
+let runCounter = 0
+
+async function runScenario(config: ScenarioConfig = {}, deployment = baselineDeployment()): Promise<RunResult> {
+ await chmod(fakeKubectl, 0o755)
+ const root = await mkdtemp(join(tmpdir(), 'cumora-deployment-release-flow-'))
+ const statePath = join(root, 'fake-kubectl-state.json')
+ const state = makeState(config, deployment)
+ await writeFile(statePath, `${JSON.stringify(state, null, 2)}\n`, { mode: 0o600 })
+ const smoke = await startSmokeServer(statePath, config)
+ const runId = `flow-${process.pid}-${runCounter++}`
+ const env: NodeJS.ProcessEnv = {
+ ...process.env,
+ KUBECTL: fakeKubectl,
+ FAKE_KUBECTL_STATE: statePath,
+ DEPLOYMENT: 'cumora-server',
+ DEPLOYMENT_NAMESPACE: 'default',
+ RECOVERY_SECRET: 'flow-baseline-secret',
+ RECEIPT_SECRET: 'flow-receipt-secret',
+ RECOVERY_WORKDIR: root,
+ RUNNER_TEMP: root,
+ GITHUB_WORKSPACE: repoRoot,
+ GITHUB_RUN_ID: runId,
+ GITHUB_SHA: 'abcdef1234567890',
+ CANDIDATE_SERVER_IMAGE: serverNew,
+ CANDIDATE_AGENT_IMAGE: agentNew,
+ INCLUDE_AGENT: 'Y',
+ MIGRATION_JOB_NAME: `flow-migrate-${runId}`,
+ VERIFIER_JOB_NAME: `flow-verifier-${runId}`,
+ RECOVERY_POLL_MS: '1',
+ MIGRATION_POLL_ATTEMPTS: '1',
+ VERIFIER_POLL_ATTEMPTS: '1',
+ ROLLOUT_TIMEOUT_SECONDS: '1',
+ SMOKE_TIMEOUT_SECONDS: '5',
+ CUMORA_SMOKE_TOKEN: 'dummy-flow-token',
+ CUMORA_SMOKE_COMPANY_ID: companyId,
+ CUMORA_SMOKE_BASE: smoke.base,
+ CUMORA_SMOKE_REQUIRE_SHIPPING: String(config.shippingRequired ?? 'Y'),
+ }
+ if (config.recoveryMode !== undefined) env.RECOVERY_MODE = String(config.recoveryMode)
+ else delete env.RECOVERY_MODE
+ delete env.SMOKE_COMMAND
+
+ let child: ReturnType<typeof spawn> | undefined
+ let result: RunResult
+ try {
+ result = await new Promise<RunResult>((resolveResult) => {
+ child = spawn(process.execPath, ['--import', 'tsx', 'scripts/deploy-release.mjs', 'run'], {
+ cwd: repoRoot,
+ env,
+ stdio: ['ignore', 'pipe', 'pipe'],
+ })
+ let stdout = ''
+ let stderr = ''
+ child.stdout?.setEncoding('utf8')
+ child.stderr?.setEncoding('utf8')
+ child.stdout?.on('data', (chunk: string) => { stdout += chunk })
+ child.stderr?.on('data', (chunk: string) => { stderr += chunk })
+ const timer = setTimeout(() => child?.kill('SIGTERM'), 15_000)
+ child.on('close', async (code, signal) => {
+ clearTimeout(timer)
+ const finalState = JSON.parse(await readFile(statePath, 'utf8')) as Record<string, any>
+ resolveResult({ code, signal, stdout, stderr, state: finalState })
+ })
+ })
+ } finally {
+ if (child && child.exitCode === null) child.kill('SIGTERM')
+ await new Promise<void>((resolveClose) => smoke.server.close(() => resolveClose()))
+ await rm(root, { recursive: true, force: true })
+ }
+ return result
+}
+
+function assertNoSecretLeak(result: Pick<RunResult, 'stdout' | 'stderr'>) {
+ assert.doesNotMatch(result.stdout, new RegExp(marker))
+ assert.doesNotMatch(result.stderr, new RegExp(marker))
+}
+
+test('real deploy-release entrypoint completes candidate and stores API PATCH response receipt', async () => {
+ const result = await runScenario({ defaultCandidatePatchResponse: true, smoke: { baseline: true, candidate: true, restored: true } })
+ assert.equal(result.code, 0, result.stderr)
+ assert.match(result.stdout, /"status":"deployed"/)
+ assertNoSecretLeak(result)
+ assert.equal(result.state.phase, 'candidate')
+ assert.equal(serverContainer(result.state).image, serverNew)
+ assert.equal(serverContainer(result.state).env.find((entry: any) => entry.name === 'CUMORA_AGENT_COMPUTER_IMAGE').value, agentNew)
+ assert.equal(result.state.history.patches.length, 1)
+ assert.equal(result.state.history.rollouts.length, 1)
+ assert.deepEqual(result.state.history.rollouts[0], { phase: 'candidate', setting: 'success' })
+ const receipt = decodeSecret(result.state, 'flow-receipt-secret', 'receipt.json')
+ assert.equal(receipt.serverImage, serverNew)
+ assert.equal(receipt.podTemplate.metadata.annotations['fake.cumora.ai/writer'], 'api-default')
+ assert.equal(receipt.podTemplateHash, stateHash(result.state.deployment.spec.template))
+ const baseline = decodeSecret(result.state, 'flow-baseline-secret', 'baseline.json')
+ assert.equal(baseline.serverImage, serverOld)
+ assert.deepEqual(receipt.podTemplate.spec.volumes, baseline.podTemplate.spec.volumes)
+ assert.deepEqual(receipt.podTemplate.spec.securityContext, baseline.podTemplate.spec.securityContext)
+ assert.deepEqual(receipt.podTemplate.spec.containers.find((container: any) => container.name === 'server').envFrom, baseline.podTemplate.spec.containers.find((container: any) => container.name === 'server').envFrom)
+})
+
+test('candidate smoke failure restores exact baseline and leaves process failed', async () => {
+ const result = await runScenario({ smoke: { baseline: true, candidate: false, restored: true } })
+ assert.notEqual(result.code, 0)
+ assert.match(result.stderr, /candidate_failed_recovered|recovery_failed/)
+ assertNoSecretLeak(result)
+ assert.equal(result.state.phase, 'restored')
+ assert.equal(serverContainer(result.state).image, serverOld)
+ assert.equal(serverContainer(result.state).env.find((entry: any) => entry.name === 'CUMORA_AGENT_COMPUTER_IMAGE').value, agentOld)
+ assert.equal(result.state.history.patches.length, 2)
+ assert.deepEqual(result.state.history.rollouts.map((entry: any) => entry.phase), ['candidate', 'restored'])
+ const baseline = decodeSecret(result.state, 'flow-baseline-secret', 'baseline.json')
+ const restoredTemplateHash = stateHash(result.state.deployment.spec.template)
+ assert.equal(restoredTemplateHash, baseline.podTemplateHash)
+})
+
+test('rollout timeout follows the same guarded recovery path', async () => {
+ const result = await runScenario({ candidateRollout: 'timeout', smoke: { baseline: true, candidate: true, restored: true } })
+ assert.notEqual(result.code, 0)
+ assert.match(result.stderr, /candidate_failed_recovered|recovery_failed/)
+ assertNoSecretLeak(result)
+ assert.equal(result.state.phase, 'restored')
+ assert.equal(serverContainer(result.state).image, serverOld)
+ assert.equal(result.state.history.patches.length, 2)
+})
+
+for (const [name, failureMessage] of [
+ ['kubectl timed-out condition', 'error: timed out waiting for the condition'],
+ ['deployment progress deadline', 'deployment exceeded its progress deadline'],
+] as const) {
+ test(`${name} is a nonzero rollout failure with guarded recovery`, async () => {
+ const result = await runScenario({
+ candidateRollout: 'failure',
+ rolloutFailureMessage: failureMessage,
+ smoke: { baseline: true, candidate: true, restored: true },
+ })
+ assert.notEqual(result.code, 0)
+ assert.match(result.stderr, /candidate_failed_recovered|recovery_failed/)
+ assertNoSecretLeak(result)
+ assert.equal(result.state.history.rollouts[0].failureMessage, failureMessage)
+ assert.equal(result.state.history.patches.length, 2)
+ assert.equal(result.state.phase, 'restored')
+ assert.equal(serverContainer(result.state).image, serverOld)
+ })
+}
+
+const verifierCases: Array<[string, ScenarioConfig, RegExp]> = [
+ ['schema_ahead', { verifierResult: 'schema_ahead' }, /schema_incompatible/],
+ ['incompatible', { verifierResult: 'schema_behind' }, /schema_incompatible/],
+ ['unknown', { verifierResult: 'unknown' }, /schema_unknown/],
+ ['malformed', { verifierResult: 'malformed' }, /schema_unknown/],
+ ['missing', { verifierStatus: 'missing' }, /job_timeout/],
+ ['failed', { verifierStatus: 'failed' }, /job_failed/],
+ ['timedout', { verifierStatus: 'timedout' }, /job_timeout/],
+]
+
+for (const [name, verifierConfig, expectedError] of verifierCases) {
+ test(`verifier ${name} never restores a failed candidate`, async () => {
+ const result = await runScenario({
+ ...verifierConfig,
+ smoke: { baseline: true, candidate: false, restored: true },
+ })
+ assert.notEqual(result.code, 0)
+ assert.match(result.stderr, expectedError)
+ assertNoSecretLeak(result)
+ assert.equal(result.state.history.patches.length, 1)
+ assert.equal(result.state.phase, 'candidate')
+ assert.equal(serverContainer(result.state).image, serverNew)
+ })
+}
+
+test('CAS drift and UID changes prevent automatic recovery', async () => {
+ for (const drift of ['drift-template', 'drift-uid']) {
+ const result = await runScenario({
+ candidateRollout: drift,
+ smoke: { baseline: true, candidate: true, restored: true },
+ })
+ assert.notEqual(result.code, 0, drift)
+ assertNoSecretLeak(result)
+ assert.equal(result.state.history.patches.length, 1, drift)
+ assert.equal(serverContainer(result.state).image, serverNew, drift)
+ }
+})
+
+test('a progress-deadline rollout failure enters the same guarded recovery path', async () => {
+ const result = await runScenario({
+ candidateRollout: 'progressing',
+ // The endpoint can still be served by old Pods while the candidate is
+ // progressing. The rollout failure still enters the full CAS/verifier
+ // recovery path; the release remains nonzero.
+ smoke: { baseline: true, candidate: true, restored: true },
+ })
+ assert.notEqual(result.code, 0)
+ assertNoSecretLeak(result)
+ assert.equal(result.state.history.patches.length, 2)
+ assert.equal(result.state.phase, 'restored')
+ assert.equal(serverContainer(result.state).image, serverOld)
+})
+
+test('baseline and restore tolerate Shipping 404 while candidate smoke keeps Shipping strict', async () => {
+ const deployed = await runScenario({
+ shipping404Phases: ['baseline', 'restored'],
+ smoke: { baseline: true, candidate: true, restored: true },
+ })
+ assert.equal(deployed.code, 0, deployed.stderr)
+ assertNoSecretLeak(deployed)
+ assert.equal(deployed.state.phase, 'candidate')
+ assert.equal(deployed.state.history.patches.length, 1)
+ assert.equal(serverContainer(deployed.state).image, serverNew)
+
+ const recovered = await runScenario({
+ shipping404Phases: ['baseline', 'restored'],
+ smoke: { baseline: true, candidate: false, restored: true },
+ })
+ assert.notEqual(recovered.code, 0)
+ assert.match(recovered.stderr, /candidate_failed_recovered|recovery_failed/)
+ assertNoSecretLeak(recovered)
+ assert.equal(recovered.state.phase, 'restored')
+ assert.equal(recovered.state.history.patches.length, 2)
+ assert.equal(serverContainer(recovered.state).image, serverOld)
+})
+
+test('forward-only can publish with an unhealthy baseline but never restores a failed candidate', async () => {
+ const normal = await runScenario({
+ baselineUnhealthy: true,
+ smoke: { baseline: true, candidate: true, restored: true },
+ })
+ assert.notEqual(normal.code, 0)
+ assertNoSecretLeak(normal)
+ assert.equal(normal.state.history.patches.length, 0)
+
+ const forwardOnlySuccess = await runScenario({
+ baselineUnhealthy: true,
+ recoveryMode: 'forward-only',
+ smoke: { baseline: true, candidate: true, restored: true },
+ })
+ assert.equal(forwardOnlySuccess.code, 0, forwardOnlySuccess.stderr)
+ assertNoSecretLeak(forwardOnlySuccess)
+ assert.equal(forwardOnlySuccess.state.history.patches.length, 1)
+ assert.equal(forwardOnlySuccess.state.phase, 'candidate')
+ assert.equal(serverContainer(forwardOnlySuccess.state).image, serverNew)
+
+ const forwardOnlyFailure = await runScenario({
+ baselineUnhealthy: true,
+ recoveryMode: 'forward-only',
+ smoke: { baseline: true, candidate: false, restored: true },
+ })
+ assert.notEqual(forwardOnlyFailure.code, 0)
+ assert.match(forwardOnlyFailure.stderr, /candidate_failed_forward_only/)
+ assertNoSecretLeak(forwardOnlyFailure)
+ assert.equal(forwardOnlyFailure.state.history.patches.length, 1)
+ assert.equal(forwardOnlyFailure.state.phase, 'candidate')
+ assert.equal(serverContainer(forwardOnlyFailure.state).image, serverNew)
+})
+
+test('mutable baseline is rejected before any migration or Deployment patch', async () => {
+ const deployment = baselineDeployment()
+ ;(deployment.spec as any).template.spec.containers.find((container: any) => container.name === 'server').image = 'server:latest'
+ const result = await runScenario({ smoke: { baseline: true, candidate: true, restored: true } }, deployment)
+ assert.notEqual(result.code, 0)
+ assert.match(result.stderr, /failed|digest|baseline/i)
+ assertNoSecretLeak(result)
+ assert.equal(result.state.history.patches.length, 0)
+ assert.equal(result.state.history.appliedJobs.length, 0)
+})
+
+test('unknown agent baseline is not eligible for automatic restore', async () => {
+ const deployment = baselineDeployment()
+ const server = (deployment.spec as any).template.spec.containers.find((container: any) => container.name === 'server')
+ server.env = [{ name: 'CUMORA_AGENT_COMPUTER_IMAGE', valueFrom: { secretKeyRef: { name: 'agent-image' } } }]
+ const result = await runScenario({ smoke: { baseline: true, candidate: false, restored: true } }, deployment)
+ assert.notEqual(result.code, 0)
+ assertNoSecretLeak(result)
+ assert.equal(result.state.history.patches.length, 1)
+ assert.equal(serverContainer(result.state).image, serverNew)
+})
+
+for (const restoreFailure of ['restorePatchFailure', 'restoreRollout', 'restoreSmoke'] as const) {
+ test(`${restoreFailure} keeps release nonzero and preserves protected baseline`, async () => {
+ const config: ScenarioConfig = {
+ smoke: { baseline: true, candidate: false, restored: restoreFailure !== 'restoreSmoke' },
+ }
+ if (restoreFailure === 'restorePatchFailure') config.restorePatchFailure = 'forbidden restore patch'
+ if (restoreFailure === 'restoreRollout') config.restoreRollout = 'timeout'
+ if (restoreFailure === 'restoreSmoke') config.restoreSmoke = 'fail'
+ const result = await runScenario(config)
+ assert.notEqual(result.code, 0)
+ assertNoSecretLeak(result)
+ const baseline = decodeSecret(result.state, 'flow-baseline-secret', 'baseline.json')
+ assert.equal(baseline.serverImage, serverOld)
+ assert.equal(result.state.history.patches.length, restoreFailure === 'restorePatchFailure' ? 1 : 2)
+ if (restoreFailure === 'restorePatchFailure') assert.equal(serverContainer(result.state).image, serverNew)
+ if (restoreFailure === 'restoreRollout') assert.equal(serverContainer(result.state).image, serverOld)
+ })
+}
+
+test('migration failure creates the baseline record but never patches the Deployment', async () => {
+ const result = await runScenario({ migrationStatus: 'failed', smoke: { baseline: true, candidate: true, restored: true } })
+ assert.notEqual(result.code, 0)
+ assert.match(result.stderr, /job_failed/)
+ assertNoSecretLeak(result)
+ assert.equal(result.state.history.patches.length, 0)
+ assert.equal(result.state.history.appliedJobs.length, 1)
+ const baseline = decodeSecret(result.state, 'flow-baseline-secret', 'baseline.json')
+ assert.equal(baseline.serverImage, serverOld)
+})
+
+test('an active migration Job from another run blocks a new migration writer', async () => {
+ const state = makeState({ smoke: { baseline: true, candidate: true, restored: true } })
+ state.jobs['old-run-migration'] = {
+ apiVersion: 'batch/v1',
+ kind: 'Job',
+ metadata: { name: 'old-run-migration', labels: { app: 'cumora-schema-migration' } },
+ status: { active: 1 },
+ }
+ const result = await runScenarioWithState(state)
+ assert.notEqual(result.code, 0)
+ assert.match(result.stderr, /migration_residual_job_present/)
+ assertNoSecretLeak(result)
+ assert.equal(result.state.history.patches.length, 0)
+ assert.equal(result.state.history.appliedJobs.length, 0)
+})
+
+test('a residual Job with active Pods is blocking even when succeeded is nonzero', async () => {
+ const state = makeState({ smoke: { baseline: true, candidate: true, restored: true } })
+ state.jobs['active-and-succeeded-migration'] = {
+ apiVersion: 'batch/v1',
+ kind: 'Job',
+ metadata: { name: 'active-and-succeeded-migration', labels: { app: 'cumora-schema-recovery' } },
+ status: { active: 1, succeeded: 1, conditions: [{ type: 'Complete', status: 'True' }] },
+ }
+ const result = await runScenarioWithState(state)
+ assert.notEqual(result.code, 0)
+ assert.match(result.stderr, /migration_residual_job_(?:present|active)/)
+ assertNoSecretLeak(result)
+ assert.equal(result.state.history.patches.length, 0)
+ assert.equal(result.state.history.appliedJobs.length, 0)
+})
+
+test('a residual Job is allowed only with terminal Complete condition and active zero', async () => {
+ const state = makeState({ smoke: { baseline: true, candidate: true, restored: true } })
+ state.jobs['completed-migration'] = {
+ apiVersion: 'batch/v1',
+ kind: 'Job',
+ metadata: { name: 'completed-migration', labels: { app: 'cumora-schema-recovery' } },
+ status: { succeeded: 1, conditions: [{ type: 'Complete', status: 'True' }] },
+ }
+ const result = await runScenarioWithState(state)
+ assert.equal(result.code, 0, result.stderr)
+ assertNoSecretLeak(result)
+ assert.equal(result.state.history.patches.length, 1)
+ assert.equal(result.state.phase, 'candidate')
+ assert.equal(serverContainer(result.state).image, serverNew)
+})
+
+test('create-only baseline race with a different protected value is rejected', async () => {
+ const result = await runScenario({
+ secretCreateRaces: { 'flow-baseline-secret': 'different' },
+ smoke: { baseline: true, candidate: true, restored: true },
+ })
+ assert.notEqual(result.code, 0)
+ assert.match(result.stderr, /failed|protected|baseline/i)
+ assertNoSecretLeak(result)
+ assert.equal(result.state.history.patches.length, 0)
+ const raced = decodeSecret(result.state, 'flow-baseline-secret', 'baseline.json')
+ assert.equal(raced.uid, 'raced-secret-uid')
+})
+
+test('receipt create race and Secret API errors fail closed', async () => {
+ const racedReceipt = await runScenario({
+ secretCreateRaces: { 'flow-receipt-secret': 'different' },
+ smoke: { baseline: true, candidate: true, restored: true },
+ })
+ assert.notEqual(racedReceipt.code, 0)
+ assert.match(racedReceipt.stderr, /candidate_receipt_conflict|failed/)
+ assertNoSecretLeak(racedReceipt)
+ assert.equal(racedReceipt.state.history.patches.length, 1)
+
+ const apiFailure = await runScenario({
+ secretReadFailures: { 'flow-baseline-secret': 'API unavailable' },
+ smoke: { baseline: true, candidate: true, restored: true },
+ })
+ assert.notEqual(apiFailure.code, 0)
+ assert.match(apiFailure.stderr, /secret_read_failed/)
+ assertNoSecretLeak(apiFailure)
+ assert.equal(apiFailure.state.history.patches.length, 0)
+})
+
+test('a concurrent writer after the PATCH response cannot become this run receipt', async () => {
+ const result = await runScenario({
+ switchWriterAfterCandidateResponse: true,
+ smoke: { baseline: true, candidate: true, restored: true },
+ })
+ assert.notEqual(result.code, 0)
+ assertNoSecretLeak(result)
+ const receipt = decodeSecret(result.state, 'flow-receipt-secret', 'receipt.json')
+ assert.equal(receipt.serverImage, serverNew)
+ assert.notEqual(receipt.podTemplateHash, stateHash(result.state.deployment.spec.template))
+ assert.equal(serverContainer(result.state).image, serverNew)
+})
+
+test('template drift after a successful candidate smoke cannot report success', async () => {
+ const result = await runScenario({
+ driftAfterSmoke: 'candidate',
+ smoke: { baseline: true, candidate: true, restored: true },
+ })
+ assert.notEqual(result.code, 0)
+ assert.match(result.stderr, /failed|candidate|template/i)
+ assertNoSecretLeak(result)
+ assert.equal(result.state.history.patches.length, 1)
+})
+
+test('template drift after successful restore smoke cannot report recovered success', async () => {
+ const result = await runScenario({
+ driftAfterSmoke: 'restored',
+ smoke: { baseline: true, candidate: false, restored: true },
+ })
+ assert.notEqual(result.code, 0)
+ assert.match(result.stderr, /recovery_failed|restore_receipt_mismatch|failed|template/i)
+ assertNoSecretLeak(result)
+ assert.equal(result.state.history.patches.length, 2)
+ assert.equal(serverContainer(result.state).image, serverOld)
+})
+
+interface EntrypointResult {
+ code: number | null
+ signal: NodeJS.Signals | null
+ stdout: string
+ stderr: string
+}
+
+function captureChild(child: ReturnType<typeof spawn>, timeoutMs = 8_000): Promise<EntrypointResult> {
+ return new Promise((resolveResult) => {
+ let stdout = ''
+ let stderr = ''
+ child.stdout?.setEncoding('utf8')
+ child.stderr?.setEncoding('utf8')
+ child.stdout?.on('data', (chunk: string) => { stdout += chunk })
+ child.stderr?.on('data', (chunk: string) => { stderr += chunk })
+ const timer = setTimeout(() => child.kill('SIGTERM'), timeoutMs)
+ child.on('close', (code, signal) => {
+ clearTimeout(timer)
+ resolveResult({ code, signal, stdout, stderr })
+ })
+ })
+}
+
+function waitMs(milliseconds: number): Promise<void> {
+ return new Promise((resolveWait) => setTimeout(resolveWait, milliseconds))
+}
+
+async function runInterruptedRecoveryScenario(removeReceipt: boolean): Promise<{ run: EntrypointResult; recovery: EntrypointResult; state: Record<string, any> }> {
+ await chmod(fakeKubectl, 0o755)
+ const root = await mkdtemp(join(tmpdir(), 'cumora-deployment-manual-recovery-'))
+ const statePath = join(root, 'fake-kubectl-state.json')
+ const config: ScenarioConfig = {
+ // Stop the real run while its rollout-status child is waiting. This
+ // leaves the immutable baseline and candidate receipt persisted exactly
+ // as an interrupted CI job would.
+ holdCandidateRolloutMs: 1_000,
+ smoke: { baseline: true, candidate: true, restored: true },
+ }
+ await writeFile(statePath, `${JSON.stringify(makeState(config), null, 2)}\n`, { mode: 0o600 })
+ const smoke = await startSmokeServer(statePath, config)
+ const runId = `manual-${process.pid}-${runCounter++}`
+ const env: NodeJS.ProcessEnv = {
+ ...process.env,
+ KUBECTL: fakeKubectl,
+ FAKE_KUBECTL_STATE: statePath,
+ DEPLOYMENT: 'cumora-server',
+ DEPLOYMENT_NAMESPACE: 'default',
+ RECOVERY_SECRET: 'flow-baseline-secret',
+ RECEIPT_SECRET: 'flow-receipt-secret',
+ RECOVERY_WORKDIR: root,
+ RUNNER_TEMP: root,
+ GITHUB_WORKSPACE: repoRoot,
+ GITHUB_RUN_ID: runId,
+ GITHUB_SHA: 'abcdef1234567890',
+ CANDIDATE_SERVER_IMAGE: serverNew,
+ CANDIDATE_AGENT_IMAGE: agentNew,
+ INCLUDE_AGENT: 'Y',
+ MIGRATION_JOB_NAME: `manual-migrate-${runId}`,
+ VERIFIER_JOB_NAME: `manual-verifier-${runId}`,
+ RECOVERY_POLL_MS: '1',
+ MIGRATION_POLL_ATTEMPTS: '1',
+ VERIFIER_POLL_ATTEMPTS: '1',
+ ROLLOUT_TIMEOUT_SECONDS: '10',
+ SMOKE_TIMEOUT_SECONDS: '5',
+ CUMORA_SMOKE_TOKEN: 'dummy-flow-token',
+ CUMORA_SMOKE_COMPANY_ID: companyId,
+ CUMORA_SMOKE_BASE: smoke.base,
+ CUMORA_SMOKE_REQUIRE_SHIPPING: 'Y',
+ }
+ delete env.SMOKE_COMMAND
+ let runChild: ReturnType<typeof spawn> | undefined
+ try {
+ runChild = spawn(process.execPath, ['--import', 'tsx', 'scripts/deploy-release.mjs', 'run'], {
+ cwd: repoRoot,
+ env,
+ stdio: ['ignore', 'pipe', 'pipe'],
+ })
+ const runResultPromise = captureChild(runChild, 8_000)
+ let interruptedState: Record<string, any> | null = null
+ for (let attempt = 0; attempt < 120; attempt += 1) {
+ try {
+ const candidateState = JSON.parse(await readFile(statePath, 'utf8')) as Record<string, any>
+ if (candidateState.history?.patches?.length === 1 && candidateState.phase === 'candidate' && candidateState.secrets?.['flow-receipt-secret']) {
+ interruptedState = candidateState
+ break
+ }
+ } catch {
+ // The fake kubectl writes a complete JSON file between calls; a poll
+ // that catches its brief replacement window simply retries.
+ }
+ await waitMs(10)
+ }
+ assert.ok(interruptedState, 'real run must persist candidate receipt before interruption')
+ runChild.kill('SIGTERM')
+ const run = await runResultPromise
+ // Let the fake rollout child finish its bounded hold before the recover
+ // command starts reading the same state file.
+ await waitMs(1_100)
+ const beforeRecovery = JSON.parse(await readFile(statePath, 'utf8')) as Record<string, any>
+ if (removeReceipt) delete beforeRecovery.secrets['flow-receipt-secret']
+ await writeFile(statePath, `${JSON.stringify(beforeRecovery, null, 2)}\n`, { mode: 0o600 })
+ const recoveryChild = spawn(process.execPath, ['--import', 'tsx', 'scripts/deploy-release.mjs', 'recover'], {
+ cwd: repoRoot,
+ env,
+ stdio: ['ignore', 'pipe', 'pipe'],
+ })
+ const recovery = await captureChild(recoveryChild)
+ const state = JSON.parse(await readFile(statePath, 'utf8')) as Record<string, any>
+ return { run, recovery, state }
+ } finally {
+ if (runChild && runChild.exitCode === null) runChild.kill('SIGTERM')
+ await new Promise<void>((resolveClose) => smoke.server.close(() => resolveClose()))
+ await rm(root, { recursive: true, force: true })
+ }
+}
+
+test('an interrupted run can recover from protected baseline and receipt via the real recover entrypoint', async () => {
+ const result = await runInterruptedRecoveryScenario(false)
+ assert.notEqual(result.run.code, 0, 'the intentionally interrupted run must not report success')
+ assert.equal(result.run.signal, 'SIGTERM')
+ assert.equal(result.recovery.code, 0, result.recovery.stderr)
+ assert.match(result.recovery.stdout, /"status":"recovered"/)
+ assertNoSecretLeak(result.recovery)
+ assert.equal(result.state.phase, 'restored')
+ assert.equal(serverContainer(result.state).image, serverOld)
+ assert.equal(result.state.history.patches.length, 2)
+})
+
+test('manual recover rejects an interrupted run when its protected receipt is missing', async () => {
+ const result = await runInterruptedRecoveryScenario(true)
+ assert.notEqual(result.run.code, 0)
+ assert.equal(result.run.signal, 'SIGTERM')
+ assert.notEqual(result.recovery.code, 0)
+ assert.match(result.recovery.stderr, /protected_receipt_missing/)
+ assertNoSecretLeak(result.recovery)
+ assert.equal(result.state.history.patches.length, 1)
+ assert.equal(result.state.phase, 'candidate')
+ assert.equal(serverContainer(result.state).image, serverNew)
+})
+
+async function runScenarioWithState(initialState: Record<string, any>): Promise<RunResult> {
+ const config = initialState.config as ScenarioConfig
+ const root = await mkdtemp(join(tmpdir(), 'cumora-deployment-release-flow-state-'))
+ const statePath = join(root, 'fake-kubectl-state.json')
+ await writeFile(statePath, `${JSON.stringify(initialState, null, 2)}\n`, { mode: 0o600 })
+ await chmod(fakeKubectl, 0o755)
+ const smoke = await startSmokeServer(statePath, config)
+ const runId = `flow-${process.pid}-${runCounter++}`
+ const env: NodeJS.ProcessEnv = {
+ ...process.env,
+ KUBECTL: fakeKubectl,
+ FAKE_KUBECTL_STATE: statePath,
+ DEPLOYMENT: 'cumora-server',
+ DEPLOYMENT_NAMESPACE: 'default',
+ RECOVERY_SECRET: 'flow-baseline-secret',
+ RECEIPT_SECRET: 'flow-receipt-secret',
+ RECOVERY_WORKDIR: root,
+ RUNNER_TEMP: root,
+ GITHUB_WORKSPACE: repoRoot,
+ GITHUB_RUN_ID: runId,
+ GITHUB_SHA: 'abcdef1234567890',
+ CANDIDATE_SERVER_IMAGE: serverNew,
+ CANDIDATE_AGENT_IMAGE: agentNew,
+ INCLUDE_AGENT: 'Y',
+ MIGRATION_JOB_NAME: `flow-migrate-${runId}`,
+ VERIFIER_JOB_NAME: `flow-verifier-${runId}`,
+ RECOVERY_POLL_MS: '1',
+ MIGRATION_POLL_ATTEMPTS: '1',
+ VERIFIER_POLL_ATTEMPTS: '1',
+ ROLLOUT_TIMEOUT_SECONDS: '1',
+ SMOKE_TIMEOUT_SECONDS: '5',
+ CUMORA_SMOKE_TOKEN: 'dummy-flow-token',
+ CUMORA_SMOKE_COMPANY_ID: companyId,
+ CUMORA_SMOKE_BASE: smoke.base,
+ CUMORA_SMOKE_REQUIRE_SHIPPING: String(config.shippingRequired ?? 'Y'),
+ }
+ if (config.recoveryMode !== undefined) env.RECOVERY_MODE = String(config.recoveryMode)
+ else delete env.RECOVERY_MODE
+ delete env.SMOKE_COMMAND
+ let child: ReturnType<typeof spawn> | undefined
+ try {
+ const result = await new Promise<RunResult>((resolveResult) => {
+ child = spawn(process.execPath, ['--import', 'tsx', 'scripts/deploy-release.mjs', 'run'], { cwd: repoRoot, env, stdio: ['ignore', 'pipe', 'pipe'] })
+ let stdout = ''
+ let stderr = ''
+ child.stdout?.setEncoding('utf8')
+ child.stderr?.setEncoding('utf8')
+ child.stdout?.on('data', (chunk: string) => { stdout += chunk })
+ child.stderr?.on('data', (chunk: string) => { stderr += chunk })
+ const timer = setTimeout(() => child?.kill('SIGTERM'), 15_000)
+ child.on('close', async (code, signal) => {
+ clearTimeout(timer)
+ const finalState = JSON.parse(await readFile(statePath, 'utf8')) as Record<string, any>
+ resolveResult({ code, signal, stdout, stderr, state: finalState })
+ })
+ })
+ return result
+ } finally {
+ if (child && child.exitCode === null) child.kill('SIGTERM')
+ await new Promise<void>((resolveClose) => smoke.server.close(() => resolveClose()))
+ await rm(root, { recursive: true, force: true })
+ }
+}
diff --git a/server/src/__tests__/fixtures/deployment-fake-kubectl.mjs b/server/src/__tests__/fixtures/deployment-fake-kubectl.mjs
new file mode 100755
index 0000000..359f163
--- /dev/null
+++ b/server/src/__tests__/fixtures/deployment-fake-kubectl.mjs
@@ -0,0 +1,366 @@
+#!/usr/bin/env node
+
+/*
+ * A deliberately small kubectl-shaped process for deploy-release flow tests.
+ *
+ * The test owns the JSON state file. This process only implements the reads
+ * and create/apply/patch operations used by scripts/deploy-release.mjs; it
+ * never talks to a Kubernetes API. In particular, patching validates all
+ * JSON Patch test operations before replacing anything, which models the
+ * atomic CAS operation the production workflow relies on.
+ */
+
+import { readFile, writeFile } from 'node:fs/promises'
+
+const statePath = process.env.FAKE_KUBECTL_STATE
+if (!statePath) {
+ process.stderr.write('FAKE_KUBECTL_STATE is required\n')
+ process.exit(2)
+}
+
+const originalArgs = process.argv.slice(2)
+const args = [...originalArgs]
+if (args[0] === '-n') args.splice(0, 2)
+const command = args.shift()
+
+function clone(value) {
+ return value === undefined ? undefined : JSON.parse(JSON.stringify(value))
+}
+
+function sorted(value) {
+ if (Array.isArray(value)) return value.map(sorted)
+ if (value && typeof value === 'object') {
+ return Object.fromEntries(Object.entries(value).sort(([a], [b]) => a.localeCompare(b)).map(([key, child]) => [key, sorted(child)]))
+ }
+ return value
+}
+
+function equal(left, right) {
+ return JSON.stringify(sorted(left)) === JSON.stringify(sorted(right))
+}
+
+function pointerSegments(path) {
+ if (path === '') return []
+ if (!path.startsWith('/')) throw new Error(`invalid JSON pointer ${path}`)
+ return path.slice(1).split('/').map((part) => part.replaceAll('~1', '/').replaceAll('~0', '~'))
+}
+
+function readPointer(root, path) {
+ let value = root
+ for (const segment of pointerSegments(path)) value = value?.[segment]
+ return value
+}
+
+async function loadState() {
+ return JSON.parse(await readFile(statePath, 'utf8'))
+}
+
+async function saveState(state) {
+ await writeFile(statePath, `${JSON.stringify(state, null, 2)}\n`, { mode: 0o600 })
+}
+
+function marker(state) {
+ // `logs job/...` is intentionally consumed by the strict schema-verifier
+ // parser, so keep that stream protocol-clean. Every other kubectl call
+ // may carry the marker: deploy-release captures child stderr and must not
+ // echo it in its own result.
+ if (command !== 'logs' && state.config?.emitSecretMarker !== false) {
+ process.stderr.write(`${state.config?.secretMarker || 'FAKE_SECRET_MARKER'}\n`)
+ }
+}
+
+function fail(message, code = 1) {
+ process.stderr.write(`${message}\n`)
+ process.exitCode = code
+}
+
+function optionValue(list, name) {
+ const index = list.indexOf(name)
+ return index >= 0 ? list[index + 1] : undefined
+}
+
+function hasOption(list, name) {
+ return list.includes(name)
+}
+
+async function stdinText() {
+ const chunks = []
+ for await (const chunk of process.stdin) chunks.push(chunk)
+ return Buffer.concat(chunks).toString('utf8')
+}
+
+function emit(value) {
+ if (value !== undefined && value !== null) process.stdout.write(`${JSON.stringify(value)}\n`)
+}
+
+function deploymentContainer(deployment, name) {
+ return deployment?.spec?.template?.spec?.containers?.find((container) => container?.name === name)
+}
+
+function serverImage(deployment) {
+ return deploymentContainer(deployment, 'server')?.image
+}
+
+function bumpHealth(deployment) {
+ const generation = Number(deployment.metadata?.generation || 1) + 1
+ deployment.metadata = { ...deployment.metadata, generation }
+ const replicas = Number(deployment.spec?.replicas || 1)
+ deployment.status = {
+ ...(deployment.status || {}),
+ observedGeneration: generation,
+ updatedReplicas: replicas,
+ readyReplicas: replicas,
+ availableReplicas: replicas,
+ }
+}
+
+function mutateTemplate(template, kind) {
+ const next = clone(template)
+ next.metadata ??= {}
+ next.metadata.annotations = { ...(next.metadata.annotations || {}), 'fake.cumora.ai/writer': kind }
+ return next
+}
+
+function applyPatch(deployment, operations) {
+ const candidate = clone(deployment)
+ for (const operation of operations) {
+ if (operation?.op !== 'test') continue
+ if (!equal(readPointer(candidate, operation.path), operation.value)) {
+ const error = new Error(`Conflict: JSON Patch test failed at ${operation.path}`)
+ error.code = 'conflict'
+ throw error
+ }
+ }
+ for (const operation of operations) {
+ if (operation?.op !== 'replace') continue
+ if (!['/spec/template', '/metadata/uid'].includes(operation.path)) {
+ throw new Error(`unsupported replace path ${operation.path}`)
+ }
+ if (operation.path === '/metadata/uid') candidate.metadata.uid = clone(operation.value)
+ else candidate.spec.template = clone(operation.value)
+ }
+ return candidate
+}
+
+function decodeSecret(secret, key) {
+ const encoded = secret?.data?.[key]
+ if (typeof encoded !== 'string') return null
+ try {
+ return JSON.parse(Buffer.from(encoded, 'base64').toString('utf8'))
+ } catch {
+ return null
+ }
+}
+
+function encodeJson(value) {
+ return Buffer.from(`${JSON.stringify(value)}\n`, 'utf8').toString('base64')
+}
+
+function raceSecret(manifest, mode) {
+ const raced = clone(manifest)
+ if (mode === 'different') {
+ const key = Object.keys(raced.data || {})[0]
+ const decoded = decodeSecret(raced, key)
+ if (decoded && typeof decoded === 'object') {
+ if ('uid' in decoded) decoded.uid = 'raced-secret-uid'
+ else decoded.podTemplateHash = 'raced-secret-hash'
+ raced.data[key] = encodeJson(decoded)
+ }
+ }
+ return raced
+}
+
+function jobStatus(state, job) {
+ const isVerifier = job?.metadata?.labels?.['recovery.cumora.ai/read-only'] === 'true'
+ const setting = isVerifier ? state.config?.verifierStatus : state.config?.migrationStatus
+ if (setting === 'failed') return { failed: 1, conditions: [{ type: 'Failed', status: 'True' }] }
+ if (setting === 'pending' || setting === 'timedout') return { active: 1 }
+ return { succeeded: 1, conditions: [{ type: 'Complete', status: 'True' }] }
+}
+
+function verifierOutput(state) {
+ if (typeof state.config?.verifierLog === 'string') return state.config.verifierLog
+ const markerName = 'CUMORA_SCHEMA_VERIFY_RESULT'
+ const status = state.config?.verifierResult || 'compatible'
+ if (status === 'compatible') {
+ return `${markerName} ${JSON.stringify({ protocol: 1, kind: 'schema-verifier', status: 'compatible', currentVersion: 7, minSupported: 7, maxSupported: 7 })}\n`
+ }
+ if (status === 'unknown') {
+ return `${markerName} ${JSON.stringify({ protocol: 1, kind: 'schema-verifier', status: 'unknown', reasonCode: 'connection_failed', currentVersion: null, minSupported: null, maxSupported: null })}\n`
+ }
+ if (status === 'malformed') return `${markerName} {bad-json}\n`
+ return `${markerName} ${JSON.stringify({ protocol: 1, kind: 'schema-verifier', status: 'incompatible', code: status, currentVersion: null, minSupported: null, maxSupported: null })}\n`
+}
+
+async function commandGet(state) {
+ const resource = args.shift()
+ const name = resource === 'jobs' ? undefined : args.shift()
+ if (resource === 'deployment') {
+ if (state.config?.deploymentReadFailure) return fail(`Error from server (Forbidden): ${state.config.deploymentReadFailure}`)
+ return emit(clone(state.deployment))
+ }
+ if (resource === 'secret') {
+ if (state.config?.secretReadFailures?.[name]) return fail(`Error from server (Forbidden): ${state.config.secretReadFailures[name]}`)
+ const secret = state.secrets?.[name]
+ if (!secret) {
+ if (hasOption(args, '--ignore-not-found')) return emit(undefined)
+ return fail(`Error from server (NotFound): secrets "${name}" not found`)
+ }
+ return emit(clone(secret))
+ }
+ if (resource === 'job') {
+ if (state.config?.jobReadFailures?.[name]) return fail(`Error from server (Forbidden): ${state.config.jobReadFailures[name]}`)
+ const job = state.jobs?.[name]
+ if (!job) {
+ if (hasOption(args, '--ignore-not-found')) return emit(undefined)
+ return fail(`Error from server (NotFound): jobs "${name}" not found`)
+ }
+ return emit(clone(job))
+ }
+ if (resource === 'jobs') {
+ const selector = optionValue(args, '-l')
+ const expectedApp = selector?.split('=')[1]
+ const items = Object.values(state.jobs || {}).filter((job) => !expectedApp || job?.metadata?.labels?.app === expectedApp)
+ if (state.config?.residualJobsReadFailure) return fail('Error from server (Forbidden): cannot list jobs')
+ return emit({ apiVersion: 'batch/v1', kind: 'JobList', items: clone(items) })
+ }
+ return fail(`unsupported get resource ${resource}`)
+}
+
+async function commandCreate(state) {
+ const manifest = JSON.parse(await stdinText())
+ const name = manifest?.metadata?.name
+ if (manifest?.kind !== 'Secret' || !name) return fail('fake kubectl only creates named Secrets')
+ const configuredRace = state.config?.secretCreateRaces?.[name]
+ const raceSeen = state.raceSeen?.[name]
+ if (configuredRace && !raceSeen) {
+ state.raceSeen ??= {}
+ state.raceSeen[name] = true
+ if (configuredRace === 'api-failure') {
+ await saveState(state)
+ return fail('Error from server (Forbidden): create Secret denied')
+ }
+ state.secrets ??= {}
+ state.secrets[name] = raceSecret(manifest, configuredRace)
+ await saveState(state)
+ return fail(`Error from server (AlreadyExists): secrets "${name}" already exists`)
+ }
+ if (state.secrets?.[name]) return fail(`Error from server (AlreadyExists): secrets "${name}" already exists`)
+ state.secrets ??= {}
+ state.secrets[name] = manifest
+ await saveState(state)
+}
+
+async function commandApply(state) {
+ const manifest = JSON.parse(await stdinText())
+ const name = manifest?.metadata?.name
+ if (manifest?.kind !== 'Job' || !name) return fail('fake kubectl only applies Jobs')
+ if (state.config?.applyJobFailure) return fail(`Error from server: ${state.config.applyJobFailure}`)
+ if (state.config?.verifierStatus === 'missing' && manifest.metadata?.labels?.['recovery.cumora.ai/read-only'] === 'true') {
+ await saveState(state)
+ return
+ }
+ state.jobs ??= {}
+ state.jobs[name] = { ...manifest, status: jobStatus(state, manifest) }
+ state.history.appliedJobs.push(name)
+ await saveState(state)
+}
+
+async function commandPatch(state) {
+ const name = args.shift()
+ const patchFile = optionValue(args, '--patch-file')
+ let raw
+ if (patchFile) raw = await readFile(patchFile, 'utf8')
+ else {
+ const inline = optionValue(args, '--patch')
+ raw = inline ?? await stdinText()
+ }
+ const operations = JSON.parse(raw)
+ let next
+ try {
+ next = applyPatch(state.deployment, operations)
+ } catch (error) {
+ return fail(error instanceof Error ? error.message : String(error))
+ }
+ const baselineImage = state.config?.baselineServerImage
+ || state.initialDeployment?.spec?.template?.spec?.containers?.find((container) => container?.name === 'server')?.image
+ const restoring = serverImage(next) === baselineImage
+ if (restoring && state.config?.restorePatchFailure) return fail(`Error from server: ${state.config.restorePatchFailure}`)
+ if (!restoring && state.config?.candidatePatchFailure) return fail(`Error from server: ${state.config.candidatePatchFailure}`)
+
+ bumpHealth(next)
+ if (!restoring && state.config?.defaultCandidatePatchResponse) {
+ next.spec.template = mutateTemplate(next.spec.template, 'api-default')
+ }
+ const response = clone(next)
+ state.deployment = next
+ state.phase = restoring ? 'restored' : 'candidate'
+ state.history.patches.push({ name, operations: clone(operations), response: clone(response), phase: state.phase })
+ if (restoring && state.config?.restoreSmoke === 'fail') state.phase = 'restored'
+ await saveState(state)
+ emit(response)
+
+ if (!restoring && state.config?.switchWriterAfterCandidateResponse) {
+ const writer = clone(state.deployment)
+ writer.spec.template = mutateTemplate(writer.spec.template, 'other-writer')
+ state.deployment = writer
+ state.phase = 'other-writer'
+ await saveState(state)
+ }
+}
+
+async function commandRollout(state) {
+ const subcommand = args.shift()
+ const resource = args.shift()
+ if (subcommand !== 'status') return fail(`unsupported rollout subcommand ${subcommand}`)
+ if (!resource?.startsWith('deployment/')) return fail('unsupported rollout target')
+ const baselineImage = state.config?.baselineServerImage
+ || state.initialDeployment?.spec?.template?.spec?.containers?.find((container) => container?.name === 'server')?.image
+ const restoring = serverImage(state.deployment) === baselineImage
+ const setting = restoring ? state.config?.restoreRollout : state.config?.candidateRollout
+ const failureMessage = restoring ? state.config?.restoreRolloutFailureMessage : state.config?.rolloutFailureMessage
+ state.history.rollouts.push({
+ phase: restoring ? 'restored' : 'candidate',
+ setting: setting || 'success',
+ ...(typeof failureMessage === 'string' ? { failureMessage } : {}),
+ })
+ if (!restoring && Number(state.config?.holdCandidateRolloutMs || 0) > 0) {
+ await new Promise((resolve) => setTimeout(resolve, Number(state.config.holdCandidateRolloutMs)))
+ }
+ if (!restoring && setting === 'drift-template') state.deployment.spec.template = mutateTemplate(state.deployment.spec.template, 'drift')
+ if (!restoring && setting === 'drift-uid') state.deployment.metadata.uid = 'writer-changed-uid'
+ if (!restoring && setting === 'progressing') {
+ state.deployment.status = { ...(state.deployment.status || {}), observedGeneration: Number(state.deployment.metadata.generation) - 1, updatedReplicas: 0, readyReplicas: 0, availableReplicas: 0 }
+ }
+ await saveState(state)
+ if (setting && setting !== 'success') return fail(failureMessage || `Error from server: rollout ${setting}`)
+}
+
+async function commandLogs(state) {
+ const name = args.shift()?.replace(/^job\//, '')
+ if (!state.jobs?.[name]) return fail(`Error from server (NotFound): jobs "${name}" not found`)
+ emitRaw(verifierOutput(state))
+}
+
+function emitRaw(value) {
+ process.stdout.write(value)
+}
+
+async function main() {
+ const state = await loadState()
+ state.secrets ??= {}
+ state.jobs ??= {}
+ state.history ??= { commands: [], patches: [], rollouts: [], appliedJobs: [] }
+ state.history.commands.push({ args: originalArgs, at: new Date().toISOString() })
+ await saveState(state)
+ marker(state)
+ if (command === 'get') await commandGet(state)
+ else if (command === 'create') await commandCreate(state)
+ else if (command === 'apply') await commandApply(state)
+ else if (command === 'patch') await commandPatch(state)
+ else if (command === 'rollout') await commandRollout(state)
+ else if (command === 'logs') await commandLogs(state)
+ else fail(`unsupported kubectl command ${command}`)
+}
+
+await main()
diff --git a/server/src/__tests__/migration-ownership.test.ts b/server/src/__tests__/migration-ownership.test.ts
index 7136e96..d27c0ca 100644
--- a/server/src/__tests__/migration-ownership.test.ts
+++ b/server/src/__tests__/migration-ownership.test.ts
@@ -25,13 +25,12 @@ test('application Pod manifests contain no per-replica migration container', asy
}
})
-test('production deploy migrates before one atomic Deployment mutation', async () => {
+test('production deploy delegates one guarded transaction to the recovery runner', async () => {
const workflow = await readRepo('.github/workflows/deploy.yml')
- const migrationAt = workflow.indexOf('- name: Run candidate migrations once')
- const patchAt = workflow.indexOf('- name: Patch deployment')
- assert.ok(migrationAt >= 0, 'deploy must create a single candidate migration Job')
- assert.ok(patchAt > migrationAt, 'migration must complete before Deployment mutation')
- assert.match(workflow, /kind:\s*"Job"/)
- assert.match(workflow, /Candidate database migration did not complete; deployment was not mutated/)
- assert.match(workflow, /\{ name: "migrate", "\$patch": "delete" \}/)
+ assert.match(workflow, /scripts\/deploy-release\.mjs run/)
+ assert.match(workflow, /CANDIDATE_SERVER_IMAGE/)
+ assert.match(workflow, /MIGRATION_REPAIR/)
+ assert.match(workflow, /RECOVERY_WORKDIR/)
+ assert.match(workflow, /concurrency:/)
+ assert.doesNotMatch(workflow, /kubectl rollout undo/)
})
diff --git a/server/src/__tests__/startup-probes.test.ts b/server/src/__tests__/startup-probes.test.ts
index 6dc98e9..95f466a 100644
--- a/server/src/__tests__/startup-probes.test.ts
+++ b/server/src/__tests__/startup-probes.test.ts
@@ -1,19 +1,21 @@
import assert from 'node:assert/strict'
-import { execFile as execFileCallback } from 'node:child_process'
import { readFile } from 'node:fs/promises'
import { fileURLToPath } from 'node:url'
import { dirname, resolve } from 'node:path'
import { test } from 'node:test'
-import { promisify } from 'node:util'
-import { load, loadAll } from 'js-yaml'
+import { loadAll } from 'js-yaml'
import { MigrationHistoryError } from '../db/migrations/manifest.js'
+import {
+ DEPLOYMENT_PROBE_CONTRACT,
+ buildCandidatePatch,
+ extractDeploymentSnapshot,
+} from '../deploy/recovery.js'
process.env.CUMORA_RUNTIME_CLIENT = 'http'
process.env.OPENAI_API_KEY ??= 'test-key'
const { verifySchemaWithBootRetry } = await import('../db/schema-version.js')
const { pool } = await import('../db/pool.js')
-const execFile = promisify(execFileCallback)
const repoRoot = resolve(dirname(fileURLToPath(import.meta.url)), '../../..')
const readRepo = (path: string): Promise<string> => readFile(resolve(repoRoot, path), 'utf8')
@@ -37,24 +39,14 @@ interface Deployment {
spec?: { template?: { spec?: { containers?: Container[] } } }
}
-interface WorkflowStep {
- name?: string
- run?: string
-}
-
-interface Workflow {
- jobs?: Record<string, { steps?: WorkflowStep[] }>
-}
-
-interface ProbePatch {
- spec?: { template?: { spec?: { containers?: Container[] } } }
-}
-
const manifestPaths = [
'server/k8s/cumora-server.gke.yaml',
'server/k8s/cumora-server.orbstack.yaml',
]
+const PROBE_FIXTURE_SERVER_IMAGE = `server@sha256:${'a'.repeat(64)}`
+const PROBE_FIXTURE_AGENT_IMAGE = `agent@sha256:${'b'.repeat(64)}`
+
async function serverContainer(path: string): Promise<Container> {
const docs = loadAll(await readRepo(path)) as unknown[]
const deployment = docs
@@ -89,41 +81,46 @@ test('both server manifests keep startup, liveness, and readiness probes on the
}
})
-test('the deploy patch reapplies the same probe contract and rollout deadline', async (t) => {
- const workflow = load(await readRepo('.github/workflows/deploy.yml')) as Workflow
- const patchStep = workflow.jobs?.deploy?.steps?.find((step) => step.name === 'Patch deployment')
- assert.ok(patchStep?.run, 'deploy must have a parsed Patch deployment step')
- const patch = patchStep.run
- assert.match(patch, /startupProbe:\s*\{\s*httpGet:\s*\{\s*path: "\/api\/livez",\s*port: "http"\s*\}/)
- assert.match(patch, /readinessProbe:\s*\{\s*httpGet:\s*\{\s*path: "\/api\/health",\s*port: "http"\s*\}/)
- assert.match(patch, /livenessProbe:\s*\{\s*httpGet:\s*\{\s*path: "\/api\/livez",\s*port: "http"\s*\}/)
- assert.match(patch, /periodSeconds: 5/)
- assert.match(patch, /timeoutSeconds: 2/)
- assert.match(patch, /failureThreshold: 60/)
- const rolloutStep = workflow.jobs?.deploy?.steps?.find((step) => step.name === 'Wait for rollout')
- assert.match(rolloutStep?.run ?? '', /kubectl rollout status deployment\/cumora-server --timeout=10m/)
-
- const filterOpen = patch.indexOf("'\n")
- const filterClose = patch.indexOf("\n')", filterOpen + 2)
- assert.ok(filterOpen >= 0 && filterClose > filterOpen, 'Patch deployment must contain a jq filter')
- const jqFilter = patch.slice(filterOpen + 2, filterClose)
- let jq: { stdout: string }
- try {
- jq = await execFile(
- 'jq',
- ['-nc', '--arg', 'server', 'fixture-server', '--arg', 'agent', 'fixture-agent', jqFilter],
- { encoding: 'utf8' },
- ) as { stdout: string }
- } catch (error) {
- if (!process.env.CI && (error as { code?: unknown })?.code === 'ENOENT') {
- t.skip('jq is unavailable locally; CI must provide jq for this workflow patch test')
- return
- }
- throw error
+test('the recovery helper reapplies the same probe contract and workflow calls it', async () => {
+ const workflowText = await readRepo('.github/workflows/deploy.yml')
+ assert.match(workflowText, /scripts\/deploy-release\.mjs run/)
+ assert.match(workflowText, /CANDIDATE_SERVER_IMAGE/)
+ assert.match(workflowText, /ROLLOUT_TIMEOUT_SECONDS/)
+ assert.match(workflowText, /VERIFIER_POLL_ATTEMPTS:\s*'75'/)
+ assert.doesNotMatch(workflowText, /kubectl rollout undo/)
+
+ const baseline = extractDeploymentSnapshot({
+ apiVersion: 'apps/v1',
+ kind: 'Deployment',
+ metadata: { name: 'cumora-server', uid: 'probe-fixture' },
+ spec: {
+ template: {
+ metadata: { labels: { app: 'cumora-server' } },
+ spec: {
+ containers: [{
+ name: 'server',
+ image: PROBE_FIXTURE_SERVER_IMAGE,
+ env: [{ name: 'CUMORA_AGENT_COMPUTER_IMAGE', value: PROBE_FIXTURE_AGENT_IMAGE }],
+ }],
+ },
+ },
+ },
+ })
+ const patch = buildCandidatePatch(baseline, {
+ server: `server@sha256:${'c'.repeat(64)}`,
+ agent: `agent@sha256:${'d'.repeat(64)}`,
+ })
+ const replacement = patch.find((operation) => {
+ const candidate = operation as any
+ return candidate?.op === 'replace' && candidate?.path === '/spec/template'
+ }) as any
+ assert.ok(replacement && typeof replacement.value === 'object' && replacement.value !== null)
+ const patchedServer = (replacement.value as { spec: { containers: Container[] } }).spec.containers
+ .find((container) => container.name === 'server')
+ assert.ok(patchedServer, 'recovery helper patch must target the server container')
+ for (const [name, expected] of Object.entries(DEPLOYMENT_PROBE_CONTRACT)) {
+ assert.deepEqual(patchedServer[name as keyof Container], expected, `${name} helper contract`)
}
- const patchObject = JSON.parse(String(jq.stdout)) as ProbePatch
- const patchedServer = patchObject.spec?.template?.spec?.containers?.find((container) => container.name === 'server')
- assert.ok(patchedServer, 'evaluated deploy patch must target the server container')
for (const path of manifestPaths) {
const manifestServer = await serverContainer(path)
assert.deepEqual(patchedServer.startupProbe, manifestServer.startupProbe, `${path} startup contract`)
diff --git a/server/src/deploy/recovery.ts b/server/src/deploy/recovery.ts
new file mode 100644
index 0000000..b12acc5
--- /dev/null
+++ b/server/src/deploy/recovery.ts
@@ -0,0 +1,804 @@
+import { createHash } from 'node:crypto'
+
+/**
+ * Deployment recovery is deliberately a data-only module. The workflow owns
+ * kubectl, while this module owns the objects passed to kubectl and all of the
+ * compare-and-swap rules. Keeping those rules here lets tests exercise the
+ * exact patch and Job objects used in production without contacting a cluster.
+ */
+
+export type JsonPrimitive = string | number | boolean | null
+export type JsonValue = unknown
+export type JsonObject = Record<string, unknown>
+
+export interface PodTemplateSpec {
+ containers: JsonObject[]
+ initContainers?: JsonObject[]
+ serviceAccountName?: string
+ imagePullSecrets?: JsonValue
+ securityContext?: JsonValue
+ volumes?: JsonValue
+ [key: string]: unknown
+}
+
+export interface PodTemplate {
+ metadata: JsonObject
+ spec: PodTemplateSpec
+ [key: string]: unknown
+}
+
+export interface JsonPatchOperation {
+ op: 'test' | 'replace'
+ path: string
+ value: JsonValue
+}
+
+export interface DeploymentHealth {
+ generation: number | null
+ observedGeneration: number | null
+ desiredReplicas: number | null
+ updatedReplicas: number | null
+ readyReplicas: number | null
+ availableReplicas: number | null
+}
+
+export interface DeploymentSnapshot {
+ apiVersion: string
+ kind: 'Deployment'
+ name: string
+ namespace: string
+ uid: string
+ podTemplate: PodTemplate
+ podTemplateHash: string
+ serverImage: string
+ agentImage: string | null
+ agentImageSource: 'missing' | 'inline' | 'unknown'
+ baselineImage: { server: string; agent: string | null }
+ health: DeploymentHealth
+ baselineSmokePassed?: boolean
+ baselineVerifiedAt?: string
+ capturedAt: string
+}
+
+export interface DeploymentReceipt {
+ uid: string
+ name: string
+ namespace: string
+ podTemplate: PodTemplate
+ podTemplateHash: string
+ serverImage: string
+ agentImage: string | null
+ agentImageSource: 'missing' | 'inline' | 'unknown'
+ baselineImage: { server: string; agent: string | null }
+ observedAt: string
+}
+
+export interface CandidateImages {
+ server: string
+ /** Omit to preserve the baseline agent image. */
+ agent?: string
+}
+
+export interface MigrationJobOptions {
+ name: string
+ image: string
+ repairMode: 'off' | 'archive-detach'
+ activeDeadlineSeconds?: number
+ ttlSecondsAfterFinished?: number
+}
+
+export interface SchemaVerifierJobOptions {
+ name: string
+ image: string
+ activeDeadlineSeconds?: number
+ ttlSecondsAfterFinished?: number
+}
+
+export interface SchemaVerifierResult {
+ [key: string]: unknown
+ protocol: 1
+ kind: 'schema-verifier'
+ status: 'compatible' | 'incompatible' | 'unknown'
+ currentVersion: number | null
+ minSupported: number | null
+ maxSupported: number | null
+ code?: 'schema_uninitialized' | 'schema_behind' | 'schema_ahead' | 'migration_history_invalid'
+ reasonCode?: string
+}
+
+export const SCHEMA_VERIFIER_MARKER = 'CUMORA_SCHEMA_VERIFY_RESULT'
+
+export const SCHEMA_INCOMPATIBLE_CODES = [
+ 'schema_uninitialized',
+ 'schema_behind',
+ 'schema_ahead',
+ 'migration_history_invalid',
+] as const
+
+export const SCHEMA_UNKNOWN_REASON_CODES = [
+ 'module_unavailable',
+ 'connection_failed',
+ 'timeout',
+ 'cleanup_failed',
+] as const
+
+const DIGEST_IMAGE_PATTERN = /@sha256:[a-f0-9]{64}$/
+
+export const DEPLOYMENT_PROBE_CONTRACT = {
+ startupProbe: {
+ httpGet: { path: '/api/livez', port: 'http' },
+ periodSeconds: 5,
+ timeoutSeconds: 2,
+ failureThreshold: 60,
+ },
+ readinessProbe: {
+ httpGet: { path: '/api/health', port: 'http' },
+ periodSeconds: 5,
+ timeoutSeconds: 2,
+ },
+ livenessProbe: {
+ httpGet: { path: '/api/livez', port: 'http' },
+ periodSeconds: 30,
+ timeoutSeconds: 3,
+ },
+} as const
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+ return typeof value === 'object' && value !== null && !Array.isArray(value)
+}
+
+function isJsonValue(value: unknown): value is JsonValue {
+ if (value === null || typeof value === 'string' || typeof value === 'number' || typeof value === 'boolean') {
+ return true
+ }
+ if (Array.isArray(value)) return value.every(isJsonValue)
+ return isRecord(value) && Object.values(value).every(isJsonValue)
+}
+
+function asJsonObject(value: unknown, label: string): JsonObject {
+ if (!isRecord(value) || !isJsonValue(value)) throw new Error(`${label} must be a JSON object`)
+ return value as JsonObject
+}
+
+function cloneJson<T>(value: T): T {
+ return JSON.parse(JSON.stringify(value)) as T
+}
+
+/** Sort object keys recursively while preserving array order. */
+export function normalizeJson(value: unknown): unknown {
+ if (Array.isArray(value)) return value.map(normalizeJson)
+ if (!isRecord(value)) return value
+ return Object.fromEntries(
+ Object.entries(value)
+ .sort(([left], [right]) => left.localeCompare(right))
+ .map(([key, child]) => [key, normalizeJson(child)]),
+ ) as JsonObject
+}
+
+export function hashJson(value: unknown): string {
+ return createHash('sha256').update(JSON.stringify(normalizeJson(value))).digest('hex')
+}
+
+export function hashPodTemplate(template: JsonObject | PodTemplate): string {
+ return hashJson(template)
+}
+
+function readString(value: unknown, label: string): string {
+ if (typeof value !== 'string' || value.length === 0) throw new Error(`${label} must be a non-empty string`)
+ return value
+}
+
+function integerOrNull(value: unknown): number | null {
+ return typeof value === 'number' && Number.isInteger(value) ? value : null
+}
+
+function assertDigestImage(value: unknown, label: string): string {
+ const image = readString(value, label)
+ if (!DIGEST_IMAGE_PATTERN.test(image)) throw new Error(`${label} must be pinned by sha256 digest`)
+ return image
+}
+
+function getObjectPath(root: unknown, path: string[], label: string): Record<string, unknown> {
+ let current: unknown = root
+ for (const segment of path) {
+ if (!isRecord(current) || !(segment in current)) throw new Error(`${label} is missing ${path.join('.')}`)
+ current = current[segment]
+ }
+ if (!isRecord(current)) throw new Error(`${label} ${path.join('.')} must be an object`)
+ return current
+}
+
+function getArrayPath(root: unknown, path: string[], label: string): unknown[] {
+ let current: unknown = root
+ for (const segment of path) {
+ if (!isRecord(current) || !(segment in current)) throw new Error(`${label} is missing ${path.join('.')}`)
+ current = current[segment]
+ }
+ if (!Array.isArray(current)) throw new Error(`${label} ${path.join('.')} must be an array`)
+ return current
+}
+
+function findNamedObject(items: unknown[], name: string, label: string): Record<string, unknown> {
+ const found = items.find((item) => isRecord(item) && item.name === name)
+ if (!isRecord(found)) throw new Error(`${label} is missing ${name}`)
+ return found
+}
+
+function agentImageValue(container: Record<string, unknown>, name: string): { value: string | null; source: 'missing' | 'inline' | 'unknown' } {
+ const env = container.env
+ if (!Array.isArray(env)) return { value: null, source: 'missing' }
+ const entry = env.find((item) => isRecord(item) && item.name === name)
+ if (!isRecord(entry)) return { value: null, source: 'missing' }
+ if (typeof entry.value !== 'string') return { value: null, source: 'unknown' }
+ return { value: entry.value, source: 'inline' }
+}
+
+function deploymentPodTemplate(deployment: unknown): PodTemplate {
+ const template = asJsonObject(getObjectPath(deployment, ['spec', 'template'], 'Deployment'), 'Deployment pod template')
+ if (!isRecord(template.metadata) || !isRecord(template.spec)) throw new Error('Deployment pod template must contain metadata and spec')
+ if (!Array.isArray(template.spec.containers)) throw new Error('Deployment pod template spec.containers must be an array')
+ return template as PodTemplate
+}
+
+function deploymentServerContainer(template: JsonObject): JsonObject {
+ const containers = getArrayPath(template, ['spec', 'containers'], 'Deployment pod template')
+ return asJsonObject(findNamedObject(containers, 'server', 'Deployment pod template containers'), 'server container')
+}
+
+export function extractDeploymentSnapshot(
+ deployment: unknown,
+ opts: { expectedName?: string; expectedNamespace?: string; capturedAt?: string } = {},
+): DeploymentSnapshot {
+ const metadata = getObjectPath(deployment, ['metadata'], 'Deployment')
+ const template = deploymentPodTemplate(deployment)
+ const server = deploymentServerContainer(template)
+ const uid = readString(metadata.uid, 'Deployment metadata.uid')
+ const apiVersion = readString((deployment as Record<string, unknown>).apiVersion, 'Deployment apiVersion')
+ if (apiVersion !== 'apps/v1') throw new Error(`unsupported Deployment apiVersion: ${apiVersion}`)
+ const kind = readString((deployment as Record<string, unknown>).kind, 'Deployment kind')
+ if (kind !== 'Deployment') throw new Error(`expected Deployment, got ${kind}`)
+ const name = readString(metadata.name, 'Deployment metadata.name')
+ // kubectl includes metadata.namespace. Fixtures and an unqualified local
+ // object can omit it; Kubernetes resolves those to the default namespace.
+ const namespace = typeof metadata.namespace === 'string' && metadata.namespace.length > 0 ? metadata.namespace : 'default'
+ if (opts.expectedName && name !== opts.expectedName) throw new Error(`unexpected Deployment name: ${name}`)
+ if (opts.expectedNamespace && namespace !== opts.expectedNamespace) throw new Error(`unexpected Deployment namespace: ${namespace}`)
+ const serverImage = assertDigestImage(server.image, 'server container image')
+ const agent = agentImageValue(server, 'CUMORA_AGENT_COMPUTER_IMAGE')
+ if (agent.value !== null) assertDigestImage(agent.value, 'agent container image')
+ return {
+ apiVersion,
+ kind: 'Deployment',
+ name,
+ namespace,
+ uid,
+ podTemplate: cloneJson(template),
+ podTemplateHash: hashPodTemplate(template),
+ serverImage,
+ agentImage: agent.value,
+ agentImageSource: agent.source,
+ baselineImage: { server: serverImage, agent: agent.value },
+ health: {
+ generation: integerOrNull(metadata.generation),
+ observedGeneration: integerOrNull(isRecord((deployment as Record<string, unknown>).status) ? (deployment as Record<string, Record<string, unknown>>).status.observedGeneration : null),
+ desiredReplicas: integerOrNull(isRecord((deployment as Record<string, unknown>).spec) ? (deployment as Record<string, Record<string, unknown>>).spec.replicas : null),
+ updatedReplicas: integerOrNull(isRecord((deployment as Record<string, unknown>).status) ? (deployment as Record<string, Record<string, unknown>>).status.updatedReplicas : null),
+ readyReplicas: integerOrNull(isRecord((deployment as Record<string, unknown>).status) ? (deployment as Record<string, Record<string, unknown>>).status.readyReplicas : null),
+ availableReplicas: integerOrNull(isRecord((deployment as Record<string, unknown>).status) ? (deployment as Record<string, Record<string, unknown>>).status.availableReplicas : null),
+ },
+ capturedAt: opts.capturedAt ?? new Date().toISOString(),
+ }
+}
+
+export function deploymentReceiptFromObject(
+ deployment: unknown,
+ opts: { observedAt?: string } = {},
+): DeploymentReceipt {
+ const snapshot = extractDeploymentSnapshot(deployment, { capturedAt: opts.observedAt })
+ return {
+ uid: snapshot.uid,
+ name: snapshot.name,
+ namespace: snapshot.namespace,
+ podTemplate: snapshot.podTemplate,
+ podTemplateHash: snapshot.podTemplateHash,
+ serverImage: snapshot.serverImage,
+ agentImage: snapshot.agentImage,
+ agentImageSource: snapshot.agentImageSource,
+ baselineImage: snapshot.baselineImage,
+ observedAt: snapshot.capturedAt,
+ }
+}
+
+function assertSnapshotShape(snapshot: DeploymentSnapshot): void {
+ if (snapshot.kind !== 'Deployment') throw new Error('recovery baseline must describe a Deployment')
+ if (snapshot.apiVersion !== 'apps/v1') throw new Error('recovery baseline must use apps/v1')
+ readString(snapshot.uid, 'recovery baseline uid')
+ readString(snapshot.name, 'recovery baseline name')
+ readString(snapshot.namespace, 'recovery baseline namespace')
+ if (!isRecord(snapshot.podTemplate)) throw new Error('recovery baseline podTemplate must be an object')
+ const actualHash = hashPodTemplate(snapshot.podTemplate)
+ if (actualHash !== snapshot.podTemplateHash) {
+ throw new Error('recovery baseline podTemplateHash does not match podTemplate')
+ }
+ assertDigestImage(snapshot.serverImage, 'recovery baseline serverImage')
+ if (snapshot.agentImage !== null && typeof snapshot.agentImage !== 'string') {
+ throw new Error('recovery baseline agentImage must be a string or null')
+ }
+ if (snapshot.agentImage !== null) assertDigestImage(snapshot.agentImage, 'recovery baseline agentImage')
+ if (!['missing', 'inline', 'unknown'].includes(snapshot.agentImageSource)) throw new Error('invalid recovery baseline agent image source')
+ if (snapshot.baselineImage?.server !== snapshot.serverImage || snapshot.baselineImage?.agent !== snapshot.agentImage) {
+ throw new Error('recovery baseline baselineImage does not match image fields')
+ }
+ const server = deploymentServerContainer(snapshot.podTemplate)
+ if (server.image !== snapshot.serverImage) throw new Error('recovery baseline server image does not match podTemplate')
+ const agent = agentImageValue(server, 'CUMORA_AGENT_COMPUTER_IMAGE')
+ if (agent.value !== snapshot.agentImage || agent.source !== snapshot.agentImageSource) {
+ throw new Error('recovery baseline agent image does not match podTemplate')
+ }
+}
+
+export function assertBaselineUnchanged(existing: DeploymentSnapshot, candidate: DeploymentSnapshot | DeploymentReceipt): void {
+ assertSnapshotShape(existing)
+ if ('apiVersion' in candidate) assertSnapshotShape(candidate)
+ else assertReceiptShape(candidate)
+ if (existing.uid !== candidate.uid || existing.podTemplateHash !== candidate.podTemplateHash) {
+ throw new Error('recovery baseline already exists and differs; refusing to replace it')
+ }
+}
+
+export function assertDeploymentHealthy(snapshot: DeploymentSnapshot): void {
+ const health = snapshot.health
+ if (!health || health.generation === null || health.observedGeneration !== health.generation) {
+ throw new Error('Deployment baseline rollout is not observed')
+ }
+ if (
+ health.desiredReplicas === null ||
+ health.updatedReplicas !== health.desiredReplicas ||
+ health.readyReplicas !== health.desiredReplicas ||
+ health.availableReplicas !== health.desiredReplicas
+ ) {
+ throw new Error('Deployment baseline rollout replicas are not healthy')
+ }
+}
+
+export function assertRecoveryEligible(baseline: DeploymentSnapshot): void {
+ assertDeploymentHealthy(baseline)
+ if (baseline.baselineSmokePassed !== true) {
+ throw new Error('Deployment baseline smoke was not proven before migration')
+ }
+}
+
+function patchContainer(
+ template: PodTemplate,
+ images: CandidateImages,
+): PodTemplate {
+ assertDigestImage(images.server, 'candidate server image')
+ if (images.agent !== undefined) assertDigestImage(images.agent, 'candidate agent image')
+ const patched = cloneJson(template)
+ const spec = patched.spec
+ const containers = spec.containers
+ const server = asJsonObject(findNamedObject(containers, 'server', 'pod template containers'), 'server container')
+ server.image = images.server
+ server.startupProbe = cloneJson(DEPLOYMENT_PROBE_CONTRACT.startupProbe as unknown as JsonObject)
+ server.readinessProbe = cloneJson(DEPLOYMENT_PROBE_CONTRACT.readinessProbe as unknown as JsonObject)
+ server.livenessProbe = cloneJson(DEPLOYMENT_PROBE_CONTRACT.livenessProbe as unknown as JsonObject)
+
+ if (images.agent !== undefined) {
+ const env = Array.isArray(server.env) ? server.env.filter((entry) => !(isRecord(entry) && entry.name === 'CUMORA_AGENT_COMPUTER_IMAGE')) : []
+ env.push({ name: 'CUMORA_AGENT_COMPUTER_IMAGE', value: images.agent })
+ server.env = env as JsonValue
+ }
+
+ if (Array.isArray(spec.initContainers)) {
+ spec.initContainers = spec.initContainers.filter((entry) => !(isRecord(entry) && entry.name === 'migrate')) as JsonObject[]
+ }
+ return patched as PodTemplate
+}
+
+export function buildCandidateTemplate(baseline: DeploymentSnapshot, images: CandidateImages): PodTemplate {
+ assertSnapshotShape(baseline)
+ return patchContainer(baseline.podTemplate, images)
+}
+
+function jsonPatchTest(path: string, value: JsonValue): JsonPatchOperation {
+ return { op: 'test', path, value: cloneJson(value) }
+}
+
+function jsonPatchReplace(path: string, value: JsonValue): JsonPatchOperation {
+ return { op: 'replace', path, value: cloneJson(value) }
+}
+
+export function buildCandidatePatch(
+ baseline: DeploymentSnapshot,
+ images: CandidateImages,
+): JsonPatchOperation[] {
+ assertSnapshotShape(baseline)
+ const candidate = buildCandidateTemplate(baseline, images)
+ return [
+ jsonPatchTest('/metadata/uid', baseline.uid),
+ jsonPatchTest('/spec/template', baseline.podTemplate),
+ jsonPatchReplace('/spec/template', candidate),
+ ]
+}
+
+export function buildRestorePatch(
+ baseline: DeploymentSnapshot,
+ current: DeploymentReceipt,
+ expectedCandidateHash: string,
+): JsonPatchOperation[] {
+ assertSnapshotShape(baseline)
+ assertReceiptShape(current)
+ if (baseline.agentImageSource === 'unknown' || current.agentImageSource === 'unknown') {
+ throw new Error('agent image valueFrom is not a safe automatic recovery baseline')
+ }
+ if (current.uid !== baseline.uid) throw new Error('Deployment UID changed; refusing automatic recovery')
+ if (current.podTemplateHash !== expectedCandidateHash) {
+ throw new Error('Deployment pod template drifted from the candidate receipt; refusing automatic recovery')
+ }
+ return [
+ jsonPatchTest('/metadata/uid', baseline.uid),
+ jsonPatchTest('/spec/template', current.podTemplate),
+ jsonPatchReplace('/spec/template', baseline.podTemplate),
+ ]
+}
+
+function assertReceiptShape(receipt: DeploymentReceipt): void {
+ readString(receipt.uid, 'candidate receipt uid')
+ readString(receipt.name, 'candidate receipt name')
+ readString(receipt.namespace, 'candidate receipt namespace')
+ if (!isRecord(receipt.podTemplate)) throw new Error('candidate receipt podTemplate must be an object')
+ if (hashPodTemplate(receipt.podTemplate) !== receipt.podTemplateHash) {
+ throw new Error('candidate receipt podTemplateHash does not match podTemplate')
+ }
+ assertDigestImage(receipt.serverImage, 'candidate receipt serverImage')
+ if (receipt.agentImage !== null) assertDigestImage(receipt.agentImage, 'candidate receipt agentImage')
+ if (!['missing', 'inline', 'unknown'].includes(receipt.agentImageSource)) throw new Error('invalid candidate receipt agent image source')
+ if (receipt.baselineImage?.server !== receipt.serverImage || receipt.baselineImage?.agent !== receipt.agentImage) {
+ throw new Error('candidate receipt baselineImage does not match image fields')
+ }
+ const server = deploymentServerContainer(receipt.podTemplate)
+ if (server.image !== receipt.serverImage) throw new Error('candidate receipt server image does not match podTemplate')
+ const agent = agentImageValue(server, 'CUMORA_AGENT_COMPUTER_IMAGE')
+ if (agent.value !== receipt.agentImage || agent.source !== receipt.agentImageSource) {
+ throw new Error('candidate receipt agent image does not match podTemplate')
+ }
+}
+
+export function assertReceiptShapeForRecovery(receipt: DeploymentReceipt): void {
+ assertReceiptShape(receipt)
+}
+
+export function assertRecoveryStateBinding(
+ baseline: DeploymentSnapshot,
+ receipt: DeploymentReceipt,
+): void {
+ assertSnapshotShape(baseline)
+ assertReceiptShape(receipt)
+ if (baseline.uid !== receipt.uid || baseline.name !== receipt.name || baseline.namespace !== receipt.namespace) {
+ throw new Error('protected recovery receipt is bound to a different Deployment')
+ }
+}
+
+function cleanContainerForJob(container: JsonObject): JsonObject {
+ const result = cloneJson(container)
+ for (const field of [
+ 'ports',
+ 'startupProbe',
+ 'readinessProbe',
+ 'livenessProbe',
+ 'lifecycle',
+ 'terminationMessagePath',
+ 'terminationMessagePolicy',
+ ]) delete result[field]
+ return result
+}
+
+function podSpecForJob(template: JsonObject): { pod: JsonObject; server: JsonObject; proxy: JsonObject; proxyStartupProbe: JsonValue } {
+ const spec = asJsonObject(getObjectPath(template, ['spec'], 'pod template'), 'pod template spec')
+ const allInit = Array.isArray(spec.initContainers) ? spec.initContainers : []
+ const allContainers = getArrayPath(spec, ['containers'], 'pod template')
+ const proxySource = [...allInit, ...allContainers].find((item) => isRecord(item) && item.name === 'cloud-sql-proxy')
+ if (!isRecord(proxySource)) throw new Error('Deployment pod template has no cloud-sql-proxy container')
+ const server = findNamedObject(allContainers, 'server', 'pod template containers')
+ const pod = cloneJson(spec)
+ delete pod.containers
+ delete pod.initContainers
+ delete pod.ephemeralContainers
+ const proxyStartupProbe = isJsonValue(proxySource.startupProbe)
+ ? cloneJson(proxySource.startupProbe)
+ : {
+ httpGet: { path: '/readiness', port: 9090 },
+ periodSeconds: 1,
+ failureThreshold: 60,
+ }
+ return { pod, server: cleanContainerForJob(server), proxy: cleanContainerForJob(proxySource), proxyStartupProbe }
+}
+
+function upsertEnv(container: JsonObject, name: string, value: string): void {
+ const env = Array.isArray(container.env) ? container.env.filter((entry) => !(isRecord(entry) && entry.name === name)) : []
+ env.push({ name, value })
+ container.env = env as JsonValue
+}
+
+function jobBase(
+ template: JsonObject,
+ options: { name: string; image: string; command: string[]; args?: string[] },
+): { metadata: JsonObject; spec: JsonObject; pod: JsonObject; server: JsonObject; proxy: JsonObject } {
+ const { pod, server, proxy, proxyStartupProbe } = podSpecForJob(template)
+ server.name = 'migrate'
+ server.image = options.image
+ server.imagePullPolicy = 'IfNotPresent'
+ server.command = options.command as unknown as JsonValue
+ delete server.args
+ if (options.args) server.args = options.args as unknown as JsonValue
+ server.workingDir = '/app'
+ pod.restartPolicy = 'Never'
+ pod.initContainers = [
+ {
+ ...proxy,
+ name: 'cloud-sql-proxy',
+ restartPolicy: 'Always',
+ startupProbe: proxyStartupProbe,
+ },
+ ] as unknown as JsonValue
+ pod.containers = [server] as unknown as JsonValue
+ return {
+ metadata: {
+ name: options.name,
+ labels: { app: 'cumora-schema-recovery' },
+ },
+ spec: {
+ backoffLimit: 0,
+ activeDeadlineSeconds: 600,
+ ttlSecondsAfterFinished: 600,
+ template: {
+ metadata: { labels: { app: 'cumora-schema-recovery', 'recovery.cumora.ai/managed': 'true' } },
+ spec: pod,
+ },
+ },
+ pod,
+ server,
+ proxy,
+ }
+}
+
+export function buildMigrationJob(
+ baseline: DeploymentSnapshot,
+ options: MigrationJobOptions,
+): JsonObject {
+ assertSnapshotShape(baseline)
+ assertDigestImage(options.image, 'migration image')
+ if (!['off', 'archive-detach'].includes(options.repairMode)) throw new Error('invalid migration repair mode')
+ const result = jobBase(baseline.podTemplate, {
+ name: options.name,
+ image: options.image,
+ command: ['npm', 'run', 'migrate'],
+ })
+ result.spec.activeDeadlineSeconds = options.activeDeadlineSeconds ?? 600
+ result.spec.ttlSecondsAfterFinished = options.ttlSecondsAfterFinished ?? 600
+ upsertEnv(result.server, 'MIGRATION_0002_REPAIR', options.repairMode)
+ return {
+ apiVersion: 'batch/v1',
+ kind: 'Job',
+ metadata: result.metadata,
+ spec: result.spec,
+ }
+}
+
+/**
+ * This is injected into the old server image during recovery. It imports the
+ * old image's own pool, schema gate, and manifest, so a newer checkout cannot
+ * accidentally decide that an old image is compatible. The program writes a
+ * single marker line and nothing else; the parser below fails closed on any
+ * extra or malformed output.
+ */
+export function buildSchemaVerifierScript(): string {
+ const marker = JSON.stringify(SCHEMA_VERIFIER_MARKER)
+ return `
+const marker = ${marker};
+let result = { status: "unknown", reasonCode: "module_unavailable", currentVersion: null, minSupported: null, maxSupported: null };
+let client;
+let pool;
+let MigrationHistoryError;
+let manifest;
+let began = false;
+let committed = false;
+try {
+ const [poolModule, schemaModule, manifestModule] = await Promise.all([
+ import("./server/src/db/pool.ts"),
+ import("./server/src/db/schema-version.ts"),
+ import("./server/src/db/migrations/manifest.ts"),
+ ]);
+ pool = poolModule.pool;
+ const verifySchemaCompatibility = schemaModule.verifySchemaCompatibility;
+ MigrationHistoryError = manifestModule.MigrationHistoryError;
+ manifest = manifestModule;
+ if (!pool || typeof pool.connect !== "function" || typeof verifySchemaCompatibility !== "function" || typeof MigrationHistoryError !== "function") throw new Error("verifier modules unavailable");
+ if (!Number.isInteger(manifest.MIN_SUPPORTED_SCHEMA_VERSION) || !Number.isInteger(manifest.MAX_SUPPORTED_SCHEMA_VERSION)) throw new Error("verifier manifest unavailable");
+ client = await pool.connect();
+ result.minSupported = manifest.MIN_SUPPORTED_SCHEMA_VERSION;
+ result.maxSupported = manifest.MAX_SUPPORTED_SCHEMA_VERSION;
+ await client.query("BEGIN READ ONLY");
+ began = true;
+ await client.query("SET LOCAL statement_timeout = '30000ms'");
+ const version = await verifySchemaCompatibility(client);
+ if (!Number.isInteger(version) || version < result.minSupported || version > result.maxSupported) throw new Error("invalid schema version");
+ await client.query("COMMIT");
+ committed = true;
+ result = { status: "compatible", currentVersion: version, minSupported: result.minSupported, maxSupported: result.maxSupported };
+} catch (error) {
+ result = typeof MigrationHistoryError === "function" && error instanceof MigrationHistoryError
+ ? { status: "incompatible", code: ["schema_uninitialized", "schema_behind", "schema_ahead", "migration_history_invalid"].includes(error.code) ? error.code : "migration_history_invalid", currentVersion: null, minSupported: result.minSupported, maxSupported: result.maxSupported }
+ : { status: "unknown", reasonCode: /timeout|deadline/i.test(String(error)) ? "timeout" : (pool ? "connection_failed" : "module_unavailable"), currentVersion: null, minSupported: result.minSupported, maxSupported: result.maxSupported };
+ if (began && !committed && client) {
+ try { await client.query("ROLLBACK"); } catch {}
+ }
+} finally {
+ try { if (client) client.release(); } catch { result = { status: "unknown", reasonCode: "cleanup_failed", currentVersion: null, minSupported: result.minSupported, maxSupported: result.maxSupported }; }
+ try { if (pool) await pool.end(); } catch { result = { status: "unknown", reasonCode: "cleanup_failed", currentVersion: null, minSupported: result.minSupported, maxSupported: result.maxSupported }; }
+}
+process.stdout.write(marker + " " + JSON.stringify({ protocol: 1, kind: "schema-verifier", ...result }) + "\\n");
+`
+}
+
+export function buildSchemaVerifierJob(
+ baseline: DeploymentSnapshot,
+ options: SchemaVerifierJobOptions,
+): JsonObject {
+ assertSnapshotShape(baseline)
+ if (options.image !== baseline.serverImage) {
+ throw new Error('schema verifier image must be the captured baseline server image')
+ }
+ const result = jobBase(baseline.podTemplate, {
+ name: options.name,
+ image: options.image,
+ command: ['node', '--import', 'tsx', '--input-type=module', '-e'],
+ args: [buildSchemaVerifierScript()],
+ })
+ // Cloud SQL proxy startup (up to 60s), image scheduling/pull, the bounded
+ // five-second pool connection, and the 30s read-only query all need room
+ // before the Job is considered a verifier timeout.
+ result.spec.activeDeadlineSeconds = options.activeDeadlineSeconds ?? 180
+ result.spec.ttlSecondsAfterFinished = options.ttlSecondsAfterFinished ?? 600
+ result.metadata.labels = {
+ app: 'cumora-schema-recovery',
+ 'recovery.cumora.ai/managed': 'true',
+ 'recovery.cumora.ai/read-only': 'true',
+ }
+ return {
+ apiVersion: 'batch/v1',
+ kind: 'Job',
+ metadata: result.metadata,
+ spec: result.spec,
+ }
+}
+
+function parseMarkerValue(value: unknown): SchemaVerifierResult | null {
+ if (!isRecord(value)) return null
+ if (value.protocol !== 1 || value.kind !== 'schema-verifier') return null
+ const currentVersion = value.currentVersion === undefined ? null : value.currentVersion
+ const minSupported = value.minSupported === undefined ? null : value.minSupported
+ const maxSupported = value.maxSupported === undefined ? null : value.maxSupported
+ if (currentVersion !== null && !Number.isInteger(currentVersion)) return null
+ if (minSupported !== null && !Number.isInteger(minSupported)) return null
+ if (maxSupported !== null && !Number.isInteger(maxSupported)) return null
+ if (value.status === 'compatible') {
+ if (
+ !Number.isInteger(currentVersion) ||
+ !Number.isInteger(minSupported) ||
+ !Number.isInteger(maxSupported) ||
+ (currentVersion as number) < (minSupported as number) ||
+ (currentVersion as number) > (maxSupported as number)
+ ) return null
+ return {
+ protocol: 1,
+ kind: 'schema-verifier',
+ status: 'compatible',
+ currentVersion: currentVersion as number,
+ minSupported: minSupported as number,
+ maxSupported: maxSupported as number,
+ }
+ }
+ if (value.status === 'incompatible' && typeof value.code === 'string' && SCHEMA_INCOMPATIBLE_CODES.includes(value.code as typeof SCHEMA_INCOMPATIBLE_CODES[number])) {
+ return {
+ protocol: 1,
+ kind: 'schema-verifier',
+ status: 'incompatible',
+ currentVersion: Number.isInteger(currentVersion) ? currentVersion as number : null,
+ minSupported: Number.isInteger(minSupported) ? minSupported as number : null,
+ maxSupported: Number.isInteger(maxSupported) ? maxSupported as number : null,
+ code: value.code as SchemaVerifierResult['code'],
+ }
+ }
+ if (value.status === 'unknown' && typeof value.reasonCode === 'string' && SCHEMA_UNKNOWN_REASON_CODES.includes(value.reasonCode as typeof SCHEMA_UNKNOWN_REASON_CODES[number])) {
+ return {
+ protocol: 1,
+ kind: 'schema-verifier',
+ status: 'unknown',
+ currentVersion: Number.isInteger(currentVersion) ? currentVersion as number : null,
+ minSupported: Number.isInteger(minSupported) ? minSupported as number : null,
+ maxSupported: Number.isInteger(maxSupported) ? maxSupported as number : null,
+ reasonCode: value.reasonCode as string,
+ }
+ }
+ return null
+}
+
+export function parseSchemaVerifierOutput(stdout: string, stderr = ''): SchemaVerifierResult {
+ const combined = `${stdout}${stderr.length > 0 ? `\n${stderr}` : ''}`
+ const lines = combined.split(/\r?\n/).map((line) => line.trim()).filter((line) => line.length > 0)
+ const markerLines = lines.filter((line) => line.startsWith(SCHEMA_VERIFIER_MARKER))
+ if (markerLines.length !== 1) return unknownVerifierResult('bad_output')
+ const markerLine = markerLines[0]
+ if (!markerLine.startsWith(`${SCHEMA_VERIFIER_MARKER} `)) return unknownVerifierResult('bad_output')
+ const payload = markerLine.slice(SCHEMA_VERIFIER_MARKER.length + 1)
+ const nonMarkerLines = combined
+ .split(/\r?\n/)
+ .map((line) => line.trim())
+ .filter((line) => line.length > 0 && !line.startsWith(`${SCHEMA_VERIFIER_MARKER} `))
+ if (nonMarkerLines.length > 0) return unknownVerifierResult('bad_output')
+ try {
+ const value = JSON.parse(payload) as unknown
+ return parseMarkerValue(value) ?? unknownVerifierResult('protocol_error')
+ } catch {
+ return unknownVerifierResult('bad_output')
+ }
+}
+
+function unknownVerifierResult(reasonCode: SchemaVerifierResult['reasonCode']): SchemaVerifierResult {
+ return {
+ protocol: 1,
+ kind: 'schema-verifier',
+ status: 'unknown',
+ currentVersion: null,
+ minSupported: null,
+ maxSupported: null,
+ reasonCode,
+ }
+}
+
+export function assertCompatibleVerifierResult(result: SchemaVerifierResult): asserts result is SchemaVerifierResult & { status: 'compatible'; currentVersion: number; minSupported: number; maxSupported: number } {
+ if (
+ result.protocol !== 1 ||
+ result.kind !== 'schema-verifier' ||
+ result.status !== 'compatible' ||
+ !Number.isInteger(result.currentVersion) ||
+ !Number.isInteger(result.minSupported) ||
+ !Number.isInteger(result.maxSupported) ||
+ (result.currentVersion as number) < (result.minSupported as number) ||
+ (result.currentVersion as number) > (result.maxSupported as number)
+ ) {
+ throw new Error(`old image schema verifier did not prove compatibility (${result.status})`)
+ }
+}
+
+export function assertDeploymentReceiptMatches(
+ baseline: DeploymentSnapshot,
+ current: DeploymentReceipt,
+ expectedCandidateHash: string,
+): void {
+ assertSnapshotShape(baseline)
+ assertReceiptShape(current)
+ if (current.uid !== baseline.uid) throw new Error('Deployment UID changed since baseline capture')
+ if (current.podTemplateHash !== expectedCandidateHash) {
+ throw new Error('Deployment template changed since candidate patch')
+ }
+}
+
+export function assertCandidateReceiptMatches(
+ baseline: DeploymentSnapshot,
+ receipt: DeploymentReceipt,
+ images: CandidateImages,
+): void {
+ assertSnapshotShape(baseline)
+ assertReceiptShape(receipt)
+ if (receipt.uid !== baseline.uid || receipt.name !== baseline.name || receipt.namespace !== baseline.namespace) {
+ throw new Error('candidate receipt Deployment identity changed')
+ }
+ if (receipt.serverImage !== images.server) throw new Error('candidate receipt server image does not match selected digest')
+ const expectedAgent = images.agent ?? baseline.agentImage
+ if (receipt.agentImage !== expectedAgent) throw new Error('candidate receipt agent image does not match selected digest')
+ if (images.agent === undefined && receipt.agentImageSource !== baseline.agentImageSource) {
+ throw new Error('candidate receipt agent image source changed')
+ }
+}