Where agent teams gather. Cross-platform team chat where AI agents are first-class teammates — with cloud or bring-your-own (Claude Code / Codex) brains.
TypeScript
fix(deploy): guard schema-compatible deployment recovery
OKNOTOK3597f8ecb769
diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.ymlindex 759ee26..989e280 100644--- a/.github/workflows/deploy.yml+++ b/.github/workflows/deploy.yml@@ -1,11 +1,7 @@-# Roll the cumora-server Deployment to a specific image tag.-#-# Production has one ignition path: an explicit workflow_dispatch. Build and-# deploy are intentionally separate so green CI produces a candidate, not a-# production mutation. GitHub's protected `production` environment supplies-# the independent approval step; this workflow then pins the selected tag to-# a digest, captures the previous revision, rolls out, runs authenticated-# smoke, and automatically rolls back if the evidence fails.+# Roll cumora-server to an immutable image digest with guarded recovery.+# The executable orchestration lives in scripts/deploy-release.mjs so the same+# capture, migration, CAS, timeout, smoke, and recovery path can be tested with+# a fake kubectl. A recovered candidate still leaves this workflow failed.name: Deploy@@ -13,7 +9,7 @@ on:workflow_dispatch:inputs:tag:- description: 'Image tag in AR (e.g. d1389e2, latest). Resolved to digest before set-image.'+ description: 'Image tag in AR (e.g. d1389e2, latest). Resolved to digest before touching GKE.'required: falsedefault: 'latest'include_agent:@@ -21,49 +17,72 @@ on:required: falsedefault: 'Y'repair_0002:- description: >-- Let migration 0002 repair its own precondition (off | archive-detach).- archive-detach copies every conversations.members id that has no- participant in that conversation's tenant into- conversation_members_detached_0002 (with its ordinal, so it is- reversible) and removes it from the array. Messages are untouched.- Runs inside 0002's transaction. Leave off unless you have read the- precheck report from a failed run.+ description: 'Migration 0002 repair mode (off | archive-detach)'required: falsedefault: 'off'type: choiceoptions: ['off', 'archive-detach']+ recovery_mode:+ description: 'Baseline mode (normal requires known-good baseline; forward-only never auto-restores)'+ required: false+ default: 'normal'+ type: choice+ options: ['normal', 'forward-only']permissions:contents: readid-token: write+concurrency:+ group: cumora-production-deploy+ cancel-in-progress: false+env:GCP_PROJECT: cumoraAR_REGION: us-west2AR_REPO: cumoraCLUSTER: cumora-prod-zCLUSTER_LOCATION: us-west2-a+ DEPLOYMENT: cumora-serverjobs:deploy:name: Roll cumora-serverruns-on: ubuntu-latest- timeout-minutes: 30+ # The candidate rollout, bounded old-image verifier, restore rollout, and+ # two authenticated smokes fit inside this job budget. A cancellation+ # cannot be relied on to run cleanup, so recovery state remains protected.+ timeout-minutes: 45environment:name: productionurl: https://api.cumora.ai+ env:+ RECOVERY_SECRET: cumora-deploy-recovery-${{ github.run_id }}+ RECEIPT_SECRET: cumora-deploy-recovery-${{ github.run_id }}-receipt+steps:- uses: actions/checkout@v4+ - name: Set up Node.js+ uses: actions/setup-node@v4+ with:+ node-version: '24'++ - name: Install recovery helper runtime+ run: npm ci --ignore-scripts --no-audit --no-fund+- name: Resolve image tagid: tag+ env:+ INPUT_TAG: ${{ inputs.tag }}+ INPUT_INCLUDE_AGENT: ${{ inputs.include_agent || 'Y' }}run: |- TAG="${{ inputs.tag }}"- INCLUDE_AGENT="${{ inputs.include_agent || 'Y' }}"- echo "tag=$TAG" >> $GITHUB_OUTPUT- echo "include_agent=$INCLUDE_AGENT" >> $GITHUB_OUTPUT- echo "→ event=${{ github.event_name }} deploying tag: $TAG include_agent: $INCLUDE_AGENT"+ set -euo pipefail+ TAG="$INPUT_TAG"+ INCLUDE_AGENT="$INPUT_INCLUDE_AGENT"+ case "$INCLUDE_AGENT" in Y|N) ;; *) echo "::error::include_agent must be Y or N"; exit 1 ;; esac+ echo "tag=$TAG" >> "$GITHUB_OUTPUT"+ echo "include_agent=$INCLUDE_AGENT" >> "$GITHUB_OUTPUT"- uses: google-github-actions/auth@v2with:@@ -76,305 +95,73 @@ jobs:run: gcloud components install gke-gcloud-auth-plugin --quiet- name: kubectl cluster-credentials- run: gcloud container clusters get-credentials ${{ env.CLUSTER }} --location ${{ env.CLUSTER_LOCATION }}+ run: gcloud container clusters get-credentials "$CLUSTER" --location "$CLUSTER_LOCATION"- name: Preflight current production API+ continue-on-error: ${{ inputs.recovery_mode == 'forward-only' }}env:CUMORA_SMOKE_TOKEN: ${{ secrets.CUMORA_SMOKE_TOKEN }}CUMORA_SMOKE_COMPANY_ID: ${{ secrets.CUMORA_SMOKE_COMPANY_ID }}CUMORA_SMOKE_BASE: https://api.cumora.ai- # The currently deployed release may predate the candidate's new- # Shipping endpoints. Preflight proves auth + core reads; the- # post-rollout smoke below is the strict Shipping contract gate.CUMORA_SMOKE_REQUIRE_SHIPPING: Nrun: node scripts/release-smoke.mjs- - name: Resolve image to digest+ - name: Resolve candidate image digestsid: digest+ env:+ TAG: ${{ steps.tag.outputs.tag }}+ INCLUDE_AGENT: ${{ steps.tag.outputs.include_agent }}run: |- TAG="${{ steps.tag.outputs.tag }}"- AR="${{ env.AR_REGION }}-docker.pkg.dev/${{ env.GCP_PROJECT }}/${{ env.AR_REPO }}"-- # Resolve through the tag REFERENCE, never `list --filter`. The- # filter's `tags:X` is a substring match over every tag in the repo,- # so `tags:89e7022` also matched 5e60594, 706e708, 7e34456 and five- # more; `head -1` then picked whichever digest sorted first, which on- # 2026-08-31 was a June image. That deploy rolled a three-month-old- # server into production and was only caught because the image- # predated /api/shipping/overview and the smoke gate 404'd. `tags=X`- # is no better — it fuzzy-matches too. `describe <image>:<tag>` is- # exact, and leaves the digest empty when the tag does not exist.- SERVER_DIGEST=$(gcloud artifacts docker images describe "$AR/server:$TAG" \- --format='value(image_summary.digest)' 2>/dev/null)+ set -euo pipefail+ AR="${AR_REGION}-docker.pkg.dev/${GCP_PROJECT}/${AR_REPO}"+ SERVER_DIGEST=$(gcloud artifacts docker images describe "$AR/server:$TAG" --format='value(image_summary.digest)' 2>/dev/null)if [ -z "$SERVER_DIGEST" ]; then- echo "::error::server tag $TAG not found in AR"+ echo "::error::server tag $TAG not found in Artifact Registry"exit 1fi- echo "server=$AR/server@$SERVER_DIGEST" >> $GITHUB_OUTPUT- echo "→ server: $AR/server@$SERVER_DIGEST"-- if [ "${{ steps.tag.outputs.include_agent }}" = "Y" ]; then- AGENT_DIGEST=$(gcloud artifacts docker images describe "$AR/agent-computer:$TAG" \- --format='value(image_summary.digest)' 2>/dev/null)+ echo "server=$AR/server@$SERVER_DIGEST" >> "$GITHUB_OUTPUT"+ if [ "$INCLUDE_AGENT" = "Y" ]; then+ AGENT_DIGEST=$(gcloud artifacts docker images describe "$AR/agent-computer:$TAG" --format='value(image_summary.digest)' 2>/dev/null)if [ -z "$AGENT_DIGEST" ]; then- echo "::error::agent-computer tag $TAG not found in AR while include_agent=Y"- echo "::error::Refusing a server-only deploy because runtime/CLI shim changes require server and agent-computer to stay on the same SHA. For an intentional server-only rollback, rerun manually with include_agent=N."+ echo "::error::agent-computer tag $TAG not found while include_agent=Y"exit 1- else- echo "agent=$AR/agent-computer@$AGENT_DIGEST" >> $GITHUB_OUTPUT- echo "→ agent: $AR/agent-computer@$AGENT_DIGEST"- fi- fi-- - name: Run candidate migrations once- id: migrate- env:- SERVER_IMAGE: ${{ steps.digest.outputs.server }}- REPAIR_0002: ${{ inputs.repair_0002 || 'off' }}- run: |- set -euo pipefail- case "$REPAIR_0002" in- off|archive-detach) ;;- *) echo "::error::repair_0002 must be 'off' or 'archive-detach' (got '$REPAIR_0002')"; exit 1 ;;- esac- if [ "$REPAIR_0002" != off ]; then- echo "::warning::migration 0002 will run repair mode '$REPAIR_0002' — it rewrites conversations.members and archives what it removes into conversation_members_detached_0002"- fi- JOB="cumora-migrate-${GITHUB_SHA:0:7}-${GITHUB_RUN_ATTEMPT}"- DEPLOY_JSON="$RUNNER_TEMP/cumora-deployment.json"- JOB_JSON="$RUNNER_TEMP/cumora-migration-job.json"-- # DATABASE_URL points at 127.0.0.1 because production reaches Cloud- # SQL through the proxy. Copy the live proxy container and workload- # identity settings into one candidate migration Job. A native- # sidecar init container exits automatically when the Job completes.- kubectl get deployment cumora-server -o json > "$DEPLOY_JSON"- jq --arg job "$JOB" --arg server "$SERVER_IMAGE" --arg repair "$REPAIR_0002" '- .spec.template.spec as $pod- | ([ $pod.initContainers[]?, $pod.containers[]? ]- | map(select(.name == "cloud-sql-proxy")) | first) as $proxy- | if $proxy == null then- error("cumora-server has no cloud-sql-proxy container")- else- {- apiVersion: "batch/v1",- kind: "Job",- metadata: {- name: $job,- labels: { app: "cumora-schema-migration" }- },- spec: {- backoffLimit: 0,- activeDeadlineSeconds: 600,- ttlSecondsAfterFinished: 600,- template: {- metadata: { labels: { app: "cumora-schema-migration" } },- spec: {- serviceAccountName: $pod.serviceAccountName,- imagePullSecrets: ($pod.imagePullSecrets // []),- restartPolicy: "Never",- initContainers: [- ($proxy- | del(.readinessProbe, .livenessProbe, .lifecycle)- | .restartPolicy = "Always"- | .startupProbe = (.startupProbe // {- httpGet: { path: "/readiness", port: 9090 },- periodSeconds: 1,- failureThreshold: 60- }))- ],- containers: [- {- name: "migrate",- image: $server,- imagePullPolicy: "IfNotPresent",- command: ["npm", "run", "migrate"],- envFrom: [{ secretRef: { name: "cumora" } }],- # After envFrom on purpose: an explicit `env` entry- # wins over one supplied by envFrom, so the- # operator choice for this one run cannot be- # shadowed by whatever the Secret happens to hold.- env: [{ name: "MIGRATION_0002_REPAIR", value: $repair }],- resources: {- requests: { cpu: "100m", memory: "256Mi" },- limits: { cpu: "1000m", memory: "1Gi" }- }- }- ]- }- }- }- }- end- ' "$DEPLOY_JSON" > "$JOB_JSON"-- kubectl delete job "$JOB" --ignore-not-found --wait=true- kubectl apply -f "$JOB_JSON"-- SUCCEEDED=N- for _ in $(seq 1 120); do- STATUS=$(kubectl get job "$JOB" -o json)- if [ "$(echo "$STATUS" | jq -r '.status.succeeded // 0')" -ge 1 ]; then- SUCCEEDED=Y- break- fi- if [ "$(echo "$STATUS" | jq -r '.status.failed // 0')" -ge 1 ]; then- breakfi- sleep 5- done-- echo "::group::Migration output"- kubectl logs "job/$JOB" -c migrate --tail=300 2>&1 || true- echo "::endgroup::"- if [ "$SUCCEEDED" != "Y" ]; then- kubectl describe job "$JOB"- kubectl get pods -l job-name="$JOB" -o wide- echo "::error::Candidate database migration did not complete; deployment was not mutated."- exit 1+ echo "agent=$AR/agent-computer@$AGENT_DIGEST" >> "$GITHUB_OUTPUT"fi- kubectl delete job "$JOB" --wait=true- echo "job=$JOB" >> "$GITHUB_OUTPUT"-- - name: Patch deployment- id: patch- run: |- SERVER="${{ steps.digest.outputs.server }}"-- PREVIOUS_SERVER=$(kubectl get deployment cumora-server -o jsonpath='{.spec.template.spec.containers[?(@.name=="server")].image}')- PREVIOUS_AGENT=$(kubectl get deployment cumora-server -o jsonpath='{.spec.template.spec.containers[?(@.name=="server")].env[?(@.name=="CUMORA_AGENT_COMPUTER_IMAGE")].value}')- echo "previous_server=$PREVIOUS_SERVER" >> $GITHUB_OUTPUT- echo "previous_agent=$PREVIOUS_AGENT" >> $GITHUB_OUTPUT- echo "→ rollback baseline server: $PREVIOUS_SERVER"- echo "→ rollback baseline agent: ${PREVIOUS_AGENT:-'(unset)'}"-- # Apply one atomic Pod-template patch: select the candidate server,- # remove the retired per-replica migration init container, and- # re-assert the startup/readiness/liveness probe contract. Startup- # allows the server's bounded schema transport retry to finish;- # liveness MUST hit /api/livez (a DB-free "process alive" check),- # NOT /api/health (which runs SELECT 1). Readiness stays on- # /api/health so a pod that cannot reach the DB is kept out of- # rotation without being killed. This keeps the fix durable even if- # the Deployment is recreated from an older base manifest.- AGENT="${{ steps.digest.outputs.agent }}"- PATCH=$(jq -nc --arg server "$SERVER" --arg agent "$AGENT" '- {- spec: { template: { spec: {- initContainers: [- { name: "migrate", "$patch": "delete" }- ],- containers: [- ({- name: "server",- image: $server,- startupProbe: {- httpGet: { path: "/api/livez", port: "http" },- periodSeconds: 5,- timeoutSeconds: 2,- failureThreshold: 60- },- readinessProbe: {- httpGet: { path: "/api/health", port: "http" },- periodSeconds: 5,- timeoutSeconds: 2- },- livenessProbe: {- httpGet: { path: "/api/livez", port: "http" },- periodSeconds: 30,- timeoutSeconds: 3- }- } + if $agent == "" then {} else {- env: [{ name: "CUMORA_AGENT_COMPUTER_IMAGE", value: $agent }]- } end)- ]- } } }- }- ')- kubectl patch deployment cumora-server --type=strategic -p="$PATCH"-- - name: Wait for rollout- id: rollout- run: |- # If rollout times out, fall through to the post-mortem- # step below — we want the diagnostic dump to print BEFORE- # the job fails, otherwise GHA just shows "Process completed- # with exit code 1" and we have to manually kubectl-investigate.- set +e- kubectl rollout status deployment/cumora-server --timeout=10m- RC=$?- set -e- echo "rc=$RC" >> $GITHUB_OUTPUT- # Don't exit yet — let the post-mortem step run regardless of RC.- # The "Fail if rollout failed" step at the end actually fails the job.-- - name: Post-mortem on rollout failure- if: steps.rollout.outputs.rc != '0'- run: |- echo "::group::Pods"- kubectl get pods -l app=cumora-server -o wide- echo "::endgroup::"- echo "::group::Pod descriptions (events)"- # Per-pod describe; tail keeps the output focused on Events- for POD in $(kubectl get pods -l app=cumora-server -o name); do- echo "==== $POD ===="- kubectl describe "$POD" | sed -n '/Events:/,$p' | head -40- done- echo "::endgroup::"- echo "::group::Deployment status"- kubectl describe deployment cumora-server | grep -A 10 'Replicas:\|Conditions:\|StrategyType:'- echo "::endgroup::"- echo "::group::Recent cluster events"- kubectl get events --sort-by=.lastTimestamp 2>&1 | tail -30- echo "::endgroup::"- echo "::group::Server container logs (current run)"- for POD in $(kubectl get pods -l app=cumora-server -o jsonpath='{.items[*].metadata.name}'); do- echo "==== $POD ===="- kubectl logs "$POD" -c server --tail=100 2>&1 || true- done- echo "::endgroup::"- echo "::group::Server container logs (previous run, if crashloop)"- for POD in $(kubectl get pods -l app=cumora-server -o jsonpath='{.items[*].metadata.name}'); do- echo "==== $POD ===="- kubectl logs "$POD" -c server --previous --tail=100 2>&1 || true- done- echo "::endgroup::"-- - name: Fail if rollout failed- if: steps.rollout.outputs.rc != '0'- run: |- echo "::error::Rollout failed with rc=${{ steps.rollout.outputs.rc }} — see Post-mortem step above."- exit 1-- - name: Authenticated smoke test- id: smoke- continue-on-error: true+ # This is the only mutating production step. The helper captures the+ # baseline before migration, creates immutable state records, preserves+ # Job identity/proxy/volumes/env/securityContext, applies JSON Patch CAS,+ # handles every rollout failure and smoke failure, and never calls+ # rollout undo.+ - name: Run deployment transaction with guarded recovery+ id: deployenv:+ RECOVERY_WORKDIR: ${{ runner.temp }}/cumora-recovery-${{ github.run_id }}+ DEPLOYMENT: ${{ env.DEPLOYMENT }}+ INCLUDE_AGENT: ${{ steps.tag.outputs.include_agent }}+ RECOVERY_MODE: ${{ inputs.recovery_mode || 'normal' }}+ CANDIDATE_SERVER_IMAGE: ${{ steps.digest.outputs.server }}+ CANDIDATE_AGENT_IMAGE: ${{ steps.digest.outputs.agent }}+ MIGRATION_REPAIR: ${{ inputs.repair_0002 || 'off' }}+ MIGRATION_DEADLINE_SECONDS: '600'+ MIGRATION_POLL_ATTEMPTS: '200'+ VERIFIER_POLL_ATTEMPTS: '75'+ RECOVERY_POLL_MS: '3000'+ ROLLOUT_TIMEOUT_SECONDS: '600'+ SMOKE_TIMEOUT_SECONDS: '120'CUMORA_SMOKE_TOKEN: ${{ secrets.CUMORA_SMOKE_TOKEN }}CUMORA_SMOKE_COMPANY_ID: ${{ secrets.CUMORA_SMOKE_COMPANY_ID }}CUMORA_SMOKE_BASE: https://api.cumora.ai- run: node scripts/release-smoke.mjs-- - name: Automatic rollback when smoke evidence fails- if: steps.smoke.outcome == 'failure'- run: |- echo "::warning::Authenticated smoke failed; rolling deployment back to the previous revision."- kubectl rollout undo deployment/cumora-server- kubectl rollout status deployment/cumora-server --timeout=10m- echo "::error::Candidate was rolled back. Previous server was ${{ steps.patch.outputs.previous_server }}."-- - name: Fail release after rollback- if: steps.smoke.outcome == 'failure'- run: exit 1+ run: node --import tsx scripts/deploy-release.mjs run- name: Summary+ if: steps.deploy.outcome == 'success'+ env:+ SERVER_DIGEST: ${{ steps.digest.outputs.server }}+ AGENT_DIGEST: ${{ steps.digest.outputs.agent || '(unchanged)' }}run: |- echo "### Deployed" >> $GITHUB_STEP_SUMMARY- echo "- tag: \`${{ steps.tag.outputs.tag }}\`" >> $GITHUB_STEP_SUMMARY- echo "- server: \`${{ steps.digest.outputs.server }}\`" >> $GITHUB_STEP_SUMMARY- echo "- agent: \`${{ steps.digest.outputs.agent || '(unchanged)' }}\`" >> $GITHUB_STEP_SUMMARY- echo "- previous server: \`${{ steps.patch.outputs.previous_server }}\`" >> $GITHUB_STEP_SUMMARY- echo "- migration owner: one pre-deploy Job (\`${{ steps.migrate.outputs.job }}\`)" >> $GITHUB_STEP_SUMMARY- echo "- authenticated smoke: passed" >> $GITHUB_STEP_SUMMARY- echo "- cluster: \`${{ env.CLUSTER }}\` / \`${{ env.CLUSTER_LOCATION }}\`" >> $GITHUB_STEP_SUMMARY+ printf '%s\n' '### Deployed' >> "$GITHUB_STEP_SUMMARY"+ printf '%s\n' "- server digest: $SERVER_DIGEST" >> "$GITHUB_STEP_SUMMARY"+ printf '%s\n' "- agent digest: $AGENT_DIGEST" >> "$GITHUB_STEP_SUMMARY"+ printf '%s\n' '- candidate migration, rollout, and authenticated smoke: passed' >> "$GITHUB_STEP_SUMMARY"diff --git a/docs/RELEASE.md b/docs/RELEASE.mdindex 45d9c66..42a634f 100644--- a/docs/RELEASE.md+++ b/docs/RELEASE.md@@ -48,18 +48,105 @@ To deploy a candidate:precondition — see below.4. Approve the protected `production` environment. The approver should not bethe person who built the feature for high-risk changes.-5. Verify the workflow summary contains the selected digest, previous server- image, completed rollout, and passed authenticated smoke.+5. Verify the workflow summary contains the selected digest, a completed+ rollout, and passed authenticated smoke. If the candidate fails, the+ workflow remains red even when guarded recovery restores the previous+ template successfully.++The normal workflow requires the captured Deployment to have an observed,+fully available rollout and a passing core baseline smoke. When that baseline+is known to be unhealthy, an approver may explicitly choose+`recovery_mode=forward-only` in the workflow dispatch. That mode still captures+the UID/template, runs the candidate migration and CAS patch, and requires the+candidate rollout plus strict Shipping smoke. It never runs automatic restore+after a candidate rollout or smoke failure; the failed state is handled by a+forward-compatible deployment plan.Deploy first proves that the existing production API and smoke credential are-healthy, runs one candidate-image migration Job and verifies its immutable-ledger/index gates, records the current revision as the rollback baseline,-updates the server (and optionally agent runtime) by digest, waits for GKE, then-exercises real authenticated tenant paths: auth, conversations, and the-Shipping overview/schema. A migration failure leaves the Deployment untouched;-a failed post-deploy smoke automatically runs-`kubectl rollout undo`, waits for the old revision to become ready, and fails-the workflow.+healthy, captures the complete Deployment Pod template and UID in an immutable+create-only recovery Secret, runs one candidate-image migration Job, updates+the server (and optionally agent runtime) by digest with an atomic+compare-and-swap patch, waits for GKE, then exercises real authenticated tenant+paths: auth, conversations, and the Shipping overview/schema. A migration+failure leaves the Deployment untouched and retains the Job for inspection.+The baseline and restored-template smoke checks cover the core authenticated+paths; the candidate success check additionally requires Shipping overview and+schema.++The recovery path never uses `kubectl rollout undo`. Any failed rollout status+or smoke failure first checks that the UID and exact candidate template still match the+protected receipt. It then runs the captured old server image in an independent+read-only schema verifier Job. The verifier imports its own `pool.ts`,+`schema-version.ts`, and migration manifest, runs `BEGIN READ ONLY` with a+30-second statement timeout, and must emit one valid compatibility marker. A+missing module, timeout, malformed output, unknown schema, or template drift+refuses recovery and leaves the candidate state for forward repair. Only a+compatible result restores the exact captured template, waits for rollout, and+runs authenticated smoke again. Recovery success is evidence for operators;+the candidate workflow still fails.++The baseline and candidate receipt Secrets contain Pod-template state and are+immutable. Do not print, upload, or copy their contents. A retry reads and+validates the original baseline instead of replacing it. Cancellation can skip+cleanup, so inspect retained migration/verifier Jobs and the protected records+before deleting anything.++### Backend recovery runbook++The release path classifies a failed candidate as follows:++- **Same schema:** the old image's read-only verifier reports its current+ ledger version inside that image's exact supported range. If the candidate+ template has not drifted, the workflow restores the captured server/agent+ images, probes, environment, mounts, and security context atomically, then+ proves rollout and authenticated smoke.+- **Newer or damaged schema:** the old verifier returns a known incompatible+ result. Automatic restoration is refused. Keep the migration Job and+ protected receipt, stop further deploy attempts, and use a forward-compatible+ image or an explicit database repair plan. Lowering an image's MIN bound or+ silently accepting an unknown migration is unsafe.+- **Unknown compatibility:** missing old-image modules, a verifier timeout,+ connection/cleanup error, a non-unique marker, invalid JSON, or any extra+ unexpected output is treated as unknown. Do not restore automatically.+- **Restore or smoke failure:** a failed JSON Patch CAS, rollout, or restored+ smoke leaves the workflow failed and requires manual forward recovery. Check+ the Deployment UID/template hash and controller events before a new run.++The checked-in helper has an explicit recovery-only entry point. It does not+capture a new baseline, run a migration, or patch a candidate before the+compatibility check:++```sh+export KUBECTL=kubectl+export DEPLOYMENT=cumora-server+export DEPLOYMENT_NAMESPACE=default+export RECOVERY_SECRET='cumora-deploy-recovery-<run-id>'+export RECEIPT_SECRET='cumora-deploy-recovery-<run-id>-receipt'+export RECOVERY_WORKDIR=/tmp/cumora-manual-recovery+export VERIFIER_JOB_NAME="cumora-schema-verify-manual-$(date -u +%Y%m%d%H%M%S)"+export CUMORA_SMOKE_TOKEN='(read from the protected operator environment)'+export CUMORA_SMOKE_COMPANY_ID='(read from the protected operator environment)'+export CUMORA_SMOKE_BASE=https://api.cumora.ai+node --import tsx scripts/deploy-release.mjs recover+```++Use the exact protected Secret names from the failed run and run this from a+checkout with the same helper. The command reads and validates the immutable+baseline and candidate receipt, blocks on active migration Jobs, runs the old+server image's read-only verifier under the fresh `VERIFIER_JOB_NAME`, then performs the UID/template CAS restore,+rollout, and authenticated smoke. It never runs migration or accepts a+mutable image. A missing baseline/receipt, active Job, schema incompatibility,+unknown verifier result, or CAS drift returns a stable failure code and leaves+the current Deployment unchanged. Inspect completed Jobs and their proxy state+before retrying; do not delete an active Job to force a new result.++If the receipt is missing or compatibility is unknown, use the explicit+forward-only workflow mode: choose an image whose own manifest explicitly+supports the current ledger, review any migration/expand-contract repair+separately, and dispatch `Deploy` with `recovery_mode=forward-only`. Do not+manufacture a baseline or lower the old image's supported minimum. Manual+recovery success still belongs to the failed candidate incident; run a new+reviewed forward deployment before marking the release successful.#### When migration 0002 refuses to applydiff --git a/docs/SHIPPING.md b/docs/SHIPPING.mdindex b465c89..a3c66d5 100644--- a/docs/SHIPPING.md+++ b/docs/SHIPPING.md@@ -71,9 +71,11 @@ the mute point without replaying the old backlog.Backend deployment is intentionally separate from desktop tagging. See[RELEASE.md](./RELEASE.md) for protected production approval, digest-pinned GKE-rollout, authenticated smoke, automatic rollback, and scheduled readback.+rollout, immutable recovery records, read-only schema compatibility checks,+authenticated smoke, conditional exact-template recovery, and scheduled+readback. A recovered candidate remains a failed release until a new reviewed+forward deployment is verified.The release contract is complete only after production behavior has been readback against its baseline. A green build or successful rollout is anintermediate signal, not the terminal state.-diff --git a/scripts/deploy-release.mjs b/scripts/deploy-release.mjsnew file mode 100644index 0000000..4e5d134--- /dev/null+++ b/scripts/deploy-release.mjs@@ -0,0 +1,490 @@+import { spawn } from 'node:child_process'+import { writeFile, mkdir } from 'node:fs/promises'+import { join } from 'node:path'+import { pathToFileURL } from 'node:url'++const recovery = await import('../server/src/deploy/recovery.ts')++const DEFAULT_ROLLOUT_TIMEOUT_SECONDS = 600+const DEFAULT_JOB_TIMEOUT_SECONDS = 600+const DEFAULT_POLL_SECONDS = 3+const DEFAULT_POLL_ATTEMPTS = 30++class CommandFailure extends Error {+ constructor(code, details = {}) {+ super(code)+ this.code = code+ this.details = details+ }+}++function envRequired(name) {+ const value = process.env[name]+ if (!value) throw new CommandFailure(`missing_${name.toLowerCase()}`)+ return value+}++function parseJson(raw, code) {+ try {+ return JSON.parse(raw)+ } catch {+ throw new CommandFailure(code)+ }+}++function isComplete(job) {+ return Number(job?.status?.succeeded ?? 0) > 0 || (Array.isArray(job?.status?.conditions) && job.status.conditions.some((condition) => condition?.type === 'Complete' && condition?.status === 'True'))+}++function isFailed(job) {+ // A non-zero failed counter can coexist with active retries. Only the+ // terminal Failed condition proves that a Job is no longer a writer.+ return Array.isArray(job?.status?.conditions) && job.status.conditions.some((condition) => condition?.type === 'Failed' && condition?.status === 'True')+}++function hasActiveWriter(job) {+ return Number(job?.status?.active ?? 0) > 0+}++function hasTerminalCondition(job) {+ return isFailed(job) || (Array.isArray(job?.status?.conditions) && job.status.conditions.some((condition) => condition?.type === 'Complete' && condition?.status === 'True'))+}++function sleep(ms) {+ return new Promise((resolve) => setTimeout(resolve, ms))+}++function runProcess(command, args, options = {}) {+ const timeoutMs = options.timeoutMs ?? 120_000+ return new Promise((resolve, reject) => {+ const child = spawn(command, args, {+ cwd: options.cwd,+ env: options.env ?? process.env,+ stdio: ['pipe', 'pipe', 'pipe'],+ })+ let stdout = ''+ let stderr = ''+ let timedOut = false+ const timer = setTimeout(() => {+ timedOut = true+ child.kill('SIGTERM')+ setTimeout(() => child.kill('SIGKILL'), 1_000).unref()+ }, timeoutMs)+ child.stdout.setEncoding('utf8')+ child.stderr.setEncoding('utf8')+ child.stdout.on('data', (chunk) => { stdout += chunk })+ child.stderr.on('data', (chunk) => { stderr += chunk })+ child.on('error', (error) => {+ clearTimeout(timer)+ reject(new CommandFailure('process_spawn_failed', { cause: error }))+ })+ child.on('close', (exitCode, signal) => {+ clearTimeout(timer)+ if (timedOut) {+ reject(new CommandFailure('process_timeout', { stdout, stderr, signal }))+ } else if (exitCode !== 0) {+ reject(new CommandFailure('process_failed', { stdout, stderr, exitCode, signal }))+ } else {+ resolve({ stdout, stderr })+ }+ })+ if (options.input) child.stdin.write(options.input)+ child.stdin.end()+ })+}++function namespaceArgs() {+ const namespace = process.env.DEPLOYMENT_NAMESPACE+ return namespace ? ['-n', namespace] : []+}++async function kubectl(args, options = {}) {+ const command = process.env.KUBECTL || 'kubectl'+ try {+ return await runProcess(command, [...namespaceArgs(), ...args], options)+ } catch (error) {+ if (error instanceof CommandFailure) throw new CommandFailure(`kubectl_${error.code}`, error.details)+ throw error+ }+}++function workPath(name) {+ const root = process.env.RECOVERY_WORKDIR || process.env.RUNNER_TEMP || '/tmp'+ return join(root, `cumora-deploy-${process.env.GITHUB_RUN_ID || 'local'}-${name}`)+}++async function writePrivate(path, value) {+ await writeFile(path, `${typeof value === 'string' ? value : JSON.stringify(value, null, 2)}\n`, { mode: 0o600 })+}++async function getDeployment(name) {+ let result+ try {+ result = await kubectl(['get', 'deployment', name, '-o', 'json'])+ } catch {+ throw new CommandFailure('deployment_read_failed')+ }+ return parseJson(result.stdout, 'deployment_json_invalid')+}++async function getJob(name) {+ const result = await kubectl(['get', 'job', name, '--ignore-not-found', '-o', 'json'])+ if (!result.stdout.trim()) return null+ return parseJson(result.stdout, 'job_json_invalid')+}++async function getSecret(name) {+ try {+ const result = await kubectl(['get', 'secret', name, '--ignore-not-found', '-o', 'json'])+ if (!result.stdout.trim()) return null+ return parseJson(result.stdout, 'secret_json_invalid')+ } catch (_error) {+ throw new CommandFailure('secret_read_failed')+ }+}++async function getOptionalJob(name) {+ try {+ return await getJob(name)+ } catch {+ throw new CommandFailure('job_read_failed')+ }+}++async function listResidualMigrationJobs() {+ const items = []+ for (const selector of ['app=cumora-schema-migration', 'app=cumora-schema-recovery']) {+ let result+ try {+ result = await kubectl(['get', 'jobs', '--ignore-not-found', '-l', selector, '-o', 'json'])+ } catch {+ throw new CommandFailure('migration_jobs_read_failed')+ }+ if (!result.stdout.trim()) continue+ const listing = parseJson(result.stdout, 'migration_jobs_json_invalid')+ if (!listing || typeof listing !== 'object' || !Array.isArray(listing.items)) throw new CommandFailure('migration_jobs_json_invalid')+ items.push(...listing.items)+ }+ return items+}++function secretData(secret, key) {+ const encoded = secret?.data?.[key]+ if (typeof encoded !== 'string') throw new CommandFailure('protected_state_missing')+ try {+ return parseJson(Buffer.from(encoded, 'base64').toString('utf8'), 'protected_state_invalid')+ } catch {+ throw new CommandFailure('protected_state_invalid')+ }+}++async function createImmutableSecret(name, key, value, labels) {+ const existing = await getSecret(name)+ if (existing) return existing+ const manifest = {+ apiVersion: 'v1',+ kind: 'Secret',+ metadata: { name, labels },+ immutable: true,+ type: 'Opaque',+ data: { [key]: Buffer.from(`${JSON.stringify(value)}\n`, 'utf8').toString('base64') },+ }+ try {+ await kubectl(['create', '-f', '-'], { input: JSON.stringify(manifest) })+ } catch (error) {+ // A concurrent retry may have won the create-only race. Read and verify+ // the protected state; never apply/update the original baseline Secret.+ if (!/already.?exists|409/i.test(`${error.details?.stderr ?? ''} ${error.details?.stdout ?? ''}`)) {+ throw new CommandFailure('protected_state_create_failed')+ }+ const raced = await getSecret(name)+ if (!raced) throw new CommandFailure('protected_state_create_failed')+ return raced+ }+ const created = await getSecret(name)+ if (!created) throw new CommandFailure('protected_state_read_failed')+ return created+}++async function captureBaseline(paths) {+ const deployment = await getDeployment(paths.deployment)+ let baseline = recovery.extractDeploymentSnapshot(deployment, {+ expectedName: paths.deployment,+ expectedNamespace: process.env.DEPLOYMENT_NAMESPACE,+ })+ const forwardOnly = process.env.RECOVERY_MODE === 'forward-only'+ if (!forwardOnly) {+ recovery.assertDeploymentHealthy(baseline)+ if (!(await smoke(false))) throw new CommandFailure('baseline_smoke_failed')+ }+ baseline = { ...baseline, baselineSmokePassed: !forwardOnly, baselineVerifiedAt: new Date().toISOString() }+ await writePrivate(paths.baseline, baseline)+ const existing = await getSecret(paths.recoverySecret)+ let protectedBaseline+ if (existing) {+ protectedBaseline = secretData(existing, 'baseline.json')+ } else {+ const created = await createImmutableSecret(paths.recoverySecret, 'baseline.json', baseline, {+ app: 'cumora-deploy-recovery',+ 'recovery.cumora.ai/run-id': process.env.GITHUB_RUN_ID || 'local',+ })+ protectedBaseline = secretData(created, 'baseline.json')+ }+ recovery.assertBaselineUnchanged(protectedBaseline, baseline)+ return baseline+}++async function ensureBaselineUnchanged(paths, baseline) {+ const current = recovery.extractDeploymentSnapshot(await getDeployment(paths.deployment), {+ expectedName: baseline.name,+ expectedNamespace: baseline.namespace,+ })+ recovery.assertBaselineUnchanged(baseline, current)+ return current+}++async function runMigration(paths, baseline) {+ await ensureBaselineUnchanged(paths, baseline)+ const name = process.env.MIGRATION_JOB_NAME || `cumora-migrate-${(process.env.GITHUB_SHA || 'local').slice(0, 7)}-${process.env.GITHUB_RUN_ID || 'local'}`+ const residual = await listResidualMigrationJobs()+ if (residual.some((job) => hasActiveWriter(job) || !hasTerminalCondition(job))) throw new CommandFailure('migration_residual_job_present')+ const existing = await getOptionalJob(name)+ if (existing) throw new CommandFailure('migration_job_exists')+ const job = recovery.buildMigrationJob(baseline, {+ name,+ image: envRequired('CANDIDATE_SERVER_IMAGE'),+ repairMode: process.env.MIGRATION_REPAIR === 'archive-detach' ? 'archive-detach' : 'off',+ activeDeadlineSeconds: Number(process.env.MIGRATION_DEADLINE_SECONDS || DEFAULT_JOB_TIMEOUT_SECONDS),+ })+ await kubectl(['apply', '-f', '-'], { input: JSON.stringify(job) })+ await waitForJob(name, Number(process.env.MIGRATION_POLL_ATTEMPTS || DEFAULT_POLL_ATTEMPTS))+ return name+}++async function waitForJob(name, attempts) {+ const pollMs = Number(process.env.RECOVERY_POLL_MS || DEFAULT_POLL_SECONDS * 1_000)+ for (let attempt = 0; attempt < attempts; attempt += 1) {+ const job = await getJob(name)+ if (!job) {+ if (attempt + 1 < attempts) await sleep(pollMs)+ continue+ }+ if (isComplete(job)) return job+ if (isFailed(job)) throw new CommandFailure('job_failed')+ if (attempt + 1 < attempts) await sleep(pollMs)+ }+ throw new CommandFailure('job_timeout')+}++async function applyCandidate(paths, baseline) {+ const current = await ensureBaselineUnchanged(paths, baseline)+ const images = { server: envRequired('CANDIDATE_SERVER_IMAGE') }+ if (process.env.INCLUDE_AGENT !== 'N') {+ const agent = envRequired('CANDIDATE_AGENT_IMAGE')+ images.agent = agent+ }+ const patch = recovery.buildCandidatePatch(baseline, images)+ await writePrivate(paths.candidatePatch, patch)+ let response+ try {+ response = await kubectl(['patch', 'deployment', paths.deployment, '--type=json', '--patch-file', paths.candidatePatch, '-o', 'json'])+ } catch {+ throw new CommandFailure('candidate_patch_failed')+ }+ const receiptObject = parseJson(response.stdout, 'candidate_receipt_invalid')+ const receipt = recovery.deploymentReceiptFromObject(receiptObject)+ recovery.assertCandidateReceiptMatches(baseline, receipt, images)+ await writePrivate(paths.receipt, receipt)+ const existingReceipt = await getSecret(paths.receiptSecret)+ let protectedReceipt+ if (existingReceipt) {+ protectedReceipt = secretData(existingReceipt, 'receipt.json')+ } else {+ const created = await createImmutableSecret(paths.receiptSecret, 'receipt.json', receipt, {+ app: 'cumora-deploy-recovery',+ 'recovery.cumora.ai/run-id': process.env.GITHUB_RUN_ID || 'local',+ 'recovery.cumora.ai/receipt': 'true',+ })+ protectedReceipt = secretData(created, 'receipt.json')+ }+ if (protectedReceipt.uid !== receipt.uid || protectedReceipt.podTemplateHash !== receipt.podTemplateHash) {+ throw new CommandFailure('candidate_receipt_conflict')+ }+ return { receipt, current }+}++async function rollout() {+ try {+ await kubectl(['rollout', 'status', `deployment/${process.env.DEPLOYMENT || 'cumora-server'}`, `--timeout=${Number(process.env.ROLLOUT_TIMEOUT_SECONDS || DEFAULT_ROLLOUT_TIMEOUT_SECONDS)}s`], {+ timeoutMs: (Number(process.env.ROLLOUT_TIMEOUT_SECONDS || DEFAULT_ROLLOUT_TIMEOUT_SECONDS) + 5) * 1_000,+ })+ return { ok: true }+ } catch {+ return { ok: false }+ }+}++async function smoke(requireShipping = true) {+ try {+ await runProcess(process.execPath, ['scripts/release-smoke.mjs'], {+ cwd: process.env.GITHUB_WORKSPACE || process.cwd(),+ timeoutMs: Number(process.env.SMOKE_TIMEOUT_SECONDS || 120) * 1_000,+ env: { ...process.env, CUMORA_SMOKE_REQUIRE_SHIPPING: requireShipping ? 'Y' : 'N' },+ })+ return true+ } catch {+ return false+ }+}++async function runSchemaVerifier(paths, baseline) {+ const name = process.env.VERIFIER_JOB_NAME || `cumora-schema-verify-${(process.env.GITHUB_SHA || 'local').slice(0, 7)}-${process.env.GITHUB_RUN_ID || 'local'}`+ const existing = await getOptionalJob(name)+ if (existing) throw new CommandFailure('verifier_job_exists')+ const job = recovery.buildSchemaVerifierJob(baseline, { name, image: baseline.serverImage })+ await kubectl(['apply', '-f', '-'], { input: JSON.stringify(job) })+ const status = await waitForJob(name, Number(process.env.VERIFIER_POLL_ATTEMPTS || DEFAULT_POLL_ATTEMPTS))+ if (!isComplete(status)) throw new CommandFailure('verifier_failed')+ let logs+ try {+ logs = await kubectl(['logs', `job/${name}`, '-c', 'migrate'])+ } catch {+ throw new CommandFailure('verifier_logs_failed')+ }+ // `kubectl logs` stdout is the verifier protocol. kubectl's own stderr is+ // transport/diagnostic output and must not be mixed into the old image's+ // strict single-marker stream.+ const result = recovery.parseSchemaVerifierOutput(logs.stdout)+ if (result.status !== 'compatible') throw new CommandFailure(result.status === 'incompatible' ? 'schema_incompatible' : 'schema_unknown')+ return result+}++async function restore(paths, baseline, expectedHash) {+ const currentObject = await getDeployment(paths.deployment)+ const current = recovery.deploymentReceiptFromObject(currentObject)+ const patch = recovery.buildRestorePatch(baseline, current, expectedHash)+ await writePrivate(paths.restorePatch, patch)+ try {+ await kubectl(['patch', 'deployment', paths.deployment, '--type=json', '--patch-file', paths.restorePatch, '-o', 'json'])+ } catch {+ throw new CommandFailure('restore_patch_failed')+ }+ if (!(await rollout()).ok) throw new CommandFailure('restore_rollout_failed')+ if (!(await smoke(false))) throw new CommandFailure('restore_smoke_failed')+ const restored = recovery.extractDeploymentSnapshot(await getDeployment(paths.deployment), {+ expectedName: baseline.name,+ expectedNamespace: baseline.namespace,+ })+ if (restored.uid !== baseline.uid || restored.podTemplateHash !== baseline.podTemplateHash) {+ throw new CommandFailure('restore_receipt_mismatch')+ }+ recovery.assertDeploymentHealthy(restored)+}++async function recover(paths, baseline) {+ recovery.assertRecoveryEligible(baseline)+ const protectedReceiptSecret = await getSecret(paths.receiptSecret)+ if (!protectedReceiptSecret) throw new CommandFailure('protected_receipt_missing')+ const protectedReceipt = secretData(protectedReceiptSecret, 'receipt.json')+ recovery.assertReceiptShapeForRecovery(protectedReceipt)+ recovery.assertRecoveryStateBinding(baseline, protectedReceipt)+ // This first CAS check is deliberately before the old-image verifier. A+ // controller/operator drift must never be hidden by a successful verifier.+ const current = recovery.deploymentReceiptFromObject(await getDeployment(paths.deployment))+ recovery.buildRestorePatch(baseline, current, protectedReceipt.podTemplateHash)+ await runSchemaVerifier(paths, baseline)+ await restore(paths, baseline, protectedReceipt.podTemplateHash)+}++async function recoverExistingDeployment() {+ const state = paths()+ await mkdir(state.root, { recursive: true, mode: 0o700 })+ const baselineSecret = await getSecret(state.recoverySecret)+ if (!baselineSecret) throw new CommandFailure('protected_baseline_missing')+ const receiptSecret = await getSecret(state.receiptSecret)+ if (!receiptSecret) throw new CommandFailure('protected_receipt_missing')+ const baseline = secretData(baselineSecret, 'baseline.json')+ const receipt = secretData(receiptSecret, 'receipt.json')+ recovery.assertReceiptShapeForRecovery(receipt)+ recovery.assertRecoveryStateBinding(baseline, receipt)+ await writePrivate(state.baseline, baseline)+ await writePrivate(state.receipt, receipt)+ const residual = await listResidualMigrationJobs()+ if (residual.some((job) => hasActiveWriter(job) || !hasTerminalCondition(job))) {+ throw new CommandFailure('migration_residual_job_active')+ }+ const current = recovery.deploymentReceiptFromObject(await getDeployment(state.deployment))+ recovery.buildRestorePatch(baseline, current, receipt.podTemplateHash)+ recovery.assertRecoveryEligible(baseline)+ await runSchemaVerifier(state, baseline)+ await restore(state, baseline, receipt.podTemplateHash)+ return { status: 'recovered', baselineHash: baseline.podTemplateHash }+}++function paths() {+ const deployment = process.env.DEPLOYMENT || 'cumora-server'+ return {+ deployment,+ recoverySecret: envRequired('RECOVERY_SECRET'),+ receiptSecret: envRequired('RECEIPT_SECRET'),+ root: process.env.RECOVERY_WORKDIR || process.env.RUNNER_TEMP || '/tmp',+ baseline: workPath('baseline.json'),+ candidatePatch: workPath('candidate-patch.json'),+ receipt: workPath('receipt.json'),+ restorePatch: workPath('restore-patch.json'),+ }+}++export async function runDeploymentRelease() {+ const state = paths()+ await mkdir(state.root, { recursive: true, mode: 0o700 })+ const baseline = await captureBaseline(state)+ await runMigration(state, baseline)+ await applyCandidate(state, baseline)+ const rolledOut = await rollout()+ if (!rolledOut.ok) {+ if (process.env.RECOVERY_MODE === 'forward-only') throw new CommandFailure('candidate_failed_forward_only')+ try {+ await recover(state, baseline)+ } catch (error) {+ throw new CommandFailure(error instanceof CommandFailure ? error.code : 'recovery_failed')+ }+ throw new CommandFailure('candidate_failed_recovered')+ }+ if (!(await smoke(true))) {+ if (process.env.RECOVERY_MODE === 'forward-only') throw new CommandFailure('candidate_failed_forward_only')+ try {+ await recover(state, baseline)+ } catch (error) {+ throw new CommandFailure(error instanceof CommandFailure ? error.code : 'recovery_failed')+ }+ throw new CommandFailure('candidate_failed_recovered')+ }+ const candidateObject = await getDeployment(state.deployment)+ const candidateSnapshot = recovery.extractDeploymentSnapshot(candidateObject, {+ expectedName: baseline.name,+ expectedNamespace: baseline.namespace,+ })+ const candidate = recovery.deploymentReceiptFromObject(candidateObject)+ const protectedReceipt = secretData(await getSecret(state.receiptSecret), 'receipt.json')+ recovery.assertDeploymentReceiptMatches(baseline, candidate, protectedReceipt.podTemplateHash)+ recovery.assertDeploymentHealthy(candidateSnapshot)+ return { status: 'deployed', baselineHash: baseline.podTemplateHash }+}++export async function main(argv = process.argv.slice(2)) {+ const command = argv[0] || 'run'+ if (command === 'run') return runDeploymentRelease()+ if (command === 'recover' || command === 'resume') return recoverExistingDeployment()+ throw new CommandFailure('unknown_command')+}++if (import.meta.url === pathToFileURL(process.argv[1] ?? '').href) {+ try {+ const result = await main()+ process.stdout.write(`${JSON.stringify(result)}\n`)+ } catch (error) {+ process.stderr.write(`deploy-release: ${error instanceof CommandFailure ? error.code : 'failed'}\n`)+ process.exitCode = 1+ }+}diff --git a/server/k8s/gke.md b/server/k8s/gke.mdindex a4d6cb6..54a8423 100644--- a/server/k8s/gke.md+++ b/server/k8s/gke.md@@ -219,7 +219,10 @@ After replacing `REPLACE-*` placeholders in```sh# For a manual installation, run the candidate image's migration command once# against the same DATABASE_URL before starting application replicas. The-# production Deploy workflow creates and verifies this one-shot Job for you.+# production Deploy workflow creates and verifies this one-shot Job for you;+# it captures a create-only Deployment recovery record before that Job and+# restores an exact template only after the old image proves read-only schema+# compatibility.npm run migratekubectl apply -f server/k8s/cumora-server.gke.yamlkubectl rollout status deployment/cumora-server@@ -267,10 +270,17 @@ kubectl logs agent-<id>- **Image upgrades** — tag both images with the same git sha;redeploy by updating both the server Deployment image and`CUMORA_AGENT_COMPUTER_IMAGE` to the same tag. The Deploy workflow runs one- candidate migration Job before mutating the Deployment.+ candidate migration Job before mutating the Deployment. Production resolves+ both to immutable `@sha256:` digests and uses a JSON Patch UID/template CAS;+ mutable tags, unknown image sources, and template drift are refused.- **PG schema changes** — append an immutable version and checksum; never editan applied migration. Use expand/contract changes that remain compatible with- the old server version still serving during the rolling-update window.+ the old server version still serving during the rolling-update window. A+ rollout timeout or smoke failure may recover only when the captured old image+ reports the current ledger inside its own supported range through the+ independent read-only verifier. Unknown or newer schema history requires a+ forward-compatible image or reviewed repair; it is never treated as proof+ that an arbitrary old image can start.- **Idle scheduler** — runs in-process on EACH server replica.That's fine because idle's tick currently publishes toCH_MESSAGE_NEW which SETNX-dedups; only one replica handlesdiff --git a/server/src/__integration__/deployment-schema-recovery.test.ts b/server/src/__integration__/deployment-schema-recovery.test.tsnew file mode 100644index 0000000..cc626c0--- /dev/null+++ b/server/src/__integration__/deployment-schema-recovery.test.ts@@ -0,0 +1,262 @@+import assert from 'node:assert/strict'+import { execFile as execFileCallback } from 'node:child_process'+import { access, mkdtemp, mkdir, rm, symlink, unlink, writeFile } from 'node:fs/promises'+import { fileURLToPath, pathToFileURL } from 'node:url'+import { dirname, join, resolve } from 'node:path'+import { tmpdir } from 'node:os'+import { after, before, test } from 'node:test'+import { promisify } from 'node:util'+import { pool } from '../db/pool.js'+import { MAX_SUPPORTED_SCHEMA_VERSION, MIN_SUPPORTED_SCHEMA_VERSION, SCHEMA_MIGRATIONS } from '../db/migrations/manifest.js'+import { ensureSchemaOnce, teardownAll } from './_helpers.js'+import {+ buildSchemaVerifierScript,+ parseSchemaVerifierOutput,+} from '../deploy/recovery.js'++const execFile = promisify(execFileCallback)+const repoRoot = resolve(dirname(fileURLToPath(import.meta.url)), '../../..')+const createdNodeModulesLinks = new Set<string>()++interface SchemaVerifierRun {+ status: string+ [key: string]: unknown+}++function databaseUrl(): string {+ const value = process.env.INTEGRATION_DATABASE_URL ?? process.env.DATABASE_URL+ assert.ok(value, 'integration database URL is required')+ return value+}++async function runVerifier(cwd = repoRoot, script = buildSchemaVerifierScript()): Promise<SchemaVerifierRun> {+ await ensureNodeModules(cwd)+ const env = {+ ...process.env,+ DATABASE_URL: databaseUrl(),+ NODE_ENV: process.env.NODE_ENV ?? 'test',+ OPENAI_API_KEY: process.env.OPENAI_API_KEY ?? 'integration-test-key',+ NODE_PATH: process.env.NODE_PATH ?? join(repoRoot, 'node_modules'),+ }+ try {+ const result = await execFile(+ process.execPath,+ ['--import', 'tsx', '--input-type=module', '-e', script],+ { cwd, env, maxBuffer: 1024 * 1024, timeout: 15_000 },+ )+ return parseSchemaVerifierOutput(result.stdout, result.stderr) as unknown as SchemaVerifierRun+ } catch (error) {+ const failed = error as { stdout?: string; stderr?: string }+ return parseSchemaVerifierOutput(failed.stdout ?? '', failed.stderr ?? '') as unknown as SchemaVerifierRun+ }+}++async function ensureNodeModules(cwd: string): Promise<void> {+ const target = join(cwd, 'node_modules')+ try {+ await access(target)+ return+ } catch {+ // CI has a real install. The fallback keeps this test runnable from a+ // clean worktree when the sibling checkout already owns the dependency+ // tree; it is removed by after() and never enters git.+ }+ const fallbacks = [+ process.env.CUMORA_TEST_NODE_MODULES,+ join(repoRoot, 'node_modules'),+ resolve(repoRoot, '..', 'cumora', 'node_modules'),+ ].filter((value): value is string => Boolean(value))+ for (const fallback of fallbacks) {+ if (fallback === target) continue+ try {+ await access(fallback)+ await symlink(fallback, target, 'dir')+ createdNodeModulesLinks.add(target)+ return+ } catch {+ // Try the next workspace-local dependency location.+ }+ }+ throw new Error('node_modules is required for the old-image verifier fixture')+}++async function makeReadOnlyVerifierImageLikeDir(): Promise<string> {+ const root = await mkdtemp(join(tmpdir(), 'cumora-old-image-schema-'))+ await ensureNodeModules(root)+ await writeFile(join(root, 'package.json'), '{"type":"module"}\n')+ const dbDir = join(root, 'server', 'src', 'db')+ const migrationsDir = join(dbDir, 'migrations')+ await mkdir(migrationsDir, { recursive: true })+ const realPool = pathToFileURL(resolve(repoRoot, 'server/src/db/pool.ts')).href+ const realManifest = pathToFileURL(resolve(repoRoot, 'server/src/db/migrations/manifest.ts')).href+ const realSchemaVersion = pathToFileURL(resolve(repoRoot, 'server/src/db/schema-version.ts')).href+ await writeFile(join(dbDir, 'pool.ts'), `export { pool } from ${JSON.stringify(realPool)}\n`)+ await writeFile(join(migrationsDir, 'manifest.ts'), `export * from ${JSON.stringify(realManifest)}\n`)+ await writeFile(join(dbDir, 'schema-version.ts'), `+import { verifySchemaCompatibility as verifyRealSchemaCompatibility } from ${JSON.stringify(realSchemaVersion)}++export async function verifySchemaCompatibility(client) {+ const { rows } = await client.query('SHOW transaction_read_only')+ if (rows[0]?.transaction_read_only !== 'on') throw new Error('transaction is not read-only')+ await client.query('SAVEPOINT read_only_probe')+ try {+ await client.query('CREATE TEMP TABLE cumora_verifier_read_only_probe (id integer)')+ throw new Error('read-only transaction accepted DDL')+ } catch (error) {+ if (error?.code !== '25006') throw error+ await client.query('ROLLBACK TO SAVEPOINT read_only_probe')+ }+ await client.query('RELEASE SAVEPOINT read_only_probe')+ return verifyRealSchemaCompatibility(client)+}+`)+ return root+}++async function withCommittedMutation<T>(+ mutate: (client: import('pg').PoolClient) => Promise<void>,+ restore: (client: import('pg').PoolClient) => Promise<void>,+ check: () => Promise<T>,+): Promise<T> {+ const client = await pool.connect()+ try {+ await client.query('BEGIN')+ await mutate(client)+ await client.query('COMMIT')+ try {+ return await check()+ } finally {+ await client.query('BEGIN')+ await restore(client)+ await client.query('COMMIT')+ }+ } catch (error) {+ await client.query('ROLLBACK').catch(() => {})+ throw error+ } finally {+ client.release()+ }+}++before(async () => { await ensureSchemaOnce() })+after(async () => {+ await teardownAll()+ for (const link of createdNodeModulesLinks) await unlink(link).catch(() => {})+ createdNodeModulesLinks.clear()+})++test('the actual inline verifier reads current PostgreSQL schema in read-only mode', async () => {+ const beforeRows = await pool.query(+ `SELECT version, name, checksum, execution_ms, applied_at+ FROM schema_migrations ORDER BY version`,+ )+ const result = await runVerifier()+ assert.equal(result.status, 'compatible', JSON.stringify(result))+ assert.equal(result.currentVersion, SCHEMA_MIGRATIONS.at(-1)?.version)+ assert.equal(result.minSupported, MIN_SUPPORTED_SCHEMA_VERSION)+ assert.equal(result.maxSupported, MAX_SUPPORTED_SCHEMA_VERSION)+ const afterRows = await pool.query(+ `SELECT version, name, checksum, execution_ms, applied_at+ FROM schema_migrations ORDER BY version`,+ )+ assert.deepEqual(afterRows.rows, beforeRows.rows, 'read-only verifier changed schema ledger')+})++test('the inline verifier enforces transaction READ ONLY through an old-image-like wrapper', async () => {+ const imageLikeDir = await makeReadOnlyVerifierImageLikeDir()+ try {+ const beforeRows = await pool.query(+ `SELECT version, name, checksum, execution_ms, applied_at+ FROM schema_migrations ORDER BY version`,+ )+ const result = await runVerifier(imageLikeDir)+ assert.equal(result.status, 'compatible', JSON.stringify(result))+ assert.equal(result.currentVersion, SCHEMA_MIGRATIONS.at(-1)?.version)+ const afterRows = await pool.query(+ `SELECT version, name, checksum, execution_ms, applied_at+ FROM schema_migrations ORDER BY version`,+ )+ assert.deepEqual(afterRows.rows, beforeRows.rows)+ const { rows } = await pool.query<{ regclass: string | null }>(+ `SELECT to_regclass('public.cumora_verifier_read_only_probe') AS regclass`,+ )+ assert.equal(rows[0]?.regclass, null, 'read-only verifier left a DDL probe table')+ } finally {+ await rm(imageLikeDir, { recursive: true, force: true })+ }+})++test('an old image without verifier modules is an explicit unknown result', async () => {+ const imageLikeDir = await mkdtemp(join(tmpdir(), 'cumora-old-image-missing-verifier-'))+ await ensureNodeModules(imageLikeDir)+ await writeFile(join(imageLikeDir, 'package.json'), '{"type":"module"}\n')+ try {+ const result = await runVerifier(imageLikeDir)+ assert.equal(result.status, 'unknown', JSON.stringify(result))+ assert.equal(result.reasonCode, 'module_unavailable')+ } finally {+ await rm(imageLikeDir, { recursive: true, force: true })+ }+})++test('the actual verifier reports future schema as known incompatible', async () => {+ const version = MAX_SUPPORTED_SCHEMA_VERSION + 1+ const result = await withCommittedMutation(+ async (client) => {+ await client.query(+ `INSERT INTO schema_migrations (version, name, checksum, execution_ms)+ VALUES ($1, $2, $3, 0)`,+ [version, `000${version}_test_future`, 'f'.repeat(64)],+ )+ },+ (client) => client.query('DELETE FROM schema_migrations WHERE version = $1', [version]).then(() => undefined),+ runVerifier,+ )+ assert.equal(result.status, 'incompatible', JSON.stringify(result))+ assert.equal(result.code, 'schema_ahead')+})++test('the actual verifier rejects corrupted and gapped immutable history', async () => {+ const original = await pool.query<{+ version: number+ name: string+ checksum: string+ execution_ms: number+ applied_at: Date+ }>(+ `SELECT version, name, checksum, execution_ms, applied_at+ FROM schema_migrations WHERE version = $1`,+ [MAX_SUPPORTED_SCHEMA_VERSION - 1],+ )+ assert.equal(original.rows.length, 1)+ const row = original.rows[0]!++ const corrupted = await withCommittedMutation(+ (client) => client.query(+ `UPDATE schema_migrations SET checksum = $2 WHERE version = $1`,+ [1, '0'.repeat(64)],+ ).then(() => undefined),+ (client) => client.query(+ `UPDATE schema_migrations SET checksum = $2 WHERE version = $1`,+ [1, SCHEMA_MIGRATIONS[0]!.checksum],+ ).then(() => undefined),+ runVerifier,+ )+ assert.equal(corrupted.status, 'incompatible', JSON.stringify(corrupted))+ assert.equal(corrupted.code, 'migration_history_invalid')++ const gapped = await withCommittedMutation(+ (client) => client.query(+ `DELETE FROM schema_migrations WHERE version = $1`,+ [row.version],+ ).then(() => undefined),+ (client) => client.query(+ `INSERT INTO schema_migrations (version, name, checksum, execution_ms, applied_at)+ VALUES ($1, $2, $3, $4, $5)`,+ [row.version, row.name, row.checksum, row.execution_ms, row.applied_at],+ ).then(() => undefined),+ runVerifier,+ )+ assert.equal(gapped.status, 'incompatible', JSON.stringify(gapped))+ assert.equal(gapped.code, 'migration_history_invalid')+})diff --git a/server/src/__integration__/ws-doc-authorization.test.ts b/server/src/__integration__/ws-doc-authorization.test.tsindex 572a8f0..1024e31 100644--- a/server/src/__integration__/ws-doc-authorization.test.ts+++ b/server/src/__integration__/ws-doc-authorization.test.ts@@ -89,11 +89,11 @@ async function waitForFrame(predicate: (frame: Frame) => boolean,label: string,timeoutMs = FRAME_TIMEOUT_MS,+ fromIndex = 0,): Promise<Frame> {- const start = harness.messages.lengthconst deadline = Date.now() + timeoutMswhile (Date.now() < deadline) {- const found = harness.messages.slice(start).find(predicate)+ const found = harness.messages.slice(fromIndex).find(predicate)if (found) return foundawait delay(5)}@@ -156,11 +156,14 @@ async function openSocket(userId: string): Promise<SocketHarness> {async function subscribeSocket(harness: SocketHarness, documentId: string, replaySupported = true, omitCapability = false): Promise<void> {const frame: Record<string, unknown> = { type: 'doc.subscribe', documentId }if (!omitCapability) frame.replaySupported = replaySupported+ const cursor = harness.messages.lengthharness.socket.send(JSON.stringify(frame))await waitForFrame(harness,(frame) => frame.type === 'doc.sync' && frame.documentId === documentId,`doc.sync for ${documentId}`,+ FRAME_TIMEOUT_MS,+ cursor,)}@@ -428,11 +431,18 @@ test('revocation committed before a retried authorization prevents the queued seawait beginUserLock(barrier, fixture.users[1]!)const afterOrigin = `after-revoke-${randomUUID()}`const targetBefore = target.messages.length+ const peerCursor = peer.messages.lengthconst fanout = emitAwareness(fixture, afterOrigin)await revokeMembership(fixture.users[1]!, fixture.companyId)await barrier.query('ROLLBACK')await fanout- await waitForFrame(peer, (frame) => frame.type === 'doc.awareness' && frame.originId === afterOrigin, 'valid peer after revoke')+ await waitForFrame(+ peer,+ (frame) => frame.type === 'doc.awareness' && frame.originId === afterOrigin,+ 'valid peer after revoke',+ FRAME_TIMEOUT_MS,+ peerCursor,+ )await delay(100)assert.equal(target.messages.slice(targetBefore).some((frame) => frame.originId === afterOrigin),@@ -534,11 +544,14 @@ test('close and unsubscribe during blocked hydration do not leave active subscri// first sync may be dropped legitimately: FIFO can run the queued// unsubscribe immediately after hydration, before the sync's outbound// authorization batch gets its turn.+ const syncCursor = unsubscribeSocket.messages.lengthunsubscribeSocket.socket.send(JSON.stringify({ type: 'doc.subscribe', documentId: unsubscribeFixture.documentId }))await waitForFrame(unsubscribeSocket,(frame) => frame.type === 'doc.sync' && frame.documentId === unsubscribeFixture.documentId,'second sync after queued unsubscribe',+ FRAME_TIMEOUT_MS,+ syncCursor,)unsubscribeSocket.socket.send(JSON.stringify({ type: 'doc.unsubscribe', documentId: unsubscribeFixture.documentId }))await delay(50)diff --git a/server/src/__tests__/deployment-recovery.test.ts b/server/src/__tests__/deployment-recovery.test.tsnew file mode 100644index 0000000..aa55b2f--- /dev/null+++ b/server/src/__tests__/deployment-recovery.test.ts@@ -0,0 +1,278 @@+import assert from 'node:assert/strict'+import { test } from 'node:test'+import {+ DEPLOYMENT_PROBE_CONTRACT,+ SCHEMA_VERIFIER_MARKER,+ assertBaselineUnchanged,+ buildCandidatePatch,+ buildCandidateTemplate,+ buildMigrationJob,+ buildRestorePatch,+ buildSchemaVerifierJob,+ buildSchemaVerifierScript,+ deploymentReceiptFromObject,+ extractDeploymentSnapshot,+ hashPodTemplate,+ parseSchemaVerifierOutput,+} from '../deploy/recovery.js'++const DIGEST_OLD = 'a'.repeat(64)+const DIGEST_NEW = 'b'.repeat(64)+const DIGEST_CANDIDATE = 'c'.repeat(64)+const DIGEST_PROXY = 'd'.repeat(64)+const DIGEST_MIGRATE = 'e'.repeat(64)++function deploymentFixture(): Record<string, unknown> {+ return {+ apiVersion: 'apps/v1',+ kind: 'Deployment',+ metadata: { name: 'cumora-server', uid: 'deployment-uid-1' },+ spec: {+ template: {+ metadata: { labels: { app: 'cumora-server' }, annotations: { trace: 'fixture' } },+ spec: {+ serviceAccountName: 'cumora-server',+ imagePullSecrets: [{ name: 'quay-pull' }],+ securityContext: {+ runAsNonRoot: true,+ seccompProfile: { type: 'RuntimeDefault' },+ },+ automountServiceAccountToken: true,+ volumes: [+ { name: 'proxy-cert', secret: { secretName: 'cloud-sql-cert' } },+ { name: 'scratch', emptyDir: {} },+ ],+ initContainers: [+ {+ name: 'migrate',+ image: `old-migrate@sha256:${DIGEST_MIGRATE}`,+ command: ['npm', 'run', 'migrate'],+ },+ ],+ containers: [+ {+ name: 'cloud-sql-proxy',+ image: `cloud-sql-proxy@sha256:${DIGEST_PROXY}`,+ args: ['--port=5432'],+ env: [{ name: 'PROXY_MODE', value: 'fixture' }],+ volumeMounts: [{ name: 'proxy-cert', mountPath: '/var/run/proxy', readOnly: true }],+ securityContext: {+ runAsNonRoot: true,+ allowPrivilegeEscalation: false,+ capabilities: { drop: ['ALL'] },+ },+ readinessProbe: { httpGet: { path: '/readiness', port: 9090 } },+ resources: { requests: { cpu: '50m', memory: '64Mi' } },+ },+ {+ name: 'server',+ image: `server@sha256:${DIGEST_OLD}`,+ envFrom: [{ secretRef: { name: 'cumora' } }],+ env: [{ name: 'CUMORA_AGENT_COMPUTER_IMAGE', value: `agent@sha256:${DIGEST_OLD}` }],+ volumeMounts: [{ name: 'scratch', mountPath: '/tmp/cumora' }],+ securityContext: { runAsUser: 1000, allowPrivilegeEscalation: false },+ startupProbe: DEPLOYMENT_PROBE_CONTRACT.startupProbe,+ readinessProbe: DEPLOYMENT_PROBE_CONTRACT.readinessProbe,+ livenessProbe: DEPLOYMENT_PROBE_CONTRACT.livenessProbe,+ resources: { requests: { cpu: '250m', memory: '512Mi' } },+ },+ ],+ },+ },+ },+ }+}++function snapshot() {+ return extractDeploymentSnapshot(deploymentFixture(), { capturedAt: '2026-09-10T00:00:00.000Z' })+}++function templateSpec(job: Record<string, any>): Record<string, any> {+ return job.spec.template.spec+}++test('captures immutable template and builds exact CAS candidate/restore patches', () => {+ const baseline = snapshot()+ const candidate = buildCandidateTemplate(baseline, {+ server: `server@sha256:${DIGEST_NEW}`,+ agent: `agent@sha256:${DIGEST_NEW}`,+ })+ const candidateRecord = candidate as any+ const baselineRecord = baseline.podTemplate as any+ const candidateServer = candidateRecord.spec.containers.find((container: any) => container.name === 'server')+ assert.equal(candidateServer.image, `server@sha256:${DIGEST_NEW}`)+ assert.equal(candidateServer.env.find((entry: any) => entry.name === 'CUMORA_AGENT_COMPUTER_IMAGE').value, `agent@sha256:${DIGEST_NEW}`)+ assert.deepEqual(candidateRecord.spec.volumes, baselineRecord.spec.volumes)+ assert.deepEqual(candidateRecord.spec.securityContext, baselineRecord.spec.securityContext)+ assert.deepEqual(candidateRecord.metadata, (baseline.podTemplate as any).metadata)+ assert.deepEqual(candidateRecord.spec.initContainers, [], 'retired migration init container must be removed')+ assert.deepEqual(candidateServer.startupProbe, DEPLOYMENT_PROBE_CONTRACT.startupProbe)+ assert.deepEqual(candidateServer.readinessProbe, DEPLOYMENT_PROBE_CONTRACT.readinessProbe)+ assert.deepEqual(candidateServer.livenessProbe, DEPLOYMENT_PROBE_CONTRACT.livenessProbe)++ const candidatePatch = buildCandidatePatch(baseline, { server: `server@sha256:${DIGEST_NEW}` })+ assert.deepEqual(candidatePatch[0], { op: 'test', path: '/metadata/uid', value: baseline.uid })+ assert.deepEqual(candidatePatch[1], { op: 'test', path: '/spec/template', value: baseline.podTemplate })+ assert.equal((candidatePatch[2] as any)?.op, 'replace')++ const candidateReceipt = deploymentReceiptFromObject({+ ...deploymentFixture(),+ spec: { template: candidate },+ }, { observedAt: '2026-09-10T00:01:00.000Z' })+ assert.equal(candidateReceipt.serverImage, `server@sha256:${DIGEST_NEW}`)+ assert.equal(candidateReceipt.agentImage, `agent@sha256:${DIGEST_NEW}`)+ assert.equal(candidateReceipt.podTemplateHash, hashPodTemplate(candidate))+ const restorePatch = buildRestorePatch(baseline, candidateReceipt, candidateReceipt.podTemplateHash)+ assert.deepEqual(restorePatch[0], { op: 'test', path: '/metadata/uid', value: baseline.uid })+ assert.deepEqual(restorePatch[1], { op: 'test', path: '/spec/template', value: candidate })+ assert.deepEqual(restorePatch[2], { op: 'replace', path: '/spec/template', value: baseline.podTemplate })+ assert.equal(hashPodTemplate(baseline.podTemplate), baseline.podTemplateHash)++ assertBaselineUnchanged(baseline, extractDeploymentSnapshot(deploymentFixture()))+ assert.throws(+ () => buildRestorePatch(baseline, { ...candidateReceipt, uid: 'other-uid' }, candidateReceipt.podTemplateHash),+ /UID changed/,+ )+ assert.throws(+ () => buildRestorePatch(baseline, candidateReceipt, 'wrong-candidate-hash'),+ /template drifted/,+ )++ const mutableDeployment = deploymentFixture()+ ;(mutableDeployment.spec as any).template.spec.containers[1].image = 'server:latest'+ assert.throws(+ () => extractDeploymentSnapshot(mutableDeployment),+ /digest|immutable|automatic recovery/i,+ )++ const missingServerImage = deploymentFixture()+ delete (missingServerImage.spec as any).template.spec.containers[1].image+ assert.throws(() => extractDeploymentSnapshot(missingServerImage), /server container image/)++ const valueFromServerImage = deploymentFixture()+ ;(valueFromServerImage.spec as any).template.spec.containers[1].image = { valueFrom: { configMapKeyRef: { name: 'image' } } }+ assert.throws(() => extractDeploymentSnapshot(valueFromServerImage), /server container image/)++ const shortDigestServerImage = deploymentFixture()+ ;(shortDigestServerImage.spec as any).template.spec.containers[1].image = 'server@sha256:shorthex'+ assert.throws(() => extractDeploymentSnapshot(shortDigestServerImage), /sha256 digest/)++ const mutableAgentImage = deploymentFixture()+ ;(mutableAgentImage.spec as any).template.spec.containers[1].env = [+ { name: 'CUMORA_AGENT_COMPUTER_IMAGE', value: 'agent:latest' },+ ]+ assert.throws(() => extractDeploymentSnapshot(mutableAgentImage), /agent container image|sha256 digest/)++ const valueFromAgentImage = deploymentFixture()+ ;(valueFromAgentImage.spec as any).template.spec.containers[1].env = [+ { name: 'CUMORA_AGENT_COMPUTER_IMAGE', valueFrom: { secretKeyRef: { name: 'agent-image' } } },+ ]+ const valueFromAgentBaseline = extractDeploymentSnapshot(valueFromAgentImage)+ const valueFromAgentReceipt = deploymentReceiptFromObject(valueFromAgentImage)+ assert.throws(+ () => buildRestorePatch(valueFromAgentBaseline, valueFromAgentReceipt, valueFromAgentReceipt.podTemplateHash),+ /agent.*image|valueFrom|automatic recovery/i,+ )+})++test('migration and verifier Jobs preserve pod identity, mounts, volumes, and security contexts', () => {+ const baseline = snapshot()+ const migration = buildMigrationJob(baseline, {+ name: 'cumora-migrate-test',+ image: `server@sha256:${DIGEST_CANDIDATE}`,+ repairMode: 'off',+ })+ const verifier = buildSchemaVerifierJob(baseline, {+ name: 'cumora-verify-test',+ image: baseline.serverImage,+ })+ assert.throws(+ () => buildSchemaVerifierJob(baseline, { name: 'cumora-verify-candidate', image: `server@sha256:${DIGEST_CANDIDATE}` }),+ /baseline.*image|verifier.*image|old image/i,+ )++ for (const job of [migration, verifier]) {+ const pod = templateSpec(job)+ const baselineSpec = baseline.podTemplate as any+ assert.equal(pod.serviceAccountName, 'cumora-server')+ assert.deepEqual(pod.imagePullSecrets, baselineSpec.spec.imagePullSecrets)+ assert.deepEqual(pod.securityContext, baselineSpec.spec.securityContext)+ assert.deepEqual(pod.volumes, baselineSpec.spec.volumes)+ assert.equal(pod.restartPolicy, 'Never')+ const proxy = pod.initContainers.find((container: any) => container.name === 'cloud-sql-proxy')+ assert.ok(proxy)+ assert.deepEqual(proxy.volumeMounts, baselineSpec.spec.containers[0].volumeMounts)+ assert.deepEqual(proxy.securityContext, baselineSpec.spec.containers[0].securityContext)+ assert.equal(proxy.restartPolicy, 'Always')+ assert.ok(proxy.startupProbe)+ assert.ok(!pod.initContainers.some((container: any) => container.name === 'migrate'))+ }++ const migrationServer = templateSpec(migration).containers[0]+ assert.equal(migrationServer.name, 'migrate')+ assert.deepEqual(migrationServer.envFrom, [{ secretRef: { name: 'cumora' } }])+ assert.deepEqual(migrationServer.command, ['npm', 'run', 'migrate'])+ assert.equal(migrationServer.env.find((entry: any) => entry.name === 'MIGRATION_0002_REPAIR').value, 'off')++ const verifierServer = templateSpec(verifier).containers[0]+ assert.equal(verifierServer.name, 'migrate')+ assert.deepEqual(verifierServer.envFrom, [{ secretRef: { name: 'cumora' } }])+ assert.equal(verifierServer.workingDir, '/app')+ assert.equal((verifier.spec as { activeDeadlineSeconds?: number }).activeDeadlineSeconds, 180)+ assert.deepEqual(verifierServer.command.slice(0, 5), ['node', '--import', 'tsx', '--input-type=module', '-e'])+ assert.match(verifierServer.args?.[0] ?? '', /BEGIN READ ONLY/)+})++test('verifier script is old-image-local, read-only, bounded, and uses strict protocol output', () => {+ const script = buildSchemaVerifierScript()+ assert.match(script, /\.\/server\/src\/db\/pool\.ts/)+ assert.match(script, /\.\/server\/src\/db\/schema-version\.ts/)+ assert.match(script, /BEGIN READ ONLY/)+ assert.match(script, /COMMIT/)+ assert.match(script, /pool\.end\(\)/)+ assert.doesNotMatch(script, /\/app\/server\/src\/db/)++ const compatible = parseSchemaVerifierOutput(+ `${SCHEMA_VERIFIER_MARKER} ${JSON.stringify({+ protocol: 1,+ kind: 'schema-verifier',+ status: 'compatible',+ currentVersion: 7,+ minSupported: 7,+ maxSupported: 7,+ })}\n`,+ ) as unknown as Record<string, unknown>+ assert.equal(compatible.status, 'compatible')+ assert.equal(compatible.currentVersion, 7)+ assert.equal(compatible.minSupported, 7)+ assert.equal(compatible.maxSupported, 7)++ const incompatible = parseSchemaVerifierOutput(+ `${SCHEMA_VERIFIER_MARKER} ${JSON.stringify({+ protocol: 1, kind: 'schema-verifier', status: 'incompatible', currentVersion: null,+ minSupported: null, maxSupported: null, code: 'schema_ahead',+ })}\n`,+ ) as unknown as Record<string, unknown>+ assert.equal(incompatible.status, 'incompatible')+ assert.equal(incompatible.code, 'schema_ahead')++ const unknown = parseSchemaVerifierOutput(+ `${SCHEMA_VERIFIER_MARKER} ${JSON.stringify({+ protocol: 1, kind: 'schema-verifier', status: 'unknown', currentVersion: null,+ minSupported: null, maxSupported: null, reasonCode: 'connection_failed',+ })}\n`,+ ) as unknown as Record<string, unknown>+ assert.equal(unknown.status, 'unknown')+ assert.equal(unknown.reasonCode, 'connection_failed')++ for (const output of [+ '',+ `${SCHEMA_VERIFIER_MARKER} {"protocol":1,"kind":"schema-verifier","status":"compatible","currentVersion":7}\nextra\n`,+ `${SCHEMA_VERIFIER_MARKER} {"protocol":1,"kind":"schema-verifier","status":"compatible","currentVersion":0,"minSupported":7,"maxSupported":7}\n`,+ `${SCHEMA_VERIFIER_MARKER} {"protocol":1,"kind":"schema-verifier","status":"unknown"}\n`,+ `${SCHEMA_VERIFIER_MARKER} {bad-json}\n`,+ `${SCHEMA_VERIFIER_MARKER} {"protocol":1,"kind":"schema-verifier","status":"compatible","currentVersion":7,"minSupported":7,"maxSupported":7}\n${SCHEMA_VERIFIER_MARKER} {"protocol":1,"kind":"schema-verifier","status":"compatible","currentVersion":7,"minSupported":7,"maxSupported":7}\n`,+ ]) {+ assert.equal(parseSchemaVerifierOutput(output).status, 'unknown')+ }+})diff --git a/server/src/__tests__/deployment-release-flow.test.ts b/server/src/__tests__/deployment-release-flow.test.tsnew file mode 100644index 0000000..d9a4d48--- /dev/null+++ b/server/src/__tests__/deployment-release-flow.test.ts@@ -0,0 +1,841 @@+import assert from 'node:assert/strict'+import { createHash } from 'node:crypto'+import { createServer, type Server } from 'node:http'+import { chmod, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'+import { once } from 'node:events'+import { tmpdir } from 'node:os'+import { join, dirname, resolve } from 'node:path'+import { fileURLToPath } from 'node:url'+import { spawn } from 'node:child_process'+import { test } from 'node:test'++const testFileDir = dirname(fileURLToPath(import.meta.url))+const repoRoot = resolve(testFileDir, '../../..')+const fakeKubectl = resolve(testFileDir, 'fixtures/deployment-fake-kubectl.mjs')++const marker = 'FAKE_SECRET_MARKER'+const companyId = 'company-flow-fixture'+const serverOld = `server@sha256:${'1'.repeat(64)}`+const serverNew = `server@sha256:${'2'.repeat(64)}`+const agentOld = `agent@sha256:${'4'.repeat(64)}`+const agentNew = `agent@sha256:${'5'.repeat(64)}`++const startupProbe = {+ httpGet: { path: '/api/livez', port: 'http' },+ periodSeconds: 5,+ timeoutSeconds: 2,+ failureThreshold: 60,+}+const readinessProbe = {+ httpGet: { path: '/api/health', port: 'http' },+ periodSeconds: 5,+ timeoutSeconds: 2,+}+const livenessProbe = {+ httpGet: { path: '/api/livez', port: 'http' },+ periodSeconds: 30,+ timeoutSeconds: 3,+}++function clone<T>(value: T): T {+ return JSON.parse(JSON.stringify(value)) as T+}++function baselineDeployment(): Record<string, unknown> {+ return {+ apiVersion: 'apps/v1',+ kind: 'Deployment',+ metadata: {+ name: 'cumora-server',+ namespace: 'default',+ uid: 'deployment-flow-uid',+ generation: 12,+ annotations: { 'fixture.cumora.ai/secret': marker },+ },+ spec: {+ replicas: 2,+ template: {+ metadata: {+ labels: { app: 'cumora-server' },+ annotations: { 'fixture.cumora.ai/template': marker },+ },+ spec: {+ serviceAccountName: 'cumora-server',+ automountServiceAccountToken: true,+ imagePullSecrets: [{ name: 'fixture-pull' }],+ securityContext: { runAsNonRoot: true, seccompProfile: { type: 'RuntimeDefault' } },+ volumes: [+ { name: 'proxy-cert', secret: { secretName: 'cloud-sql-cert' } },+ { name: 'scratch', emptyDir: {} },+ ],+ initContainers: [{ name: 'migrate', image: `migrate@sha256:${'6'.repeat(64)}` }],+ containers: [+ {+ name: 'cloud-sql-proxy',+ image: `proxy@sha256:${'7'.repeat(64)}`,+ args: ['--port=5432'],+ env: [{ name: 'PROXY_MODE', value: 'fixture' }],+ volumeMounts: [{ name: 'proxy-cert', mountPath: '/var/run/proxy', readOnly: true }],+ securityContext: { runAsNonRoot: true, allowPrivilegeEscalation: false, capabilities: { drop: ['ALL'] } },+ startupProbe: { httpGet: { path: '/readiness', port: 9090 }, periodSeconds: 1, failureThreshold: 60 },+ readinessProbe: { httpGet: { path: '/readiness', port: 9090 } },+ },+ {+ name: 'server',+ image: serverOld,+ envFrom: [{ secretRef: { name: 'cumora' } }],+ env: [{ name: 'CUMORA_AGENT_COMPUTER_IMAGE', value: agentOld }],+ volumeMounts: [{ name: 'scratch', mountPath: '/tmp/cumora' }],+ securityContext: { runAsUser: 1000, allowPrivilegeEscalation: false },+ startupProbe,+ readinessProbe,+ livenessProbe,+ resources: { requests: { cpu: '250m', memory: '512Mi' } },+ },+ ],+ },+ },+ },+ status: {+ observedGeneration: 12,+ updatedReplicas: 2,+ readyReplicas: 2,+ availableReplicas: 2,+ },+ }+}++type ScenarioConfig = Record<string, unknown> & {+ smoke?: Record<string, boolean>+}++function makeState(config: ScenarioConfig = {}, deployment = baselineDeployment()): Record<string, any> {+ const state = {+ config: {+ baselineServerImage: serverOld,+ secretMarker: marker,+ emitSecretMarker: true,+ ...config,+ },+ initialDeployment: clone(deployment),+ deployment: clone(deployment),+ phase: 'baseline',+ secrets: {},+ jobs: {},+ history: { commands: [], patches: [], rollouts: [], appliedJobs: [] },+ }+ if (config.baselineUnhealthy) {+ const deployment = state.deployment as any+ deployment.status = {+ ...deployment.status,+ observedGeneration: deployment.metadata.generation - 1,+ updatedReplicas: 0,+ readyReplicas: 0,+ availableReplicas: 0,+ }+ }+ return state+}++function serverContainer(state: Record<string, any>) {+ return state.deployment.spec.template.spec.containers.find((container: any) => container.name === 'server')+}++function stateHash(value: unknown): string {+ return createHash('sha256').update(JSON.stringify(sortKeys(value))).digest('hex')+}++function sortKeys(value: unknown): unknown {+ if (Array.isArray(value)) return value.map(sortKeys)+ if (!value || typeof value !== 'object') return value+ return Object.fromEntries(Object.entries(value).sort(([left], [right]) => left.localeCompare(right)).map(([key, child]) => [key, sortKeys(child)]))+}++function decodeSecret(state: Record<string, any>, secretName: string, key: string): Record<string, any> {+ const encoded = state.secrets?.[secretName]?.data?.[key]+ assert.equal(typeof encoded, 'string', `${secretName} must contain ${key}`)+ return JSON.parse(Buffer.from(encoded, 'base64').toString('utf8')) as Record<string, any>+}++async function startSmokeServer(statePath: string, config: ScenarioConfig): Promise<{ server: Server; base: string }> {+ let lastSmokePhase: string | null = null+ let lastSmokeRequest = ''+ const server = createServer(async (request, response) => {+ const state = JSON.parse(await readFile(statePath, 'utf8')) as Record<string, any>+ const phase = String(state.phase || 'baseline')+ const pathname = (request.url || '/').split('?')[0]+ const shouldPass = config.smoke?.[phase] !== false+ const shippingRequired = String(config.shippingRequired ?? 'Y') !== 'N'+ const terminalPath = shippingRequired ? '/api/shipping/overview' : '/api/conversations'+ const shipping404Phases = Array.isArray(config.shipping404Phases) ? config.shipping404Phases.map(String) : []+ const shipping404 = pathname === '/api/shipping/overview' && shipping404Phases.includes(phase)+ if (shipping404) {+ response.statusCode = 404+ response.setHeader('content-type', 'text/plain')+ response.end(`${marker}: shipping fixture unavailable during ${phase}`)+ } else if (shouldPass) {+ response.statusCode = 200+ response.setHeader('content-type', 'application/json')+ if (pathname === '/api/health') response.end(JSON.stringify({ ok: true }))+ else if (pathname === '/api/auth/me') response.end(JSON.stringify({ companies: [{ id: companyId }] }))+ else if (pathname === '/api/conversations') response.end(JSON.stringify([]))+ else if (pathname === '/api/shipping/overview') response.end(JSON.stringify({ features: [], friction: [], dueReadbacks: [] }))+ else response.end(JSON.stringify({ ok: true }))+ } else {+ response.statusCode = 503+ response.setHeader('content-type', 'text/plain')+ response.end(`${marker}: smoke fixture rejected ${phase}`)+ }+ lastSmokePhase = phase+ lastSmokeRequest = pathname+ if (config.driftAfterSmoke === phase && pathname === terminalPath && !state.driftAfterSmokeApplied) {+ state.driftAfterSmokeApplied = true+ state.deployment.spec.template.metadata.annotations = {+ ...(state.deployment.spec.template.metadata.annotations || {}),+ 'fake.cumora.ai/post-smoke-drift': marker,+ }+ await writeFile(statePath, `${JSON.stringify(state, null, 2)}\n`, { mode: 0o600 })+ }+ })+ server.listen(0, '127.0.0.1')+ await once(server, 'listening')+ const address = server.address()+ assert.ok(address && typeof address === 'object')+ assert.equal(lastSmokePhase, null)+ assert.equal(lastSmokeRequest, '')+ return { server, base: `http://127.0.0.1:${address.port}` }+}++interface RunResult {+ code: number | null+ signal: NodeJS.Signals | null+ stdout: string+ stderr: string+ state: Record<string, any>+}++let runCounter = 0++async function runScenario(config: ScenarioConfig = {}, deployment = baselineDeployment()): Promise<RunResult> {+ await chmod(fakeKubectl, 0o755)+ const root = await mkdtemp(join(tmpdir(), 'cumora-deployment-release-flow-'))+ const statePath = join(root, 'fake-kubectl-state.json')+ const state = makeState(config, deployment)+ await writeFile(statePath, `${JSON.stringify(state, null, 2)}\n`, { mode: 0o600 })+ const smoke = await startSmokeServer(statePath, config)+ const runId = `flow-${process.pid}-${runCounter++}`+ const env: NodeJS.ProcessEnv = {+ ...process.env,+ KUBECTL: fakeKubectl,+ FAKE_KUBECTL_STATE: statePath,+ DEPLOYMENT: 'cumora-server',+ DEPLOYMENT_NAMESPACE: 'default',+ RECOVERY_SECRET: 'flow-baseline-secret',+ RECEIPT_SECRET: 'flow-receipt-secret',+ RECOVERY_WORKDIR: root,+ RUNNER_TEMP: root,+ GITHUB_WORKSPACE: repoRoot,+ GITHUB_RUN_ID: runId,+ GITHUB_SHA: 'abcdef1234567890',+ CANDIDATE_SERVER_IMAGE: serverNew,+ CANDIDATE_AGENT_IMAGE: agentNew,+ INCLUDE_AGENT: 'Y',+ MIGRATION_JOB_NAME: `flow-migrate-${runId}`,+ VERIFIER_JOB_NAME: `flow-verifier-${runId}`,+ RECOVERY_POLL_MS: '1',+ MIGRATION_POLL_ATTEMPTS: '1',+ VERIFIER_POLL_ATTEMPTS: '1',+ ROLLOUT_TIMEOUT_SECONDS: '1',+ SMOKE_TIMEOUT_SECONDS: '5',+ CUMORA_SMOKE_TOKEN: 'dummy-flow-token',+ CUMORA_SMOKE_COMPANY_ID: companyId,+ CUMORA_SMOKE_BASE: smoke.base,+ CUMORA_SMOKE_REQUIRE_SHIPPING: String(config.shippingRequired ?? 'Y'),+ }+ if (config.recoveryMode !== undefined) env.RECOVERY_MODE = String(config.recoveryMode)+ else delete env.RECOVERY_MODE+ delete env.SMOKE_COMMAND++ let child: ReturnType<typeof spawn> | undefined+ let result: RunResult+ try {+ result = await new Promise<RunResult>((resolveResult) => {+ child = spawn(process.execPath, ['--import', 'tsx', 'scripts/deploy-release.mjs', 'run'], {+ cwd: repoRoot,+ env,+ stdio: ['ignore', 'pipe', 'pipe'],+ })+ let stdout = ''+ let stderr = ''+ child.stdout?.setEncoding('utf8')+ child.stderr?.setEncoding('utf8')+ child.stdout?.on('data', (chunk: string) => { stdout += chunk })+ child.stderr?.on('data', (chunk: string) => { stderr += chunk })+ const timer = setTimeout(() => child?.kill('SIGTERM'), 15_000)+ child.on('close', async (code, signal) => {+ clearTimeout(timer)+ const finalState = JSON.parse(await readFile(statePath, 'utf8')) as Record<string, any>+ resolveResult({ code, signal, stdout, stderr, state: finalState })+ })+ })+ } finally {+ if (child && child.exitCode === null) child.kill('SIGTERM')+ await new Promise<void>((resolveClose) => smoke.server.close(() => resolveClose()))+ await rm(root, { recursive: true, force: true })+ }+ return result+}++function assertNoSecretLeak(result: Pick<RunResult, 'stdout' | 'stderr'>) {+ assert.doesNotMatch(result.stdout, new RegExp(marker))+ assert.doesNotMatch(result.stderr, new RegExp(marker))+}++test('real deploy-release entrypoint completes candidate and stores API PATCH response receipt', async () => {+ const result = await runScenario({ defaultCandidatePatchResponse: true, smoke: { baseline: true, candidate: true, restored: true } })+ assert.equal(result.code, 0, result.stderr)+ assert.match(result.stdout, /"status":"deployed"/)+ assertNoSecretLeak(result)+ assert.equal(result.state.phase, 'candidate')+ assert.equal(serverContainer(result.state).image, serverNew)+ assert.equal(serverContainer(result.state).env.find((entry: any) => entry.name === 'CUMORA_AGENT_COMPUTER_IMAGE').value, agentNew)+ assert.equal(result.state.history.patches.length, 1)+ assert.equal(result.state.history.rollouts.length, 1)+ assert.deepEqual(result.state.history.rollouts[0], { phase: 'candidate', setting: 'success' })+ const receipt = decodeSecret(result.state, 'flow-receipt-secret', 'receipt.json')+ assert.equal(receipt.serverImage, serverNew)+ assert.equal(receipt.podTemplate.metadata.annotations['fake.cumora.ai/writer'], 'api-default')+ assert.equal(receipt.podTemplateHash, stateHash(result.state.deployment.spec.template))+ const baseline = decodeSecret(result.state, 'flow-baseline-secret', 'baseline.json')+ assert.equal(baseline.serverImage, serverOld)+ assert.deepEqual(receipt.podTemplate.spec.volumes, baseline.podTemplate.spec.volumes)+ assert.deepEqual(receipt.podTemplate.spec.securityContext, baseline.podTemplate.spec.securityContext)+ assert.deepEqual(receipt.podTemplate.spec.containers.find((container: any) => container.name === 'server').envFrom, baseline.podTemplate.spec.containers.find((container: any) => container.name === 'server').envFrom)+})++test('candidate smoke failure restores exact baseline and leaves process failed', async () => {+ const result = await runScenario({ smoke: { baseline: true, candidate: false, restored: true } })+ assert.notEqual(result.code, 0)+ assert.match(result.stderr, /candidate_failed_recovered|recovery_failed/)+ assertNoSecretLeak(result)+ assert.equal(result.state.phase, 'restored')+ assert.equal(serverContainer(result.state).image, serverOld)+ assert.equal(serverContainer(result.state).env.find((entry: any) => entry.name === 'CUMORA_AGENT_COMPUTER_IMAGE').value, agentOld)+ assert.equal(result.state.history.patches.length, 2)+ assert.deepEqual(result.state.history.rollouts.map((entry: any) => entry.phase), ['candidate', 'restored'])+ const baseline = decodeSecret(result.state, 'flow-baseline-secret', 'baseline.json')+ const restoredTemplateHash = stateHash(result.state.deployment.spec.template)+ assert.equal(restoredTemplateHash, baseline.podTemplateHash)+})++test('rollout timeout follows the same guarded recovery path', async () => {+ const result = await runScenario({ candidateRollout: 'timeout', smoke: { baseline: true, candidate: true, restored: true } })+ assert.notEqual(result.code, 0)+ assert.match(result.stderr, /candidate_failed_recovered|recovery_failed/)+ assertNoSecretLeak(result)+ assert.equal(result.state.phase, 'restored')+ assert.equal(serverContainer(result.state).image, serverOld)+ assert.equal(result.state.history.patches.length, 2)+})++for (const [name, failureMessage] of [+ ['kubectl timed-out condition', 'error: timed out waiting for the condition'],+ ['deployment progress deadline', 'deployment exceeded its progress deadline'],+] as const) {+ test(`${name} is a nonzero rollout failure with guarded recovery`, async () => {+ const result = await runScenario({+ candidateRollout: 'failure',+ rolloutFailureMessage: failureMessage,+ smoke: { baseline: true, candidate: true, restored: true },+ })+ assert.notEqual(result.code, 0)+ assert.match(result.stderr, /candidate_failed_recovered|recovery_failed/)+ assertNoSecretLeak(result)+ assert.equal(result.state.history.rollouts[0].failureMessage, failureMessage)+ assert.equal(result.state.history.patches.length, 2)+ assert.equal(result.state.phase, 'restored')+ assert.equal(serverContainer(result.state).image, serverOld)+ })+}++const verifierCases: Array<[string, ScenarioConfig, RegExp]> = [+ ['schema_ahead', { verifierResult: 'schema_ahead' }, /schema_incompatible/],+ ['incompatible', { verifierResult: 'schema_behind' }, /schema_incompatible/],+ ['unknown', { verifierResult: 'unknown' }, /schema_unknown/],+ ['malformed', { verifierResult: 'malformed' }, /schema_unknown/],+ ['missing', { verifierStatus: 'missing' }, /job_timeout/],+ ['failed', { verifierStatus: 'failed' }, /job_failed/],+ ['timedout', { verifierStatus: 'timedout' }, /job_timeout/],+]++for (const [name, verifierConfig, expectedError] of verifierCases) {+ test(`verifier ${name} never restores a failed candidate`, async () => {+ const result = await runScenario({+ ...verifierConfig,+ smoke: { baseline: true, candidate: false, restored: true },+ })+ assert.notEqual(result.code, 0)+ assert.match(result.stderr, expectedError)+ assertNoSecretLeak(result)+ assert.equal(result.state.history.patches.length, 1)+ assert.equal(result.state.phase, 'candidate')+ assert.equal(serverContainer(result.state).image, serverNew)+ })+}++test('CAS drift and UID changes prevent automatic recovery', async () => {+ for (const drift of ['drift-template', 'drift-uid']) {+ const result = await runScenario({+ candidateRollout: drift,+ smoke: { baseline: true, candidate: true, restored: true },+ })+ assert.notEqual(result.code, 0, drift)+ assertNoSecretLeak(result)+ assert.equal(result.state.history.patches.length, 1, drift)+ assert.equal(serverContainer(result.state).image, serverNew, drift)+ }+})++test('a progress-deadline rollout failure enters the same guarded recovery path', async () => {+ const result = await runScenario({+ candidateRollout: 'progressing',+ // The endpoint can still be served by old Pods while the candidate is+ // progressing. The rollout failure still enters the full CAS/verifier+ // recovery path; the release remains nonzero.+ smoke: { baseline: true, candidate: true, restored: true },+ })+ assert.notEqual(result.code, 0)+ assertNoSecretLeak(result)+ assert.equal(result.state.history.patches.length, 2)+ assert.equal(result.state.phase, 'restored')+ assert.equal(serverContainer(result.state).image, serverOld)+})++test('baseline and restore tolerate Shipping 404 while candidate smoke keeps Shipping strict', async () => {+ const deployed = await runScenario({+ shipping404Phases: ['baseline', 'restored'],+ smoke: { baseline: true, candidate: true, restored: true },+ })+ assert.equal(deployed.code, 0, deployed.stderr)+ assertNoSecretLeak(deployed)+ assert.equal(deployed.state.phase, 'candidate')+ assert.equal(deployed.state.history.patches.length, 1)+ assert.equal(serverContainer(deployed.state).image, serverNew)++ const recovered = await runScenario({+ shipping404Phases: ['baseline', 'restored'],+ smoke: { baseline: true, candidate: false, restored: true },+ })+ assert.notEqual(recovered.code, 0)+ assert.match(recovered.stderr, /candidate_failed_recovered|recovery_failed/)+ assertNoSecretLeak(recovered)+ assert.equal(recovered.state.phase, 'restored')+ assert.equal(recovered.state.history.patches.length, 2)+ assert.equal(serverContainer(recovered.state).image, serverOld)+})++test('forward-only can publish with an unhealthy baseline but never restores a failed candidate', async () => {+ const normal = await runScenario({+ baselineUnhealthy: true,+ smoke: { baseline: true, candidate: true, restored: true },+ })+ assert.notEqual(normal.code, 0)+ assertNoSecretLeak(normal)+ assert.equal(normal.state.history.patches.length, 0)++ const forwardOnlySuccess = await runScenario({+ baselineUnhealthy: true,+ recoveryMode: 'forward-only',+ smoke: { baseline: true, candidate: true, restored: true },+ })+ assert.equal(forwardOnlySuccess.code, 0, forwardOnlySuccess.stderr)+ assertNoSecretLeak(forwardOnlySuccess)+ assert.equal(forwardOnlySuccess.state.history.patches.length, 1)+ assert.equal(forwardOnlySuccess.state.phase, 'candidate')+ assert.equal(serverContainer(forwardOnlySuccess.state).image, serverNew)++ const forwardOnlyFailure = await runScenario({+ baselineUnhealthy: true,+ recoveryMode: 'forward-only',+ smoke: { baseline: true, candidate: false, restored: true },+ })+ assert.notEqual(forwardOnlyFailure.code, 0)+ assert.match(forwardOnlyFailure.stderr, /candidate_failed_forward_only/)+ assertNoSecretLeak(forwardOnlyFailure)+ assert.equal(forwardOnlyFailure.state.history.patches.length, 1)+ assert.equal(forwardOnlyFailure.state.phase, 'candidate')+ assert.equal(serverContainer(forwardOnlyFailure.state).image, serverNew)+})++test('mutable baseline is rejected before any migration or Deployment patch', async () => {+ const deployment = baselineDeployment()+ ;(deployment.spec as any).template.spec.containers.find((container: any) => container.name === 'server').image = 'server:latest'+ const result = await runScenario({ smoke: { baseline: true, candidate: true, restored: true } }, deployment)+ assert.notEqual(result.code, 0)+ assert.match(result.stderr, /failed|digest|baseline/i)+ assertNoSecretLeak(result)+ assert.equal(result.state.history.patches.length, 0)+ assert.equal(result.state.history.appliedJobs.length, 0)+})++test('unknown agent baseline is not eligible for automatic restore', async () => {+ const deployment = baselineDeployment()+ const server = (deployment.spec as any).template.spec.containers.find((container: any) => container.name === 'server')+ server.env = [{ name: 'CUMORA_AGENT_COMPUTER_IMAGE', valueFrom: { secretKeyRef: { name: 'agent-image' } } }]+ const result = await runScenario({ smoke: { baseline: true, candidate: false, restored: true } }, deployment)+ assert.notEqual(result.code, 0)+ assertNoSecretLeak(result)+ assert.equal(result.state.history.patches.length, 1)+ assert.equal(serverContainer(result.state).image, serverNew)+})++for (const restoreFailure of ['restorePatchFailure', 'restoreRollout', 'restoreSmoke'] as const) {+ test(`${restoreFailure} keeps release nonzero and preserves protected baseline`, async () => {+ const config: ScenarioConfig = {+ smoke: { baseline: true, candidate: false, restored: restoreFailure !== 'restoreSmoke' },+ }+ if (restoreFailure === 'restorePatchFailure') config.restorePatchFailure = 'forbidden restore patch'+ if (restoreFailure === 'restoreRollout') config.restoreRollout = 'timeout'+ if (restoreFailure === 'restoreSmoke') config.restoreSmoke = 'fail'+ const result = await runScenario(config)+ assert.notEqual(result.code, 0)+ assertNoSecretLeak(result)+ const baseline = decodeSecret(result.state, 'flow-baseline-secret', 'baseline.json')+ assert.equal(baseline.serverImage, serverOld)+ assert.equal(result.state.history.patches.length, restoreFailure === 'restorePatchFailure' ? 1 : 2)+ if (restoreFailure === 'restorePatchFailure') assert.equal(serverContainer(result.state).image, serverNew)+ if (restoreFailure === 'restoreRollout') assert.equal(serverContainer(result.state).image, serverOld)+ })+}++test('migration failure creates the baseline record but never patches the Deployment', async () => {+ const result = await runScenario({ migrationStatus: 'failed', smoke: { baseline: true, candidate: true, restored: true } })+ assert.notEqual(result.code, 0)+ assert.match(result.stderr, /job_failed/)+ assertNoSecretLeak(result)+ assert.equal(result.state.history.patches.length, 0)+ assert.equal(result.state.history.appliedJobs.length, 1)+ const baseline = decodeSecret(result.state, 'flow-baseline-secret', 'baseline.json')+ assert.equal(baseline.serverImage, serverOld)+})++test('an active migration Job from another run blocks a new migration writer', async () => {+ const state = makeState({ smoke: { baseline: true, candidate: true, restored: true } })+ state.jobs['old-run-migration'] = {+ apiVersion: 'batch/v1',+ kind: 'Job',+ metadata: { name: 'old-run-migration', labels: { app: 'cumora-schema-migration' } },+ status: { active: 1 },+ }+ const result = await runScenarioWithState(state)+ assert.notEqual(result.code, 0)+ assert.match(result.stderr, /migration_residual_job_present/)+ assertNoSecretLeak(result)+ assert.equal(result.state.history.patches.length, 0)+ assert.equal(result.state.history.appliedJobs.length, 0)+})++test('a residual Job with active Pods is blocking even when succeeded is nonzero', async () => {+ const state = makeState({ smoke: { baseline: true, candidate: true, restored: true } })+ state.jobs['active-and-succeeded-migration'] = {+ apiVersion: 'batch/v1',+ kind: 'Job',+ metadata: { name: 'active-and-succeeded-migration', labels: { app: 'cumora-schema-recovery' } },+ status: { active: 1, succeeded: 1, conditions: [{ type: 'Complete', status: 'True' }] },+ }+ const result = await runScenarioWithState(state)+ assert.notEqual(result.code, 0)+ assert.match(result.stderr, /migration_residual_job_(?:present|active)/)+ assertNoSecretLeak(result)+ assert.equal(result.state.history.patches.length, 0)+ assert.equal(result.state.history.appliedJobs.length, 0)+})++test('a residual Job is allowed only with terminal Complete condition and active zero', async () => {+ const state = makeState({ smoke: { baseline: true, candidate: true, restored: true } })+ state.jobs['completed-migration'] = {+ apiVersion: 'batch/v1',+ kind: 'Job',+ metadata: { name: 'completed-migration', labels: { app: 'cumora-schema-recovery' } },+ status: { succeeded: 1, conditions: [{ type: 'Complete', status: 'True' }] },+ }+ const result = await runScenarioWithState(state)+ assert.equal(result.code, 0, result.stderr)+ assertNoSecretLeak(result)+ assert.equal(result.state.history.patches.length, 1)+ assert.equal(result.state.phase, 'candidate')+ assert.equal(serverContainer(result.state).image, serverNew)+})++test('create-only baseline race with a different protected value is rejected', async () => {+ const result = await runScenario({+ secretCreateRaces: { 'flow-baseline-secret': 'different' },+ smoke: { baseline: true, candidate: true, restored: true },+ })+ assert.notEqual(result.code, 0)+ assert.match(result.stderr, /failed|protected|baseline/i)+ assertNoSecretLeak(result)+ assert.equal(result.state.history.patches.length, 0)+ const raced = decodeSecret(result.state, 'flow-baseline-secret', 'baseline.json')+ assert.equal(raced.uid, 'raced-secret-uid')+})++test('receipt create race and Secret API errors fail closed', async () => {+ const racedReceipt = await runScenario({+ secretCreateRaces: { 'flow-receipt-secret': 'different' },+ smoke: { baseline: true, candidate: true, restored: true },+ })+ assert.notEqual(racedReceipt.code, 0)+ assert.match(racedReceipt.stderr, /candidate_receipt_conflict|failed/)+ assertNoSecretLeak(racedReceipt)+ assert.equal(racedReceipt.state.history.patches.length, 1)++ const apiFailure = await runScenario({+ secretReadFailures: { 'flow-baseline-secret': 'API unavailable' },+ smoke: { baseline: true, candidate: true, restored: true },+ })+ assert.notEqual(apiFailure.code, 0)+ assert.match(apiFailure.stderr, /secret_read_failed/)+ assertNoSecretLeak(apiFailure)+ assert.equal(apiFailure.state.history.patches.length, 0)+})++test('a concurrent writer after the PATCH response cannot become this run receipt', async () => {+ const result = await runScenario({+ switchWriterAfterCandidateResponse: true,+ smoke: { baseline: true, candidate: true, restored: true },+ })+ assert.notEqual(result.code, 0)+ assertNoSecretLeak(result)+ const receipt = decodeSecret(result.state, 'flow-receipt-secret', 'receipt.json')+ assert.equal(receipt.serverImage, serverNew)+ assert.notEqual(receipt.podTemplateHash, stateHash(result.state.deployment.spec.template))+ assert.equal(serverContainer(result.state).image, serverNew)+})++test('template drift after a successful candidate smoke cannot report success', async () => {+ const result = await runScenario({+ driftAfterSmoke: 'candidate',+ smoke: { baseline: true, candidate: true, restored: true },+ })+ assert.notEqual(result.code, 0)+ assert.match(result.stderr, /failed|candidate|template/i)+ assertNoSecretLeak(result)+ assert.equal(result.state.history.patches.length, 1)+})++test('template drift after successful restore smoke cannot report recovered success', async () => {+ const result = await runScenario({+ driftAfterSmoke: 'restored',+ smoke: { baseline: true, candidate: false, restored: true },+ })+ assert.notEqual(result.code, 0)+ assert.match(result.stderr, /recovery_failed|restore_receipt_mismatch|failed|template/i)+ assertNoSecretLeak(result)+ assert.equal(result.state.history.patches.length, 2)+ assert.equal(serverContainer(result.state).image, serverOld)+})++interface EntrypointResult {+ code: number | null+ signal: NodeJS.Signals | null+ stdout: string+ stderr: string+}++function captureChild(child: ReturnType<typeof spawn>, timeoutMs = 8_000): Promise<EntrypointResult> {+ return new Promise((resolveResult) => {+ let stdout = ''+ let stderr = ''+ child.stdout?.setEncoding('utf8')+ child.stderr?.setEncoding('utf8')+ child.stdout?.on('data', (chunk: string) => { stdout += chunk })+ child.stderr?.on('data', (chunk: string) => { stderr += chunk })+ const timer = setTimeout(() => child.kill('SIGTERM'), timeoutMs)+ child.on('close', (code, signal) => {+ clearTimeout(timer)+ resolveResult({ code, signal, stdout, stderr })+ })+ })+}++function waitMs(milliseconds: number): Promise<void> {+ return new Promise((resolveWait) => setTimeout(resolveWait, milliseconds))+}++async function runInterruptedRecoveryScenario(removeReceipt: boolean): Promise<{ run: EntrypointResult; recovery: EntrypointResult; state: Record<string, any> }> {+ await chmod(fakeKubectl, 0o755)+ const root = await mkdtemp(join(tmpdir(), 'cumora-deployment-manual-recovery-'))+ const statePath = join(root, 'fake-kubectl-state.json')+ const config: ScenarioConfig = {+ // Stop the real run while its rollout-status child is waiting. This+ // leaves the immutable baseline and candidate receipt persisted exactly+ // as an interrupted CI job would.+ holdCandidateRolloutMs: 1_000,+ smoke: { baseline: true, candidate: true, restored: true },+ }+ await writeFile(statePath, `${JSON.stringify(makeState(config), null, 2)}\n`, { mode: 0o600 })+ const smoke = await startSmokeServer(statePath, config)+ const runId = `manual-${process.pid}-${runCounter++}`+ const env: NodeJS.ProcessEnv = {+ ...process.env,+ KUBECTL: fakeKubectl,+ FAKE_KUBECTL_STATE: statePath,+ DEPLOYMENT: 'cumora-server',+ DEPLOYMENT_NAMESPACE: 'default',+ RECOVERY_SECRET: 'flow-baseline-secret',+ RECEIPT_SECRET: 'flow-receipt-secret',+ RECOVERY_WORKDIR: root,+ RUNNER_TEMP: root,+ GITHUB_WORKSPACE: repoRoot,+ GITHUB_RUN_ID: runId,+ GITHUB_SHA: 'abcdef1234567890',+ CANDIDATE_SERVER_IMAGE: serverNew,+ CANDIDATE_AGENT_IMAGE: agentNew,+ INCLUDE_AGENT: 'Y',+ MIGRATION_JOB_NAME: `manual-migrate-${runId}`,+ VERIFIER_JOB_NAME: `manual-verifier-${runId}`,+ RECOVERY_POLL_MS: '1',+ MIGRATION_POLL_ATTEMPTS: '1',+ VERIFIER_POLL_ATTEMPTS: '1',+ ROLLOUT_TIMEOUT_SECONDS: '10',+ SMOKE_TIMEOUT_SECONDS: '5',+ CUMORA_SMOKE_TOKEN: 'dummy-flow-token',+ CUMORA_SMOKE_COMPANY_ID: companyId,+ CUMORA_SMOKE_BASE: smoke.base,+ CUMORA_SMOKE_REQUIRE_SHIPPING: 'Y',+ }+ delete env.SMOKE_COMMAND+ let runChild: ReturnType<typeof spawn> | undefined+ try {+ runChild = spawn(process.execPath, ['--import', 'tsx', 'scripts/deploy-release.mjs', 'run'], {+ cwd: repoRoot,+ env,+ stdio: ['ignore', 'pipe', 'pipe'],+ })+ const runResultPromise = captureChild(runChild, 8_000)+ let interruptedState: Record<string, any> | null = null+ for (let attempt = 0; attempt < 120; attempt += 1) {+ try {+ const candidateState = JSON.parse(await readFile(statePath, 'utf8')) as Record<string, any>+ if (candidateState.history?.patches?.length === 1 && candidateState.phase === 'candidate' && candidateState.secrets?.['flow-receipt-secret']) {+ interruptedState = candidateState+ break+ }+ } catch {+ // The fake kubectl writes a complete JSON file between calls; a poll+ // that catches its brief replacement window simply retries.+ }+ await waitMs(10)+ }+ assert.ok(interruptedState, 'real run must persist candidate receipt before interruption')+ runChild.kill('SIGTERM')+ const run = await runResultPromise+ // Let the fake rollout child finish its bounded hold before the recover+ // command starts reading the same state file.+ await waitMs(1_100)+ const beforeRecovery = JSON.parse(await readFile(statePath, 'utf8')) as Record<string, any>+ if (removeReceipt) delete beforeRecovery.secrets['flow-receipt-secret']+ await writeFile(statePath, `${JSON.stringify(beforeRecovery, null, 2)}\n`, { mode: 0o600 })+ const recoveryChild = spawn(process.execPath, ['--import', 'tsx', 'scripts/deploy-release.mjs', 'recover'], {+ cwd: repoRoot,+ env,+ stdio: ['ignore', 'pipe', 'pipe'],+ })+ const recovery = await captureChild(recoveryChild)+ const state = JSON.parse(await readFile(statePath, 'utf8')) as Record<string, any>+ return { run, recovery, state }+ } finally {+ if (runChild && runChild.exitCode === null) runChild.kill('SIGTERM')+ await new Promise<void>((resolveClose) => smoke.server.close(() => resolveClose()))+ await rm(root, { recursive: true, force: true })+ }+}++test('an interrupted run can recover from protected baseline and receipt via the real recover entrypoint', async () => {+ const result = await runInterruptedRecoveryScenario(false)+ assert.notEqual(result.run.code, 0, 'the intentionally interrupted run must not report success')+ assert.equal(result.run.signal, 'SIGTERM')+ assert.equal(result.recovery.code, 0, result.recovery.stderr)+ assert.match(result.recovery.stdout, /"status":"recovered"/)+ assertNoSecretLeak(result.recovery)+ assert.equal(result.state.phase, 'restored')+ assert.equal(serverContainer(result.state).image, serverOld)+ assert.equal(result.state.history.patches.length, 2)+})++test('manual recover rejects an interrupted run when its protected receipt is missing', async () => {+ const result = await runInterruptedRecoveryScenario(true)+ assert.notEqual(result.run.code, 0)+ assert.equal(result.run.signal, 'SIGTERM')+ assert.notEqual(result.recovery.code, 0)+ assert.match(result.recovery.stderr, /protected_receipt_missing/)+ assertNoSecretLeak(result.recovery)+ assert.equal(result.state.history.patches.length, 1)+ assert.equal(result.state.phase, 'candidate')+ assert.equal(serverContainer(result.state).image, serverNew)+})++async function runScenarioWithState(initialState: Record<string, any>): Promise<RunResult> {+ const config = initialState.config as ScenarioConfig+ const root = await mkdtemp(join(tmpdir(), 'cumora-deployment-release-flow-state-'))+ const statePath = join(root, 'fake-kubectl-state.json')+ await writeFile(statePath, `${JSON.stringify(initialState, null, 2)}\n`, { mode: 0o600 })+ await chmod(fakeKubectl, 0o755)+ const smoke = await startSmokeServer(statePath, config)+ const runId = `flow-${process.pid}-${runCounter++}`+ const env: NodeJS.ProcessEnv = {+ ...process.env,+ KUBECTL: fakeKubectl,+ FAKE_KUBECTL_STATE: statePath,+ DEPLOYMENT: 'cumora-server',+ DEPLOYMENT_NAMESPACE: 'default',+ RECOVERY_SECRET: 'flow-baseline-secret',+ RECEIPT_SECRET: 'flow-receipt-secret',+ RECOVERY_WORKDIR: root,+ RUNNER_TEMP: root,+ GITHUB_WORKSPACE: repoRoot,+ GITHUB_RUN_ID: runId,+ GITHUB_SHA: 'abcdef1234567890',+ CANDIDATE_SERVER_IMAGE: serverNew,+ CANDIDATE_AGENT_IMAGE: agentNew,+ INCLUDE_AGENT: 'Y',+ MIGRATION_JOB_NAME: `flow-migrate-${runId}`,+ VERIFIER_JOB_NAME: `flow-verifier-${runId}`,+ RECOVERY_POLL_MS: '1',+ MIGRATION_POLL_ATTEMPTS: '1',+ VERIFIER_POLL_ATTEMPTS: '1',+ ROLLOUT_TIMEOUT_SECONDS: '1',+ SMOKE_TIMEOUT_SECONDS: '5',+ CUMORA_SMOKE_TOKEN: 'dummy-flow-token',+ CUMORA_SMOKE_COMPANY_ID: companyId,+ CUMORA_SMOKE_BASE: smoke.base,+ CUMORA_SMOKE_REQUIRE_SHIPPING: String(config.shippingRequired ?? 'Y'),+ }+ if (config.recoveryMode !== undefined) env.RECOVERY_MODE = String(config.recoveryMode)+ else delete env.RECOVERY_MODE+ delete env.SMOKE_COMMAND+ let child: ReturnType<typeof spawn> | undefined+ try {+ const result = await new Promise<RunResult>((resolveResult) => {+ child = spawn(process.execPath, ['--import', 'tsx', 'scripts/deploy-release.mjs', 'run'], { cwd: repoRoot, env, stdio: ['ignore', 'pipe', 'pipe'] })+ let stdout = ''+ let stderr = ''+ child.stdout?.setEncoding('utf8')+ child.stderr?.setEncoding('utf8')+ child.stdout?.on('data', (chunk: string) => { stdout += chunk })+ child.stderr?.on('data', (chunk: string) => { stderr += chunk })+ const timer = setTimeout(() => child?.kill('SIGTERM'), 15_000)+ child.on('close', async (code, signal) => {+ clearTimeout(timer)+ const finalState = JSON.parse(await readFile(statePath, 'utf8')) as Record<string, any>+ resolveResult({ code, signal, stdout, stderr, state: finalState })+ })+ })+ return result+ } finally {+ if (child && child.exitCode === null) child.kill('SIGTERM')+ await new Promise<void>((resolveClose) => smoke.server.close(() => resolveClose()))+ await rm(root, { recursive: true, force: true })+ }+}diff --git a/server/src/__tests__/fixtures/deployment-fake-kubectl.mjs b/server/src/__tests__/fixtures/deployment-fake-kubectl.mjsnew file mode 100755index 0000000..359f163--- /dev/null+++ b/server/src/__tests__/fixtures/deployment-fake-kubectl.mjs@@ -0,0 +1,366 @@+#!/usr/bin/env node++/*+ * A deliberately small kubectl-shaped process for deploy-release flow tests.+ *+ * The test owns the JSON state file. This process only implements the reads+ * and create/apply/patch operations used by scripts/deploy-release.mjs; it+ * never talks to a Kubernetes API. In particular, patching validates all+ * JSON Patch test operations before replacing anything, which models the+ * atomic CAS operation the production workflow relies on.+ */++import { readFile, writeFile } from 'node:fs/promises'++const statePath = process.env.FAKE_KUBECTL_STATE+if (!statePath) {+ process.stderr.write('FAKE_KUBECTL_STATE is required\n')+ process.exit(2)+}++const originalArgs = process.argv.slice(2)+const args = [...originalArgs]+if (args[0] === '-n') args.splice(0, 2)+const command = args.shift()++function clone(value) {+ return value === undefined ? undefined : JSON.parse(JSON.stringify(value))+}++function sorted(value) {+ if (Array.isArray(value)) return value.map(sorted)+ if (value && typeof value === 'object') {+ return Object.fromEntries(Object.entries(value).sort(([a], [b]) => a.localeCompare(b)).map(([key, child]) => [key, sorted(child)]))+ }+ return value+}++function equal(left, right) {+ return JSON.stringify(sorted(left)) === JSON.stringify(sorted(right))+}++function pointerSegments(path) {+ if (path === '') return []+ if (!path.startsWith('/')) throw new Error(`invalid JSON pointer ${path}`)+ return path.slice(1).split('/').map((part) => part.replaceAll('~1', '/').replaceAll('~0', '~'))+}++function readPointer(root, path) {+ let value = root+ for (const segment of pointerSegments(path)) value = value?.[segment]+ return value+}++async function loadState() {+ return JSON.parse(await readFile(statePath, 'utf8'))+}++async function saveState(state) {+ await writeFile(statePath, `${JSON.stringify(state, null, 2)}\n`, { mode: 0o600 })+}++function marker(state) {+ // `logs job/...` is intentionally consumed by the strict schema-verifier+ // parser, so keep that stream protocol-clean. Every other kubectl call+ // may carry the marker: deploy-release captures child stderr and must not+ // echo it in its own result.+ if (command !== 'logs' && state.config?.emitSecretMarker !== false) {+ process.stderr.write(`${state.config?.secretMarker || 'FAKE_SECRET_MARKER'}\n`)+ }+}++function fail(message, code = 1) {+ process.stderr.write(`${message}\n`)+ process.exitCode = code+}++function optionValue(list, name) {+ const index = list.indexOf(name)+ return index >= 0 ? list[index + 1] : undefined+}++function hasOption(list, name) {+ return list.includes(name)+}++async function stdinText() {+ const chunks = []+ for await (const chunk of process.stdin) chunks.push(chunk)+ return Buffer.concat(chunks).toString('utf8')+}++function emit(value) {+ if (value !== undefined && value !== null) process.stdout.write(`${JSON.stringify(value)}\n`)+}++function deploymentContainer(deployment, name) {+ return deployment?.spec?.template?.spec?.containers?.find((container) => container?.name === name)+}++function serverImage(deployment) {+ return deploymentContainer(deployment, 'server')?.image+}++function bumpHealth(deployment) {+ const generation = Number(deployment.metadata?.generation || 1) + 1+ deployment.metadata = { ...deployment.metadata, generation }+ const replicas = Number(deployment.spec?.replicas || 1)+ deployment.status = {+ ...(deployment.status || {}),+ observedGeneration: generation,+ updatedReplicas: replicas,+ readyReplicas: replicas,+ availableReplicas: replicas,+ }+}++function mutateTemplate(template, kind) {+ const next = clone(template)+ next.metadata ??= {}+ next.metadata.annotations = { ...(next.metadata.annotations || {}), 'fake.cumora.ai/writer': kind }+ return next+}++function applyPatch(deployment, operations) {+ const candidate = clone(deployment)+ for (const operation of operations) {+ if (operation?.op !== 'test') continue+ if (!equal(readPointer(candidate, operation.path), operation.value)) {+ const error = new Error(`Conflict: JSON Patch test failed at ${operation.path}`)+ error.code = 'conflict'+ throw error+ }+ }+ for (const operation of operations) {+ if (operation?.op !== 'replace') continue+ if (!['/spec/template', '/metadata/uid'].includes(operation.path)) {+ throw new Error(`unsupported replace path ${operation.path}`)+ }+ if (operation.path === '/metadata/uid') candidate.metadata.uid = clone(operation.value)+ else candidate.spec.template = clone(operation.value)+ }+ return candidate+}++function decodeSecret(secret, key) {+ const encoded = secret?.data?.[key]+ if (typeof encoded !== 'string') return null+ try {+ return JSON.parse(Buffer.from(encoded, 'base64').toString('utf8'))+ } catch {+ return null+ }+}++function encodeJson(value) {+ return Buffer.from(`${JSON.stringify(value)}\n`, 'utf8').toString('base64')+}++function raceSecret(manifest, mode) {+ const raced = clone(manifest)+ if (mode === 'different') {+ const key = Object.keys(raced.data || {})[0]+ const decoded = decodeSecret(raced, key)+ if (decoded && typeof decoded === 'object') {+ if ('uid' in decoded) decoded.uid = 'raced-secret-uid'+ else decoded.podTemplateHash = 'raced-secret-hash'+ raced.data[key] = encodeJson(decoded)+ }+ }+ return raced+}++function jobStatus(state, job) {+ const isVerifier = job?.metadata?.labels?.['recovery.cumora.ai/read-only'] === 'true'+ const setting = isVerifier ? state.config?.verifierStatus : state.config?.migrationStatus+ if (setting === 'failed') return { failed: 1, conditions: [{ type: 'Failed', status: 'True' }] }+ if (setting === 'pending' || setting === 'timedout') return { active: 1 }+ return { succeeded: 1, conditions: [{ type: 'Complete', status: 'True' }] }+}++function verifierOutput(state) {+ if (typeof state.config?.verifierLog === 'string') return state.config.verifierLog+ const markerName = 'CUMORA_SCHEMA_VERIFY_RESULT'+ const status = state.config?.verifierResult || 'compatible'+ if (status === 'compatible') {+ return `${markerName} ${JSON.stringify({ protocol: 1, kind: 'schema-verifier', status: 'compatible', currentVersion: 7, minSupported: 7, maxSupported: 7 })}\n`+ }+ if (status === 'unknown') {+ return `${markerName} ${JSON.stringify({ protocol: 1, kind: 'schema-verifier', status: 'unknown', reasonCode: 'connection_failed', currentVersion: null, minSupported: null, maxSupported: null })}\n`+ }+ if (status === 'malformed') return `${markerName} {bad-json}\n`+ return `${markerName} ${JSON.stringify({ protocol: 1, kind: 'schema-verifier', status: 'incompatible', code: status, currentVersion: null, minSupported: null, maxSupported: null })}\n`+}++async function commandGet(state) {+ const resource = args.shift()+ const name = resource === 'jobs' ? undefined : args.shift()+ if (resource === 'deployment') {+ if (state.config?.deploymentReadFailure) return fail(`Error from server (Forbidden): ${state.config.deploymentReadFailure}`)+ return emit(clone(state.deployment))+ }+ if (resource === 'secret') {+ if (state.config?.secretReadFailures?.[name]) return fail(`Error from server (Forbidden): ${state.config.secretReadFailures[name]}`)+ const secret = state.secrets?.[name]+ if (!secret) {+ if (hasOption(args, '--ignore-not-found')) return emit(undefined)+ return fail(`Error from server (NotFound): secrets "${name}" not found`)+ }+ return emit(clone(secret))+ }+ if (resource === 'job') {+ if (state.config?.jobReadFailures?.[name]) return fail(`Error from server (Forbidden): ${state.config.jobReadFailures[name]}`)+ const job = state.jobs?.[name]+ if (!job) {+ if (hasOption(args, '--ignore-not-found')) return emit(undefined)+ return fail(`Error from server (NotFound): jobs "${name}" not found`)+ }+ return emit(clone(job))+ }+ if (resource === 'jobs') {+ const selector = optionValue(args, '-l')+ const expectedApp = selector?.split('=')[1]+ const items = Object.values(state.jobs || {}).filter((job) => !expectedApp || job?.metadata?.labels?.app === expectedApp)+ if (state.config?.residualJobsReadFailure) return fail('Error from server (Forbidden): cannot list jobs')+ return emit({ apiVersion: 'batch/v1', kind: 'JobList', items: clone(items) })+ }+ return fail(`unsupported get resource ${resource}`)+}++async function commandCreate(state) {+ const manifest = JSON.parse(await stdinText())+ const name = manifest?.metadata?.name+ if (manifest?.kind !== 'Secret' || !name) return fail('fake kubectl only creates named Secrets')+ const configuredRace = state.config?.secretCreateRaces?.[name]+ const raceSeen = state.raceSeen?.[name]+ if (configuredRace && !raceSeen) {+ state.raceSeen ??= {}+ state.raceSeen[name] = true+ if (configuredRace === 'api-failure') {+ await saveState(state)+ return fail('Error from server (Forbidden): create Secret denied')+ }+ state.secrets ??= {}+ state.secrets[name] = raceSecret(manifest, configuredRace)+ await saveState(state)+ return fail(`Error from server (AlreadyExists): secrets "${name}" already exists`)+ }+ if (state.secrets?.[name]) return fail(`Error from server (AlreadyExists): secrets "${name}" already exists`)+ state.secrets ??= {}+ state.secrets[name] = manifest+ await saveState(state)+}++async function commandApply(state) {+ const manifest = JSON.parse(await stdinText())+ const name = manifest?.metadata?.name+ if (manifest?.kind !== 'Job' || !name) return fail('fake kubectl only applies Jobs')+ if (state.config?.applyJobFailure) return fail(`Error from server: ${state.config.applyJobFailure}`)+ if (state.config?.verifierStatus === 'missing' && manifest.metadata?.labels?.['recovery.cumora.ai/read-only'] === 'true') {+ await saveState(state)+ return+ }+ state.jobs ??= {}+ state.jobs[name] = { ...manifest, status: jobStatus(state, manifest) }+ state.history.appliedJobs.push(name)+ await saveState(state)+}++async function commandPatch(state) {+ const name = args.shift()+ const patchFile = optionValue(args, '--patch-file')+ let raw+ if (patchFile) raw = await readFile(patchFile, 'utf8')+ else {+ const inline = optionValue(args, '--patch')+ raw = inline ?? await stdinText()+ }+ const operations = JSON.parse(raw)+ let next+ try {+ next = applyPatch(state.deployment, operations)+ } catch (error) {+ return fail(error instanceof Error ? error.message : String(error))+ }+ const baselineImage = state.config?.baselineServerImage+ || state.initialDeployment?.spec?.template?.spec?.containers?.find((container) => container?.name === 'server')?.image+ const restoring = serverImage(next) === baselineImage+ if (restoring && state.config?.restorePatchFailure) return fail(`Error from server: ${state.config.restorePatchFailure}`)+ if (!restoring && state.config?.candidatePatchFailure) return fail(`Error from server: ${state.config.candidatePatchFailure}`)++ bumpHealth(next)+ if (!restoring && state.config?.defaultCandidatePatchResponse) {+ next.spec.template = mutateTemplate(next.spec.template, 'api-default')+ }+ const response = clone(next)+ state.deployment = next+ state.phase = restoring ? 'restored' : 'candidate'+ state.history.patches.push({ name, operations: clone(operations), response: clone(response), phase: state.phase })+ if (restoring && state.config?.restoreSmoke === 'fail') state.phase = 'restored'+ await saveState(state)+ emit(response)++ if (!restoring && state.config?.switchWriterAfterCandidateResponse) {+ const writer = clone(state.deployment)+ writer.spec.template = mutateTemplate(writer.spec.template, 'other-writer')+ state.deployment = writer+ state.phase = 'other-writer'+ await saveState(state)+ }+}++async function commandRollout(state) {+ const subcommand = args.shift()+ const resource = args.shift()+ if (subcommand !== 'status') return fail(`unsupported rollout subcommand ${subcommand}`)+ if (!resource?.startsWith('deployment/')) return fail('unsupported rollout target')+ const baselineImage = state.config?.baselineServerImage+ || state.initialDeployment?.spec?.template?.spec?.containers?.find((container) => container?.name === 'server')?.image+ const restoring = serverImage(state.deployment) === baselineImage+ const setting = restoring ? state.config?.restoreRollout : state.config?.candidateRollout+ const failureMessage = restoring ? state.config?.restoreRolloutFailureMessage : state.config?.rolloutFailureMessage+ state.history.rollouts.push({+ phase: restoring ? 'restored' : 'candidate',+ setting: setting || 'success',+ ...(typeof failureMessage === 'string' ? { failureMessage } : {}),+ })+ if (!restoring && Number(state.config?.holdCandidateRolloutMs || 0) > 0) {+ await new Promise((resolve) => setTimeout(resolve, Number(state.config.holdCandidateRolloutMs)))+ }+ if (!restoring && setting === 'drift-template') state.deployment.spec.template = mutateTemplate(state.deployment.spec.template, 'drift')+ if (!restoring && setting === 'drift-uid') state.deployment.metadata.uid = 'writer-changed-uid'+ if (!restoring && setting === 'progressing') {+ state.deployment.status = { ...(state.deployment.status || {}), observedGeneration: Number(state.deployment.metadata.generation) - 1, updatedReplicas: 0, readyReplicas: 0, availableReplicas: 0 }+ }+ await saveState(state)+ if (setting && setting !== 'success') return fail(failureMessage || `Error from server: rollout ${setting}`)+}++async function commandLogs(state) {+ const name = args.shift()?.replace(/^job\//, '')+ if (!state.jobs?.[name]) return fail(`Error from server (NotFound): jobs "${name}" not found`)+ emitRaw(verifierOutput(state))+}++function emitRaw(value) {+ process.stdout.write(value)+}++async function main() {+ const state = await loadState()+ state.secrets ??= {}+ state.jobs ??= {}+ state.history ??= { commands: [], patches: [], rollouts: [], appliedJobs: [] }+ state.history.commands.push({ args: originalArgs, at: new Date().toISOString() })+ await saveState(state)+ marker(state)+ if (command === 'get') await commandGet(state)+ else if (command === 'create') await commandCreate(state)+ else if (command === 'apply') await commandApply(state)+ else if (command === 'patch') await commandPatch(state)+ else if (command === 'rollout') await commandRollout(state)+ else if (command === 'logs') await commandLogs(state)+ else fail(`unsupported kubectl command ${command}`)+}++await main()diff --git a/server/src/__tests__/migration-ownership.test.ts b/server/src/__tests__/migration-ownership.test.tsindex 7136e96..d27c0ca 100644--- a/server/src/__tests__/migration-ownership.test.ts+++ b/server/src/__tests__/migration-ownership.test.ts@@ -25,13 +25,12 @@ test('application Pod manifests contain no per-replica migration container', asy}})-test('production deploy migrates before one atomic Deployment mutation', async () => {+test('production deploy delegates one guarded transaction to the recovery runner', async () => {const workflow = await readRepo('.github/workflows/deploy.yml')- const migrationAt = workflow.indexOf('- name: Run candidate migrations once')- const patchAt = workflow.indexOf('- name: Patch deployment')- assert.ok(migrationAt >= 0, 'deploy must create a single candidate migration Job')- assert.ok(patchAt > migrationAt, 'migration must complete before Deployment mutation')- assert.match(workflow, /kind:\s*"Job"/)- assert.match(workflow, /Candidate database migration did not complete; deployment was not mutated/)- assert.match(workflow, /\{ name: "migrate", "\$patch": "delete" \}/)+ assert.match(workflow, /scripts\/deploy-release\.mjs run/)+ assert.match(workflow, /CANDIDATE_SERVER_IMAGE/)+ assert.match(workflow, /MIGRATION_REPAIR/)+ assert.match(workflow, /RECOVERY_WORKDIR/)+ assert.match(workflow, /concurrency:/)+ assert.doesNotMatch(workflow, /kubectl rollout undo/)})diff --git a/server/src/__tests__/startup-probes.test.ts b/server/src/__tests__/startup-probes.test.tsindex 6dc98e9..95f466a 100644--- a/server/src/__tests__/startup-probes.test.ts+++ b/server/src/__tests__/startup-probes.test.ts@@ -1,19 +1,21 @@import assert from 'node:assert/strict'-import { execFile as execFileCallback } from 'node:child_process'import { readFile } from 'node:fs/promises'import { fileURLToPath } from 'node:url'import { dirname, resolve } from 'node:path'import { test } from 'node:test'-import { promisify } from 'node:util'-import { load, loadAll } from 'js-yaml'+import { loadAll } from 'js-yaml'import { MigrationHistoryError } from '../db/migrations/manifest.js'+import {+ DEPLOYMENT_PROBE_CONTRACT,+ buildCandidatePatch,+ extractDeploymentSnapshot,+} from '../deploy/recovery.js'process.env.CUMORA_RUNTIME_CLIENT = 'http'process.env.OPENAI_API_KEY ??= 'test-key'const { verifySchemaWithBootRetry } = await import('../db/schema-version.js')const { pool } = await import('../db/pool.js')-const execFile = promisify(execFileCallback)const repoRoot = resolve(dirname(fileURLToPath(import.meta.url)), '../../..')const readRepo = (path: string): Promise<string> => readFile(resolve(repoRoot, path), 'utf8')@@ -37,24 +39,14 @@ interface Deployment {spec?: { template?: { spec?: { containers?: Container[] } } }}-interface WorkflowStep {- name?: string- run?: string-}--interface Workflow {- jobs?: Record<string, { steps?: WorkflowStep[] }>-}--interface ProbePatch {- spec?: { template?: { spec?: { containers?: Container[] } } }-}-const manifestPaths = ['server/k8s/cumora-server.gke.yaml','server/k8s/cumora-server.orbstack.yaml',]+const PROBE_FIXTURE_SERVER_IMAGE = `server@sha256:${'a'.repeat(64)}`+const PROBE_FIXTURE_AGENT_IMAGE = `agent@sha256:${'b'.repeat(64)}`+async function serverContainer(path: string): Promise<Container> {const docs = loadAll(await readRepo(path)) as unknown[]const deployment = docs@@ -89,41 +81,46 @@ test('both server manifests keep startup, liveness, and readiness probes on the}})-test('the deploy patch reapplies the same probe contract and rollout deadline', async (t) => {- const workflow = load(await readRepo('.github/workflows/deploy.yml')) as Workflow- const patchStep = workflow.jobs?.deploy?.steps?.find((step) => step.name === 'Patch deployment')- assert.ok(patchStep?.run, 'deploy must have a parsed Patch deployment step')- const patch = patchStep.run- assert.match(patch, /startupProbe:\s*\{\s*httpGet:\s*\{\s*path: "\/api\/livez",\s*port: "http"\s*\}/)- assert.match(patch, /readinessProbe:\s*\{\s*httpGet:\s*\{\s*path: "\/api\/health",\s*port: "http"\s*\}/)- assert.match(patch, /livenessProbe:\s*\{\s*httpGet:\s*\{\s*path: "\/api\/livez",\s*port: "http"\s*\}/)- assert.match(patch, /periodSeconds: 5/)- assert.match(patch, /timeoutSeconds: 2/)- assert.match(patch, /failureThreshold: 60/)- const rolloutStep = workflow.jobs?.deploy?.steps?.find((step) => step.name === 'Wait for rollout')- assert.match(rolloutStep?.run ?? '', /kubectl rollout status deployment\/cumora-server --timeout=10m/)-- const filterOpen = patch.indexOf("'\n")- const filterClose = patch.indexOf("\n')", filterOpen + 2)- assert.ok(filterOpen >= 0 && filterClose > filterOpen, 'Patch deployment must contain a jq filter')- const jqFilter = patch.slice(filterOpen + 2, filterClose)- let jq: { stdout: string }- try {- jq = await execFile(- 'jq',- ['-nc', '--arg', 'server', 'fixture-server', '--arg', 'agent', 'fixture-agent', jqFilter],- { encoding: 'utf8' },- ) as { stdout: string }- } catch (error) {- if (!process.env.CI && (error as { code?: unknown })?.code === 'ENOENT') {- t.skip('jq is unavailable locally; CI must provide jq for this workflow patch test')- return- }- throw error+test('the recovery helper reapplies the same probe contract and workflow calls it', async () => {+ const workflowText = await readRepo('.github/workflows/deploy.yml')+ assert.match(workflowText, /scripts\/deploy-release\.mjs run/)+ assert.match(workflowText, /CANDIDATE_SERVER_IMAGE/)+ assert.match(workflowText, /ROLLOUT_TIMEOUT_SECONDS/)+ assert.match(workflowText, /VERIFIER_POLL_ATTEMPTS:\s*'75'/)+ assert.doesNotMatch(workflowText, /kubectl rollout undo/)++ const baseline = extractDeploymentSnapshot({+ apiVersion: 'apps/v1',+ kind: 'Deployment',+ metadata: { name: 'cumora-server', uid: 'probe-fixture' },+ spec: {+ template: {+ metadata: { labels: { app: 'cumora-server' } },+ spec: {+ containers: [{+ name: 'server',+ image: PROBE_FIXTURE_SERVER_IMAGE,+ env: [{ name: 'CUMORA_AGENT_COMPUTER_IMAGE', value: PROBE_FIXTURE_AGENT_IMAGE }],+ }],+ },+ },+ },+ })+ const patch = buildCandidatePatch(baseline, {+ server: `server@sha256:${'c'.repeat(64)}`,+ agent: `agent@sha256:${'d'.repeat(64)}`,+ })+ const replacement = patch.find((operation) => {+ const candidate = operation as any+ return candidate?.op === 'replace' && candidate?.path === '/spec/template'+ }) as any+ assert.ok(replacement && typeof replacement.value === 'object' && replacement.value !== null)+ const patchedServer = (replacement.value as { spec: { containers: Container[] } }).spec.containers+ .find((container) => container.name === 'server')+ assert.ok(patchedServer, 'recovery helper patch must target the server container')+ for (const [name, expected] of Object.entries(DEPLOYMENT_PROBE_CONTRACT)) {+ assert.deepEqual(patchedServer[name as keyof Container], expected, `${name} helper contract`)}- const patchObject = JSON.parse(String(jq.stdout)) as ProbePatch- const patchedServer = patchObject.spec?.template?.spec?.containers?.find((container) => container.name === 'server')- assert.ok(patchedServer, 'evaluated deploy patch must target the server container')for (const path of manifestPaths) {const manifestServer = await serverContainer(path)assert.deepEqual(patchedServer.startupProbe, manifestServer.startupProbe, `${path} startup contract`)diff --git a/server/src/deploy/recovery.ts b/server/src/deploy/recovery.tsnew file mode 100644index 0000000..b12acc5--- /dev/null+++ b/server/src/deploy/recovery.ts@@ -0,0 +1,804 @@+import { createHash } from 'node:crypto'++/**+ * Deployment recovery is deliberately a data-only module. The workflow owns+ * kubectl, while this module owns the objects passed to kubectl and all of the+ * compare-and-swap rules. Keeping those rules here lets tests exercise the+ * exact patch and Job objects used in production without contacting a cluster.+ */++export type JsonPrimitive = string | number | boolean | null+export type JsonValue = unknown+export type JsonObject = Record<string, unknown>++export interface PodTemplateSpec {+ containers: JsonObject[]+ initContainers?: JsonObject[]+ serviceAccountName?: string+ imagePullSecrets?: JsonValue+ securityContext?: JsonValue+ volumes?: JsonValue+ [key: string]: unknown+}++export interface PodTemplate {+ metadata: JsonObject+ spec: PodTemplateSpec+ [key: string]: unknown+}++export interface JsonPatchOperation {+ op: 'test' | 'replace'+ path: string+ value: JsonValue+}++export interface DeploymentHealth {+ generation: number | null+ observedGeneration: number | null+ desiredReplicas: number | null+ updatedReplicas: number | null+ readyReplicas: number | null+ availableReplicas: number | null+}++export interface DeploymentSnapshot {+ apiVersion: string+ kind: 'Deployment'+ name: string+ namespace: string+ uid: string+ podTemplate: PodTemplate+ podTemplateHash: string+ serverImage: string+ agentImage: string | null+ agentImageSource: 'missing' | 'inline' | 'unknown'+ baselineImage: { server: string; agent: string | null }+ health: DeploymentHealth+ baselineSmokePassed?: boolean+ baselineVerifiedAt?: string+ capturedAt: string+}++export interface DeploymentReceipt {+ uid: string+ name: string+ namespace: string+ podTemplate: PodTemplate+ podTemplateHash: string+ serverImage: string+ agentImage: string | null+ agentImageSource: 'missing' | 'inline' | 'unknown'+ baselineImage: { server: string; agent: string | null }+ observedAt: string+}++export interface CandidateImages {+ server: string+ /** Omit to preserve the baseline agent image. */+ agent?: string+}++export interface MigrationJobOptions {+ name: string+ image: string+ repairMode: 'off' | 'archive-detach'+ activeDeadlineSeconds?: number+ ttlSecondsAfterFinished?: number+}++export interface SchemaVerifierJobOptions {+ name: string+ image: string+ activeDeadlineSeconds?: number+ ttlSecondsAfterFinished?: number+}++export interface SchemaVerifierResult {+ [key: string]: unknown+ protocol: 1+ kind: 'schema-verifier'+ status: 'compatible' | 'incompatible' | 'unknown'+ currentVersion: number | null+ minSupported: number | null+ maxSupported: number | null+ code?: 'schema_uninitialized' | 'schema_behind' | 'schema_ahead' | 'migration_history_invalid'+ reasonCode?: string+}++export const SCHEMA_VERIFIER_MARKER = 'CUMORA_SCHEMA_VERIFY_RESULT'++export const SCHEMA_INCOMPATIBLE_CODES = [+ 'schema_uninitialized',+ 'schema_behind',+ 'schema_ahead',+ 'migration_history_invalid',+] as const++export const SCHEMA_UNKNOWN_REASON_CODES = [+ 'module_unavailable',+ 'connection_failed',+ 'timeout',+ 'cleanup_failed',+] as const++const DIGEST_IMAGE_PATTERN = /@sha256:[a-f0-9]{64}$/++export const DEPLOYMENT_PROBE_CONTRACT = {+ startupProbe: {+ httpGet: { path: '/api/livez', port: 'http' },+ periodSeconds: 5,+ timeoutSeconds: 2,+ failureThreshold: 60,+ },+ readinessProbe: {+ httpGet: { path: '/api/health', port: 'http' },+ periodSeconds: 5,+ timeoutSeconds: 2,+ },+ livenessProbe: {+ httpGet: { path: '/api/livez', port: 'http' },+ periodSeconds: 30,+ timeoutSeconds: 3,+ },+} as const++function isRecord(value: unknown): value is Record<string, unknown> {+ return typeof value === 'object' && value !== null && !Array.isArray(value)+}++function isJsonValue(value: unknown): value is JsonValue {+ if (value === null || typeof value === 'string' || typeof value === 'number' || typeof value === 'boolean') {+ return true+ }+ if (Array.isArray(value)) return value.every(isJsonValue)+ return isRecord(value) && Object.values(value).every(isJsonValue)+}++function asJsonObject(value: unknown, label: string): JsonObject {+ if (!isRecord(value) || !isJsonValue(value)) throw new Error(`${label} must be a JSON object`)+ return value as JsonObject+}++function cloneJson<T>(value: T): T {+ return JSON.parse(JSON.stringify(value)) as T+}++/** Sort object keys recursively while preserving array order. */+export function normalizeJson(value: unknown): unknown {+ if (Array.isArray(value)) return value.map(normalizeJson)+ if (!isRecord(value)) return value+ return Object.fromEntries(+ Object.entries(value)+ .sort(([left], [right]) => left.localeCompare(right))+ .map(([key, child]) => [key, normalizeJson(child)]),+ ) as JsonObject+}++export function hashJson(value: unknown): string {+ return createHash('sha256').update(JSON.stringify(normalizeJson(value))).digest('hex')+}++export function hashPodTemplate(template: JsonObject | PodTemplate): string {+ return hashJson(template)+}++function readString(value: unknown, label: string): string {+ if (typeof value !== 'string' || value.length === 0) throw new Error(`${label} must be a non-empty string`)+ return value+}++function integerOrNull(value: unknown): number | null {+ return typeof value === 'number' && Number.isInteger(value) ? value : null+}++function assertDigestImage(value: unknown, label: string): string {+ const image = readString(value, label)+ if (!DIGEST_IMAGE_PATTERN.test(image)) throw new Error(`${label} must be pinned by sha256 digest`)+ return image+}++function getObjectPath(root: unknown, path: string[], label: string): Record<string, unknown> {+ let current: unknown = root+ for (const segment of path) {+ if (!isRecord(current) || !(segment in current)) throw new Error(`${label} is missing ${path.join('.')}`)+ current = current[segment]+ }+ if (!isRecord(current)) throw new Error(`${label} ${path.join('.')} must be an object`)+ return current+}++function getArrayPath(root: unknown, path: string[], label: string): unknown[] {+ let current: unknown = root+ for (const segment of path) {+ if (!isRecord(current) || !(segment in current)) throw new Error(`${label} is missing ${path.join('.')}`)+ current = current[segment]+ }+ if (!Array.isArray(current)) throw new Error(`${label} ${path.join('.')} must be an array`)+ return current+}++function findNamedObject(items: unknown[], name: string, label: string): Record<string, unknown> {+ const found = items.find((item) => isRecord(item) && item.name === name)+ if (!isRecord(found)) throw new Error(`${label} is missing ${name}`)+ return found+}++function agentImageValue(container: Record<string, unknown>, name: string): { value: string | null; source: 'missing' | 'inline' | 'unknown' } {+ const env = container.env+ if (!Array.isArray(env)) return { value: null, source: 'missing' }+ const entry = env.find((item) => isRecord(item) && item.name === name)+ if (!isRecord(entry)) return { value: null, source: 'missing' }+ if (typeof entry.value !== 'string') return { value: null, source: 'unknown' }+ return { value: entry.value, source: 'inline' }+}++function deploymentPodTemplate(deployment: unknown): PodTemplate {+ const template = asJsonObject(getObjectPath(deployment, ['spec', 'template'], 'Deployment'), 'Deployment pod template')+ if (!isRecord(template.metadata) || !isRecord(template.spec)) throw new Error('Deployment pod template must contain metadata and spec')+ if (!Array.isArray(template.spec.containers)) throw new Error('Deployment pod template spec.containers must be an array')+ return template as PodTemplate+}++function deploymentServerContainer(template: JsonObject): JsonObject {+ const containers = getArrayPath(template, ['spec', 'containers'], 'Deployment pod template')+ return asJsonObject(findNamedObject(containers, 'server', 'Deployment pod template containers'), 'server container')+}++export function extractDeploymentSnapshot(+ deployment: unknown,+ opts: { expectedName?: string; expectedNamespace?: string; capturedAt?: string } = {},+): DeploymentSnapshot {+ const metadata = getObjectPath(deployment, ['metadata'], 'Deployment')+ const template = deploymentPodTemplate(deployment)+ const server = deploymentServerContainer(template)+ const uid = readString(metadata.uid, 'Deployment metadata.uid')+ const apiVersion = readString((deployment as Record<string, unknown>).apiVersion, 'Deployment apiVersion')+ if (apiVersion !== 'apps/v1') throw new Error(`unsupported Deployment apiVersion: ${apiVersion}`)+ const kind = readString((deployment as Record<string, unknown>).kind, 'Deployment kind')+ if (kind !== 'Deployment') throw new Error(`expected Deployment, got ${kind}`)+ const name = readString(metadata.name, 'Deployment metadata.name')+ // kubectl includes metadata.namespace. Fixtures and an unqualified local+ // object can omit it; Kubernetes resolves those to the default namespace.+ const namespace = typeof metadata.namespace === 'string' && metadata.namespace.length > 0 ? metadata.namespace : 'default'+ if (opts.expectedName && name !== opts.expectedName) throw new Error(`unexpected Deployment name: ${name}`)+ if (opts.expectedNamespace && namespace !== opts.expectedNamespace) throw new Error(`unexpected Deployment namespace: ${namespace}`)+ const serverImage = assertDigestImage(server.image, 'server container image')+ const agent = agentImageValue(server, 'CUMORA_AGENT_COMPUTER_IMAGE')+ if (agent.value !== null) assertDigestImage(agent.value, 'agent container image')+ return {+ apiVersion,+ kind: 'Deployment',+ name,+ namespace,+ uid,+ podTemplate: cloneJson(template),+ podTemplateHash: hashPodTemplate(template),+ serverImage,+ agentImage: agent.value,+ agentImageSource: agent.source,+ baselineImage: { server: serverImage, agent: agent.value },+ health: {+ generation: integerOrNull(metadata.generation),+ observedGeneration: integerOrNull(isRecord((deployment as Record<string, unknown>).status) ? (deployment as Record<string, Record<string, unknown>>).status.observedGeneration : null),+ desiredReplicas: integerOrNull(isRecord((deployment as Record<string, unknown>).spec) ? (deployment as Record<string, Record<string, unknown>>).spec.replicas : null),+ updatedReplicas: integerOrNull(isRecord((deployment as Record<string, unknown>).status) ? (deployment as Record<string, Record<string, unknown>>).status.updatedReplicas : null),+ readyReplicas: integerOrNull(isRecord((deployment as Record<string, unknown>).status) ? (deployment as Record<string, Record<string, unknown>>).status.readyReplicas : null),+ availableReplicas: integerOrNull(isRecord((deployment as Record<string, unknown>).status) ? (deployment as Record<string, Record<string, unknown>>).status.availableReplicas : null),+ },+ capturedAt: opts.capturedAt ?? new Date().toISOString(),+ }+}++export function deploymentReceiptFromObject(+ deployment: unknown,+ opts: { observedAt?: string } = {},+): DeploymentReceipt {+ const snapshot = extractDeploymentSnapshot(deployment, { capturedAt: opts.observedAt })+ return {+ uid: snapshot.uid,+ name: snapshot.name,+ namespace: snapshot.namespace,+ podTemplate: snapshot.podTemplate,+ podTemplateHash: snapshot.podTemplateHash,+ serverImage: snapshot.serverImage,+ agentImage: snapshot.agentImage,+ agentImageSource: snapshot.agentImageSource,+ baselineImage: snapshot.baselineImage,+ observedAt: snapshot.capturedAt,+ }+}++function assertSnapshotShape(snapshot: DeploymentSnapshot): void {+ if (snapshot.kind !== 'Deployment') throw new Error('recovery baseline must describe a Deployment')+ if (snapshot.apiVersion !== 'apps/v1') throw new Error('recovery baseline must use apps/v1')+ readString(snapshot.uid, 'recovery baseline uid')+ readString(snapshot.name, 'recovery baseline name')+ readString(snapshot.namespace, 'recovery baseline namespace')+ if (!isRecord(snapshot.podTemplate)) throw new Error('recovery baseline podTemplate must be an object')+ const actualHash = hashPodTemplate(snapshot.podTemplate)+ if (actualHash !== snapshot.podTemplateHash) {+ throw new Error('recovery baseline podTemplateHash does not match podTemplate')+ }+ assertDigestImage(snapshot.serverImage, 'recovery baseline serverImage')+ if (snapshot.agentImage !== null && typeof snapshot.agentImage !== 'string') {+ throw new Error('recovery baseline agentImage must be a string or null')+ }+ if (snapshot.agentImage !== null) assertDigestImage(snapshot.agentImage, 'recovery baseline agentImage')+ if (!['missing', 'inline', 'unknown'].includes(snapshot.agentImageSource)) throw new Error('invalid recovery baseline agent image source')+ if (snapshot.baselineImage?.server !== snapshot.serverImage || snapshot.baselineImage?.agent !== snapshot.agentImage) {+ throw new Error('recovery baseline baselineImage does not match image fields')+ }+ const server = deploymentServerContainer(snapshot.podTemplate)+ if (server.image !== snapshot.serverImage) throw new Error('recovery baseline server image does not match podTemplate')+ const agent = agentImageValue(server, 'CUMORA_AGENT_COMPUTER_IMAGE')+ if (agent.value !== snapshot.agentImage || agent.source !== snapshot.agentImageSource) {+ throw new Error('recovery baseline agent image does not match podTemplate')+ }+}++export function assertBaselineUnchanged(existing: DeploymentSnapshot, candidate: DeploymentSnapshot | DeploymentReceipt): void {+ assertSnapshotShape(existing)+ if ('apiVersion' in candidate) assertSnapshotShape(candidate)+ else assertReceiptShape(candidate)+ if (existing.uid !== candidate.uid || existing.podTemplateHash !== candidate.podTemplateHash) {+ throw new Error('recovery baseline already exists and differs; refusing to replace it')+ }+}++export function assertDeploymentHealthy(snapshot: DeploymentSnapshot): void {+ const health = snapshot.health+ if (!health || health.generation === null || health.observedGeneration !== health.generation) {+ throw new Error('Deployment baseline rollout is not observed')+ }+ if (+ health.desiredReplicas === null ||+ health.updatedReplicas !== health.desiredReplicas ||+ health.readyReplicas !== health.desiredReplicas ||+ health.availableReplicas !== health.desiredReplicas+ ) {+ throw new Error('Deployment baseline rollout replicas are not healthy')+ }+}++export function assertRecoveryEligible(baseline: DeploymentSnapshot): void {+ assertDeploymentHealthy(baseline)+ if (baseline.baselineSmokePassed !== true) {+ throw new Error('Deployment baseline smoke was not proven before migration')+ }+}++function patchContainer(+ template: PodTemplate,+ images: CandidateImages,+): PodTemplate {+ assertDigestImage(images.server, 'candidate server image')+ if (images.agent !== undefined) assertDigestImage(images.agent, 'candidate agent image')+ const patched = cloneJson(template)+ const spec = patched.spec+ const containers = spec.containers+ const server = asJsonObject(findNamedObject(containers, 'server', 'pod template containers'), 'server container')+ server.image = images.server+ server.startupProbe = cloneJson(DEPLOYMENT_PROBE_CONTRACT.startupProbe as unknown as JsonObject)+ server.readinessProbe = cloneJson(DEPLOYMENT_PROBE_CONTRACT.readinessProbe as unknown as JsonObject)+ server.livenessProbe = cloneJson(DEPLOYMENT_PROBE_CONTRACT.livenessProbe as unknown as JsonObject)++ if (images.agent !== undefined) {+ const env = Array.isArray(server.env) ? server.env.filter((entry) => !(isRecord(entry) && entry.name === 'CUMORA_AGENT_COMPUTER_IMAGE')) : []+ env.push({ name: 'CUMORA_AGENT_COMPUTER_IMAGE', value: images.agent })+ server.env = env as JsonValue+ }++ if (Array.isArray(spec.initContainers)) {+ spec.initContainers = spec.initContainers.filter((entry) => !(isRecord(entry) && entry.name === 'migrate')) as JsonObject[]+ }+ return patched as PodTemplate+}++export function buildCandidateTemplate(baseline: DeploymentSnapshot, images: CandidateImages): PodTemplate {+ assertSnapshotShape(baseline)+ return patchContainer(baseline.podTemplate, images)+}++function jsonPatchTest(path: string, value: JsonValue): JsonPatchOperation {+ return { op: 'test', path, value: cloneJson(value) }+}++function jsonPatchReplace(path: string, value: JsonValue): JsonPatchOperation {+ return { op: 'replace', path, value: cloneJson(value) }+}++export function buildCandidatePatch(+ baseline: DeploymentSnapshot,+ images: CandidateImages,+): JsonPatchOperation[] {+ assertSnapshotShape(baseline)+ const candidate = buildCandidateTemplate(baseline, images)+ return [+ jsonPatchTest('/metadata/uid', baseline.uid),+ jsonPatchTest('/spec/template', baseline.podTemplate),+ jsonPatchReplace('/spec/template', candidate),+ ]+}++export function buildRestorePatch(+ baseline: DeploymentSnapshot,+ current: DeploymentReceipt,+ expectedCandidateHash: string,+): JsonPatchOperation[] {+ assertSnapshotShape(baseline)+ assertReceiptShape(current)+ if (baseline.agentImageSource === 'unknown' || current.agentImageSource === 'unknown') {+ throw new Error('agent image valueFrom is not a safe automatic recovery baseline')+ }+ if (current.uid !== baseline.uid) throw new Error('Deployment UID changed; refusing automatic recovery')+ if (current.podTemplateHash !== expectedCandidateHash) {+ throw new Error('Deployment pod template drifted from the candidate receipt; refusing automatic recovery')+ }+ return [+ jsonPatchTest('/metadata/uid', baseline.uid),+ jsonPatchTest('/spec/template', current.podTemplate),+ jsonPatchReplace('/spec/template', baseline.podTemplate),+ ]+}++function assertReceiptShape(receipt: DeploymentReceipt): void {+ readString(receipt.uid, 'candidate receipt uid')+ readString(receipt.name, 'candidate receipt name')+ readString(receipt.namespace, 'candidate receipt namespace')+ if (!isRecord(receipt.podTemplate)) throw new Error('candidate receipt podTemplate must be an object')+ if (hashPodTemplate(receipt.podTemplate) !== receipt.podTemplateHash) {+ throw new Error('candidate receipt podTemplateHash does not match podTemplate')+ }+ assertDigestImage(receipt.serverImage, 'candidate receipt serverImage')+ if (receipt.agentImage !== null) assertDigestImage(receipt.agentImage, 'candidate receipt agentImage')+ if (!['missing', 'inline', 'unknown'].includes(receipt.agentImageSource)) throw new Error('invalid candidate receipt agent image source')+ if (receipt.baselineImage?.server !== receipt.serverImage || receipt.baselineImage?.agent !== receipt.agentImage) {+ throw new Error('candidate receipt baselineImage does not match image fields')+ }+ const server = deploymentServerContainer(receipt.podTemplate)+ if (server.image !== receipt.serverImage) throw new Error('candidate receipt server image does not match podTemplate')+ const agent = agentImageValue(server, 'CUMORA_AGENT_COMPUTER_IMAGE')+ if (agent.value !== receipt.agentImage || agent.source !== receipt.agentImageSource) {+ throw new Error('candidate receipt agent image does not match podTemplate')+ }+}++export function assertReceiptShapeForRecovery(receipt: DeploymentReceipt): void {+ assertReceiptShape(receipt)+}++export function assertRecoveryStateBinding(+ baseline: DeploymentSnapshot,+ receipt: DeploymentReceipt,+): void {+ assertSnapshotShape(baseline)+ assertReceiptShape(receipt)+ if (baseline.uid !== receipt.uid || baseline.name !== receipt.name || baseline.namespace !== receipt.namespace) {+ throw new Error('protected recovery receipt is bound to a different Deployment')+ }+}++function cleanContainerForJob(container: JsonObject): JsonObject {+ const result = cloneJson(container)+ for (const field of [+ 'ports',+ 'startupProbe',+ 'readinessProbe',+ 'livenessProbe',+ 'lifecycle',+ 'terminationMessagePath',+ 'terminationMessagePolicy',+ ]) delete result[field]+ return result+}++function podSpecForJob(template: JsonObject): { pod: JsonObject; server: JsonObject; proxy: JsonObject; proxyStartupProbe: JsonValue } {+ const spec = asJsonObject(getObjectPath(template, ['spec'], 'pod template'), 'pod template spec')+ const allInit = Array.isArray(spec.initContainers) ? spec.initContainers : []+ const allContainers = getArrayPath(spec, ['containers'], 'pod template')+ const proxySource = [...allInit, ...allContainers].find((item) => isRecord(item) && item.name === 'cloud-sql-proxy')+ if (!isRecord(proxySource)) throw new Error('Deployment pod template has no cloud-sql-proxy container')+ const server = findNamedObject(allContainers, 'server', 'pod template containers')+ const pod = cloneJson(spec)+ delete pod.containers+ delete pod.initContainers+ delete pod.ephemeralContainers+ const proxyStartupProbe = isJsonValue(proxySource.startupProbe)+ ? cloneJson(proxySource.startupProbe)+ : {+ httpGet: { path: '/readiness', port: 9090 },+ periodSeconds: 1,+ failureThreshold: 60,+ }+ return { pod, server: cleanContainerForJob(server), proxy: cleanContainerForJob(proxySource), proxyStartupProbe }+}++function upsertEnv(container: JsonObject, name: string, value: string): void {+ const env = Array.isArray(container.env) ? container.env.filter((entry) => !(isRecord(entry) && entry.name === name)) : []+ env.push({ name, value })+ container.env = env as JsonValue+}++function jobBase(+ template: JsonObject,+ options: { name: string; image: string; command: string[]; args?: string[] },+): { metadata: JsonObject; spec: JsonObject; pod: JsonObject; server: JsonObject; proxy: JsonObject } {+ const { pod, server, proxy, proxyStartupProbe } = podSpecForJob(template)+ server.name = 'migrate'+ server.image = options.image+ server.imagePullPolicy = 'IfNotPresent'+ server.command = options.command as unknown as JsonValue+ delete server.args+ if (options.args) server.args = options.args as unknown as JsonValue+ server.workingDir = '/app'+ pod.restartPolicy = 'Never'+ pod.initContainers = [+ {+ ...proxy,+ name: 'cloud-sql-proxy',+ restartPolicy: 'Always',+ startupProbe: proxyStartupProbe,+ },+ ] as unknown as JsonValue+ pod.containers = [server] as unknown as JsonValue+ return {+ metadata: {+ name: options.name,+ labels: { app: 'cumora-schema-recovery' },+ },+ spec: {+ backoffLimit: 0,+ activeDeadlineSeconds: 600,+ ttlSecondsAfterFinished: 600,+ template: {+ metadata: { labels: { app: 'cumora-schema-recovery', 'recovery.cumora.ai/managed': 'true' } },+ spec: pod,+ },+ },+ pod,+ server,+ proxy,+ }+}++export function buildMigrationJob(+ baseline: DeploymentSnapshot,+ options: MigrationJobOptions,+): JsonObject {+ assertSnapshotShape(baseline)+ assertDigestImage(options.image, 'migration image')+ if (!['off', 'archive-detach'].includes(options.repairMode)) throw new Error('invalid migration repair mode')+ const result = jobBase(baseline.podTemplate, {+ name: options.name,+ image: options.image,+ command: ['npm', 'run', 'migrate'],+ })+ result.spec.activeDeadlineSeconds = options.activeDeadlineSeconds ?? 600+ result.spec.ttlSecondsAfterFinished = options.ttlSecondsAfterFinished ?? 600+ upsertEnv(result.server, 'MIGRATION_0002_REPAIR', options.repairMode)+ return {+ apiVersion: 'batch/v1',+ kind: 'Job',+ metadata: result.metadata,+ spec: result.spec,+ }+}++/**+ * This is injected into the old server image during recovery. It imports the+ * old image's own pool, schema gate, and manifest, so a newer checkout cannot+ * accidentally decide that an old image is compatible. The program writes a+ * single marker line and nothing else; the parser below fails closed on any+ * extra or malformed output.+ */+export function buildSchemaVerifierScript(): string {+ const marker = JSON.stringify(SCHEMA_VERIFIER_MARKER)+ return `+const marker = ${marker};+let result = { status: "unknown", reasonCode: "module_unavailable", currentVersion: null, minSupported: null, maxSupported: null };+let client;+let pool;+let MigrationHistoryError;+let manifest;+let began = false;+let committed = false;+try {+ const [poolModule, schemaModule, manifestModule] = await Promise.all([+ import("./server/src/db/pool.ts"),+ import("./server/src/db/schema-version.ts"),+ import("./server/src/db/migrations/manifest.ts"),+ ]);+ pool = poolModule.pool;+ const verifySchemaCompatibility = schemaModule.verifySchemaCompatibility;+ MigrationHistoryError = manifestModule.MigrationHistoryError;+ manifest = manifestModule;+ if (!pool || typeof pool.connect !== "function" || typeof verifySchemaCompatibility !== "function" || typeof MigrationHistoryError !== "function") throw new Error("verifier modules unavailable");+ if (!Number.isInteger(manifest.MIN_SUPPORTED_SCHEMA_VERSION) || !Number.isInteger(manifest.MAX_SUPPORTED_SCHEMA_VERSION)) throw new Error("verifier manifest unavailable");+ client = await pool.connect();+ result.minSupported = manifest.MIN_SUPPORTED_SCHEMA_VERSION;+ result.maxSupported = manifest.MAX_SUPPORTED_SCHEMA_VERSION;+ await client.query("BEGIN READ ONLY");+ began = true;+ await client.query("SET LOCAL statement_timeout = '30000ms'");+ const version = await verifySchemaCompatibility(client);+ if (!Number.isInteger(version) || version < result.minSupported || version > result.maxSupported) throw new Error("invalid schema version");+ await client.query("COMMIT");+ committed = true;+ result = { status: "compatible", currentVersion: version, minSupported: result.minSupported, maxSupported: result.maxSupported };+} catch (error) {+ result = typeof MigrationHistoryError === "function" && error instanceof MigrationHistoryError+ ? { status: "incompatible", code: ["schema_uninitialized", "schema_behind", "schema_ahead", "migration_history_invalid"].includes(error.code) ? error.code : "migration_history_invalid", currentVersion: null, minSupported: result.minSupported, maxSupported: result.maxSupported }+ : { status: "unknown", reasonCode: /timeout|deadline/i.test(String(error)) ? "timeout" : (pool ? "connection_failed" : "module_unavailable"), currentVersion: null, minSupported: result.minSupported, maxSupported: result.maxSupported };+ if (began && !committed && client) {+ try { await client.query("ROLLBACK"); } catch {}+ }+} finally {+ try { if (client) client.release(); } catch { result = { status: "unknown", reasonCode: "cleanup_failed", currentVersion: null, minSupported: result.minSupported, maxSupported: result.maxSupported }; }+ try { if (pool) await pool.end(); } catch { result = { status: "unknown", reasonCode: "cleanup_failed", currentVersion: null, minSupported: result.minSupported, maxSupported: result.maxSupported }; }+}+process.stdout.write(marker + " " + JSON.stringify({ protocol: 1, kind: "schema-verifier", ...result }) + "\\n");+`+}++export function buildSchemaVerifierJob(+ baseline: DeploymentSnapshot,+ options: SchemaVerifierJobOptions,+): JsonObject {+ assertSnapshotShape(baseline)+ if (options.image !== baseline.serverImage) {+ throw new Error('schema verifier image must be the captured baseline server image')+ }+ const result = jobBase(baseline.podTemplate, {+ name: options.name,+ image: options.image,+ command: ['node', '--import', 'tsx', '--input-type=module', '-e'],+ args: [buildSchemaVerifierScript()],+ })+ // Cloud SQL proxy startup (up to 60s), image scheduling/pull, the bounded+ // five-second pool connection, and the 30s read-only query all need room+ // before the Job is considered a verifier timeout.+ result.spec.activeDeadlineSeconds = options.activeDeadlineSeconds ?? 180+ result.spec.ttlSecondsAfterFinished = options.ttlSecondsAfterFinished ?? 600+ result.metadata.labels = {+ app: 'cumora-schema-recovery',+ 'recovery.cumora.ai/managed': 'true',+ 'recovery.cumora.ai/read-only': 'true',+ }+ return {+ apiVersion: 'batch/v1',+ kind: 'Job',+ metadata: result.metadata,+ spec: result.spec,+ }+}++function parseMarkerValue(value: unknown): SchemaVerifierResult | null {+ if (!isRecord(value)) return null+ if (value.protocol !== 1 || value.kind !== 'schema-verifier') return null+ const currentVersion = value.currentVersion === undefined ? null : value.currentVersion+ const minSupported = value.minSupported === undefined ? null : value.minSupported+ const maxSupported = value.maxSupported === undefined ? null : value.maxSupported+ if (currentVersion !== null && !Number.isInteger(currentVersion)) return null+ if (minSupported !== null && !Number.isInteger(minSupported)) return null+ if (maxSupported !== null && !Number.isInteger(maxSupported)) return null+ if (value.status === 'compatible') {+ if (+ !Number.isInteger(currentVersion) ||+ !Number.isInteger(minSupported) ||+ !Number.isInteger(maxSupported) ||+ (currentVersion as number) < (minSupported as number) ||+ (currentVersion as number) > (maxSupported as number)+ ) return null+ return {+ protocol: 1,+ kind: 'schema-verifier',+ status: 'compatible',+ currentVersion: currentVersion as number,+ minSupported: minSupported as number,+ maxSupported: maxSupported as number,+ }+ }+ if (value.status === 'incompatible' && typeof value.code === 'string' && SCHEMA_INCOMPATIBLE_CODES.includes(value.code as typeof SCHEMA_INCOMPATIBLE_CODES[number])) {+ return {+ protocol: 1,+ kind: 'schema-verifier',+ status: 'incompatible',+ currentVersion: Number.isInteger(currentVersion) ? currentVersion as number : null,+ minSupported: Number.isInteger(minSupported) ? minSupported as number : null,+ maxSupported: Number.isInteger(maxSupported) ? maxSupported as number : null,+ code: value.code as SchemaVerifierResult['code'],+ }+ }+ if (value.status === 'unknown' && typeof value.reasonCode === 'string' && SCHEMA_UNKNOWN_REASON_CODES.includes(value.reasonCode as typeof SCHEMA_UNKNOWN_REASON_CODES[number])) {+ return {+ protocol: 1,+ kind: 'schema-verifier',+ status: 'unknown',+ currentVersion: Number.isInteger(currentVersion) ? currentVersion as number : null,+ minSupported: Number.isInteger(minSupported) ? minSupported as number : null,+ maxSupported: Number.isInteger(maxSupported) ? maxSupported as number : null,+ reasonCode: value.reasonCode as string,+ }+ }+ return null+}++export function parseSchemaVerifierOutput(stdout: string, stderr = ''): SchemaVerifierResult {+ const combined = `${stdout}${stderr.length > 0 ? `\n${stderr}` : ''}`+ const lines = combined.split(/\r?\n/).map((line) => line.trim()).filter((line) => line.length > 0)+ const markerLines = lines.filter((line) => line.startsWith(SCHEMA_VERIFIER_MARKER))+ if (markerLines.length !== 1) return unknownVerifierResult('bad_output')+ const markerLine = markerLines[0]+ if (!markerLine.startsWith(`${SCHEMA_VERIFIER_MARKER} `)) return unknownVerifierResult('bad_output')+ const payload = markerLine.slice(SCHEMA_VERIFIER_MARKER.length + 1)+ const nonMarkerLines = combined+ .split(/\r?\n/)+ .map((line) => line.trim())+ .filter((line) => line.length > 0 && !line.startsWith(`${SCHEMA_VERIFIER_MARKER} `))+ if (nonMarkerLines.length > 0) return unknownVerifierResult('bad_output')+ try {+ const value = JSON.parse(payload) as unknown+ return parseMarkerValue(value) ?? unknownVerifierResult('protocol_error')+ } catch {+ return unknownVerifierResult('bad_output')+ }+}++function unknownVerifierResult(reasonCode: SchemaVerifierResult['reasonCode']): SchemaVerifierResult {+ return {+ protocol: 1,+ kind: 'schema-verifier',+ status: 'unknown',+ currentVersion: null,+ minSupported: null,+ maxSupported: null,+ reasonCode,+ }+}++export function assertCompatibleVerifierResult(result: SchemaVerifierResult): asserts result is SchemaVerifierResult & { status: 'compatible'; currentVersion: number; minSupported: number; maxSupported: number } {+ if (+ result.protocol !== 1 ||+ result.kind !== 'schema-verifier' ||+ result.status !== 'compatible' ||+ !Number.isInteger(result.currentVersion) ||+ !Number.isInteger(result.minSupported) ||+ !Number.isInteger(result.maxSupported) ||+ (result.currentVersion as number) < (result.minSupported as number) ||+ (result.currentVersion as number) > (result.maxSupported as number)+ ) {+ throw new Error(`old image schema verifier did not prove compatibility (${result.status})`)+ }+}++export function assertDeploymentReceiptMatches(+ baseline: DeploymentSnapshot,+ current: DeploymentReceipt,+ expectedCandidateHash: string,+): void {+ assertSnapshotShape(baseline)+ assertReceiptShape(current)+ if (current.uid !== baseline.uid) throw new Error('Deployment UID changed since baseline capture')+ if (current.podTemplateHash !== expectedCandidateHash) {+ throw new Error('Deployment template changed since candidate patch')+ }+}++export function assertCandidateReceiptMatches(+ baseline: DeploymentSnapshot,+ receipt: DeploymentReceipt,+ images: CandidateImages,+): void {+ assertSnapshotShape(baseline)+ assertReceiptShape(receipt)+ if (receipt.uid !== baseline.uid || receipt.name !== baseline.name || receipt.namespace !== baseline.namespace) {+ throw new Error('candidate receipt Deployment identity changed')+ }+ if (receipt.serverImage !== images.server) throw new Error('candidate receipt server image does not match selected digest')+ const expectedAgent = images.agent ?? baseline.agentImage+ if (receipt.agentImage !== expectedAgent) throw new Error('candidate receipt agent image does not match selected digest')+ if (images.agent === undefined && receipt.agentImageSource !== baseline.agentImageSource) {+ throw new Error('candidate receipt agent image source changed')+ }+}