Skip to content

Where agent teams gather. Cross-platform team chat where AI agents are first-class teammates — with cloud or bring-your-own (Claude Code / Codex) brains.

TypeScript
34.8 KB
Raw Blame
import { createHash } from 'node:crypto'

/**
 * Deployment recovery is deliberately a data-only module.  The workflow owns
 * kubectl, while this module owns the objects passed to kubectl and all of the
 * compare-and-swap rules.  Keeping those rules here lets tests exercise the
 * exact patch and Job objects used in production without contacting a cluster.
 */

export type JsonPrimitive = string | number | boolean | null
export type JsonValue = unknown
export type JsonObject = Record<string, unknown>

export interface PodTemplateSpec {
  containers: JsonObject[]
  initContainers?: JsonObject[]
  serviceAccountName?: string
  imagePullSecrets?: JsonValue
  securityContext?: JsonValue
  volumes?: JsonValue
  [key: string]: unknown
}

export interface PodTemplate {
  metadata: JsonObject
  spec: PodTemplateSpec
  [key: string]: unknown
}

export interface JsonPatchOperation {
  op: 'test' | 'replace'
  path: string
  value: JsonValue
}

export interface DeploymentHealth {
  generation: number | null
  observedGeneration: number | null
  desiredReplicas: number | null
  updatedReplicas: number | null
  readyReplicas: number | null
  availableReplicas: number | null
}

export interface DeploymentSnapshot {
  apiVersion: string
  kind: 'Deployment'
  name: string
  namespace: string
  uid: string
  podTemplate: PodTemplate
  podTemplateHash: string
  serverImage: string
  agentImage: string | null
  agentImageSource: 'missing' | 'inline' | 'unknown'
  baselineImage: { server: string; agent: string | null }
  health: DeploymentHealth
  baselineSmokePassed?: boolean
  baselineVerifiedAt?: string
  capturedAt: string
}

export interface DeploymentReceipt {
  uid: string
  name: string
  namespace: string
  podTemplate: PodTemplate
  podTemplateHash: string
  serverImage: string
  agentImage: string | null
  agentImageSource: 'missing' | 'inline' | 'unknown'
  baselineImage: { server: string; agent: string | null }
  observedAt: string
}

export interface CandidateImages {
  server: string
  /** Omit to preserve the baseline agent image. */
  agent?: string
}

export interface MigrationJobOptions {
  name: string
  image: string
  repairMode: 'off' | 'archive-detach'
  activeDeadlineSeconds?: number
  ttlSecondsAfterFinished?: number
}

export interface SchemaVerifierJobOptions {
  name: string
  image: string
  activeDeadlineSeconds?: number
  ttlSecondsAfterFinished?: number
}

export interface SchemaVerifierResult {
  [key: string]: unknown
  protocol: 1
  kind: 'schema-verifier'
  status: 'compatible' | 'incompatible' | 'unknown'
  currentVersion: number | null
  minSupported: number | null
  maxSupported: number | null
  code?: 'schema_uninitialized' | 'schema_behind' | 'schema_ahead' | 'migration_history_invalid'
  reasonCode?: string
}

export const SCHEMA_VERIFIER_MARKER = 'CUMORA_SCHEMA_VERIFY_RESULT'

export const SCHEMA_INCOMPATIBLE_CODES = [
  'schema_uninitialized',
  'schema_behind',
  'schema_ahead',
  'migration_history_invalid',
] as const

export const SCHEMA_UNKNOWN_REASON_CODES = [
  'module_unavailable',
  'connection_failed',
  'timeout',
  'cleanup_failed',
] as const

const DIGEST_IMAGE_PATTERN = /@sha256:[a-f0-9]{64}$/

export const DEPLOYMENT_PROBE_CONTRACT = {
  startupProbe: {
    httpGet: { path: '/api/livez', port: 'http' },
    periodSeconds: 5,
    timeoutSeconds: 2,
    failureThreshold: 60,
  },
  readinessProbe: {
    httpGet: { path: '/api/health', port: 'http' },
    periodSeconds: 5,
    timeoutSeconds: 2,
    // /api/health races SELECT 1 against 1s, so a saturated Postgres fails a
    // random fraction of probes on every replica at once. 60s of consecutive
    // failures still pulls a pod; a flap must not (see the manifests).
    failureThreshold: 12,
  },
  livenessProbe: {
    httpGet: { path: '/api/livez', port: 'http' },
    periodSeconds: 30,
    timeoutSeconds: 3,
  },
} as const

function isRecord(value: unknown): value is Record<string, unknown> {
  return typeof value === 'object' && value !== null && !Array.isArray(value)
}

function isJsonValue(value: unknown): value is JsonValue {
  if (value === null || typeof value === 'string' || typeof value === 'number' || typeof value === 'boolean') {
    return true
  }
  if (Array.isArray(value)) return value.every(isJsonValue)
  return isRecord(value) && Object.values(value).every(isJsonValue)
}

function asJsonObject(value: unknown, label: string): JsonObject {
  if (!isRecord(value) || !isJsonValue(value)) throw new Error(`${label} must be a JSON object`)
  return value as JsonObject
}

function cloneJson<T>(value: T): T {
  return JSON.parse(JSON.stringify(value)) as T
}

/** Sort object keys recursively while preserving array order. */
export function normalizeJson(value: unknown): unknown {
  if (Array.isArray(value)) return value.map(normalizeJson)
  if (!isRecord(value)) return value
  return Object.fromEntries(
    Object.entries(value)
      .sort(([left], [right]) => left.localeCompare(right))
      .map(([key, child]) => [key, normalizeJson(child)]),
  ) as JsonObject
}

export function hashJson(value: unknown): string {
  return createHash('sha256').update(JSON.stringify(normalizeJson(value))).digest('hex')
}

export function hashPodTemplate(template: JsonObject | PodTemplate): string {
  return hashJson(template)
}

function readString(value: unknown, label: string): string {
  if (typeof value !== 'string' || value.length === 0) throw new Error(`${label} must be a non-empty string`)
  return value
}

function integerOrNull(value: unknown): number | null {
  return typeof value === 'number' && Number.isInteger(value) ? value : null
}

function assertDigestImage(value: unknown, label: string): string {
  const image = readString(value, label)
  if (!DIGEST_IMAGE_PATTERN.test(image)) throw new Error(`${label} must be pinned by sha256 digest`)
  return image
}

function getObjectPath(root: unknown, path: string[], label: string): Record<string, unknown> {
  let current: unknown = root
  for (const segment of path) {
    if (!isRecord(current) || !(segment in current)) throw new Error(`${label} is missing ${path.join('.')}`)
    current = current[segment]
  }
  if (!isRecord(current)) throw new Error(`${label} ${path.join('.')} must be an object`)
  return current
}

function getArrayPath(root: unknown, path: string[], label: string): unknown[] {
  let current: unknown = root
  for (const segment of path) {
    if (!isRecord(current) || !(segment in current)) throw new Error(`${label} is missing ${path.join('.')}`)
    current = current[segment]
  }
  if (!Array.isArray(current)) throw new Error(`${label} ${path.join('.')} must be an array`)
  return current
}

function findNamedObject(items: unknown[], name: string, label: string): Record<string, unknown> {
  const found = items.find((item) => isRecord(item) && item.name === name)
  if (!isRecord(found)) throw new Error(`${label} is missing ${name}`)
  return found
}

function agentImageValue(container: Record<string, unknown>, name: string): { value: string | null; source: 'missing' | 'inline' | 'unknown' } {
  const env = container.env
  if (!Array.isArray(env)) return { value: null, source: 'missing' }
  const entry = env.find((item) => isRecord(item) && item.name === name)
  if (!isRecord(entry)) return { value: null, source: 'missing' }
  if (typeof entry.value !== 'string') return { value: null, source: 'unknown' }
  return { value: entry.value, source: 'inline' }
}

function deploymentPodTemplate(deployment: unknown): PodTemplate {
  const template = asJsonObject(getObjectPath(deployment, ['spec', 'template'], 'Deployment'), 'Deployment pod template')
  if (!isRecord(template.metadata) || !isRecord(template.spec)) throw new Error('Deployment pod template must contain metadata and spec')
  if (!Array.isArray(template.spec.containers)) throw new Error('Deployment pod template spec.containers must be an array')
  return template as PodTemplate
}

function deploymentServerContainer(template: JsonObject): JsonObject {
  const containers = getArrayPath(template, ['spec', 'containers'], 'Deployment pod template')
  return asJsonObject(findNamedObject(containers, 'server', 'Deployment pod template containers'), 'server container')
}

export function extractDeploymentSnapshot(
  deployment: unknown,
  opts: { expectedName?: string; expectedNamespace?: string; capturedAt?: string } = {},
): DeploymentSnapshot {
  const metadata = getObjectPath(deployment, ['metadata'], 'Deployment')
  const template = deploymentPodTemplate(deployment)
  const server = deploymentServerContainer(template)
  const uid = readString(metadata.uid, 'Deployment metadata.uid')
  const apiVersion = readString((deployment as Record<string, unknown>).apiVersion, 'Deployment apiVersion')
  if (apiVersion !== 'apps/v1') throw new Error(`unsupported Deployment apiVersion: ${apiVersion}`)
  const kind = readString((deployment as Record<string, unknown>).kind, 'Deployment kind')
  if (kind !== 'Deployment') throw new Error(`expected Deployment, got ${kind}`)
  const name = readString(metadata.name, 'Deployment metadata.name')
  // kubectl includes metadata.namespace.  Fixtures and an unqualified local
  // object can omit it; Kubernetes resolves those to the default namespace.
  const namespace = typeof metadata.namespace === 'string' && metadata.namespace.length > 0 ? metadata.namespace : 'default'
  if (opts.expectedName && name !== opts.expectedName) throw new Error(`unexpected Deployment name: ${name}`)
  if (opts.expectedNamespace && namespace !== opts.expectedNamespace) throw new Error(`unexpected Deployment namespace: ${namespace}`)
  const serverImage = assertDigestImage(server.image, 'server container image')
  const agent = agentImageValue(server, 'CUMORA_AGENT_COMPUTER_IMAGE')
  if (agent.value !== null) assertDigestImage(agent.value, 'agent container image')
  return {
    apiVersion,
    kind: 'Deployment',
    name,
    namespace,
    uid,
    podTemplate: cloneJson(template),
    podTemplateHash: hashPodTemplate(template),
    serverImage,
    agentImage: agent.value,
    agentImageSource: agent.source,
    baselineImage: { server: serverImage, agent: agent.value },
    health: {
      generation: integerOrNull(metadata.generation),
      observedGeneration: integerOrNull(isRecord((deployment as Record<string, unknown>).status) ? (deployment as Record<string, Record<string, unknown>>).status.observedGeneration : null),
      desiredReplicas: integerOrNull(isRecord((deployment as Record<string, unknown>).spec) ? (deployment as Record<string, Record<string, unknown>>).spec.replicas : null),
      updatedReplicas: integerOrNull(isRecord((deployment as Record<string, unknown>).status) ? (deployment as Record<string, Record<string, unknown>>).status.updatedReplicas : null),
      readyReplicas: integerOrNull(isRecord((deployment as Record<string, unknown>).status) ? (deployment as Record<string, Record<string, unknown>>).status.readyReplicas : null),
      availableReplicas: integerOrNull(isRecord((deployment as Record<string, unknown>).status) ? (deployment as Record<string, Record<string, unknown>>).status.availableReplicas : null),
    },
    capturedAt: opts.capturedAt ?? new Date().toISOString(),
  }
}

export function deploymentReceiptFromObject(
  deployment: unknown,
  opts: { observedAt?: string } = {},
): DeploymentReceipt {
  const snapshot = extractDeploymentSnapshot(deployment, { capturedAt: opts.observedAt })
  return {
    uid: snapshot.uid,
    name: snapshot.name,
    namespace: snapshot.namespace,
    podTemplate: snapshot.podTemplate,
    podTemplateHash: snapshot.podTemplateHash,
    serverImage: snapshot.serverImage,
    agentImage: snapshot.agentImage,
    agentImageSource: snapshot.agentImageSource,
    baselineImage: snapshot.baselineImage,
    observedAt: snapshot.capturedAt,
  }
}

function assertSnapshotShape(snapshot: DeploymentSnapshot): void {
  if (snapshot.kind !== 'Deployment') throw new Error('recovery baseline must describe a Deployment')
  if (snapshot.apiVersion !== 'apps/v1') throw new Error('recovery baseline must use apps/v1')
  readString(snapshot.uid, 'recovery baseline uid')
  readString(snapshot.name, 'recovery baseline name')
  readString(snapshot.namespace, 'recovery baseline namespace')
  if (!isRecord(snapshot.podTemplate)) throw new Error('recovery baseline podTemplate must be an object')
  const actualHash = hashPodTemplate(snapshot.podTemplate)
  if (actualHash !== snapshot.podTemplateHash) {
    throw new Error('recovery baseline podTemplateHash does not match podTemplate')
  }
  assertDigestImage(snapshot.serverImage, 'recovery baseline serverImage')
  if (snapshot.agentImage !== null && typeof snapshot.agentImage !== 'string') {
    throw new Error('recovery baseline agentImage must be a string or null')
  }
  if (snapshot.agentImage !== null) assertDigestImage(snapshot.agentImage, 'recovery baseline agentImage')
  if (!['missing', 'inline', 'unknown'].includes(snapshot.agentImageSource)) throw new Error('invalid recovery baseline agent image source')
  if (snapshot.baselineImage?.server !== snapshot.serverImage || snapshot.baselineImage?.agent !== snapshot.agentImage) {
    throw new Error('recovery baseline baselineImage does not match image fields')
  }
  const server = deploymentServerContainer(snapshot.podTemplate)
  if (server.image !== snapshot.serverImage) throw new Error('recovery baseline server image does not match podTemplate')
  const agent = agentImageValue(server, 'CUMORA_AGENT_COMPUTER_IMAGE')
  if (agent.value !== snapshot.agentImage || agent.source !== snapshot.agentImageSource) {
    throw new Error('recovery baseline agent image does not match podTemplate')
  }
}

export function assertBaselineUnchanged(existing: DeploymentSnapshot, candidate: DeploymentSnapshot | DeploymentReceipt): void {
  assertSnapshotShape(existing)
  if ('apiVersion' in candidate) assertSnapshotShape(candidate)
  else assertReceiptShape(candidate)
  if (existing.uid !== candidate.uid || existing.podTemplateHash !== candidate.podTemplateHash) {
    throw new Error('recovery baseline already exists and differs; refusing to replace it')
  }
}

export function assertDeploymentHealthy(snapshot: DeploymentSnapshot): void {
  const health = snapshot.health
  if (!health || health.generation === null || health.observedGeneration !== health.generation) {
    throw new Error('Deployment baseline rollout is not observed')
  }
  if (
    health.desiredReplicas === null ||
    health.updatedReplicas !== health.desiredReplicas ||
    health.readyReplicas !== health.desiredReplicas ||
    health.availableReplicas !== health.desiredReplicas
  ) {
    throw new Error('Deployment baseline rollout replicas are not healthy')
  }
}

export function assertRecoveryEligible(baseline: DeploymentSnapshot): void {
  assertDeploymentHealthy(baseline)
  if (baseline.baselineSmokePassed !== true) {
    throw new Error('Deployment baseline smoke was not proven before migration')
  }
}

function patchContainer(
  template: PodTemplate,
  images: CandidateImages,
): PodTemplate {
  assertDigestImage(images.server, 'candidate server image')
  if (images.agent !== undefined) assertDigestImage(images.agent, 'candidate agent image')
  const patched = cloneJson(template)
  const spec = patched.spec
  const containers = spec.containers
  const server = asJsonObject(findNamedObject(containers, 'server', 'pod template containers'), 'server container')
  server.image = images.server
  server.startupProbe = cloneJson(DEPLOYMENT_PROBE_CONTRACT.startupProbe as unknown as JsonObject)
  server.readinessProbe = cloneJson(DEPLOYMENT_PROBE_CONTRACT.readinessProbe as unknown as JsonObject)
  server.livenessProbe = cloneJson(DEPLOYMENT_PROBE_CONTRACT.livenessProbe as unknown as JsonObject)

  if (images.agent !== undefined) {
    const env = Array.isArray(server.env) ? server.env.filter((entry) => !(isRecord(entry) && entry.name === 'CUMORA_AGENT_COMPUTER_IMAGE')) : []
    env.push({ name: 'CUMORA_AGENT_COMPUTER_IMAGE', value: images.agent })
    server.env = env as JsonValue
  }

  if (Array.isArray(spec.initContainers)) {
    spec.initContainers = spec.initContainers.filter((entry) => !(isRecord(entry) && entry.name === 'migrate')) as JsonObject[]
  }
  return patched as PodTemplate
}

export function buildCandidateTemplate(baseline: DeploymentSnapshot, images: CandidateImages): PodTemplate {
  assertSnapshotShape(baseline)
  return patchContainer(baseline.podTemplate, images)
}

function jsonPatchTest(path: string, value: JsonValue): JsonPatchOperation {
  return { op: 'test', path, value: cloneJson(value) }
}

function jsonPatchReplace(path: string, value: JsonValue): JsonPatchOperation {
  return { op: 'replace', path, value: cloneJson(value) }
}

export function buildCandidatePatch(
  baseline: DeploymentSnapshot,
  images: CandidateImages,
): JsonPatchOperation[] {
  assertSnapshotShape(baseline)
  const candidate = buildCandidateTemplate(baseline, images)
  return [
    jsonPatchTest('/metadata/uid', baseline.uid),
    jsonPatchTest('/spec/template', baseline.podTemplate),
    jsonPatchReplace('/spec/template', candidate),
  ]
}

export function buildRestorePatch(
  baseline: DeploymentSnapshot,
  current: DeploymentReceipt,
  expectedCandidateHash: string,
): JsonPatchOperation[] {
  assertSnapshotShape(baseline)
  assertReceiptShape(current)
  if (baseline.agentImageSource === 'unknown' || current.agentImageSource === 'unknown') {
    throw new Error('agent image valueFrom is not a safe automatic recovery baseline')
  }
  if (current.uid !== baseline.uid) throw new Error('Deployment UID changed; refusing automatic recovery')
  if (current.podTemplateHash !== expectedCandidateHash) {
    throw new Error('Deployment pod template drifted from the candidate receipt; refusing automatic recovery')
  }
  return [
    jsonPatchTest('/metadata/uid', baseline.uid),
    jsonPatchTest('/spec/template', current.podTemplate),
    jsonPatchReplace('/spec/template', baseline.podTemplate),
  ]
}

function assertReceiptShape(receipt: DeploymentReceipt): void {
  readString(receipt.uid, 'candidate receipt uid')
  readString(receipt.name, 'candidate receipt name')
  readString(receipt.namespace, 'candidate receipt namespace')
  if (!isRecord(receipt.podTemplate)) throw new Error('candidate receipt podTemplate must be an object')
  if (hashPodTemplate(receipt.podTemplate) !== receipt.podTemplateHash) {
    throw new Error('candidate receipt podTemplateHash does not match podTemplate')
  }
  assertDigestImage(receipt.serverImage, 'candidate receipt serverImage')
  if (receipt.agentImage !== null) assertDigestImage(receipt.agentImage, 'candidate receipt agentImage')
  if (!['missing', 'inline', 'unknown'].includes(receipt.agentImageSource)) throw new Error('invalid candidate receipt agent image source')
  if (receipt.baselineImage?.server !== receipt.serverImage || receipt.baselineImage?.agent !== receipt.agentImage) {
    throw new Error('candidate receipt baselineImage does not match image fields')
  }
  const server = deploymentServerContainer(receipt.podTemplate)
  if (server.image !== receipt.serverImage) throw new Error('candidate receipt server image does not match podTemplate')
  const agent = agentImageValue(server, 'CUMORA_AGENT_COMPUTER_IMAGE')
  if (agent.value !== receipt.agentImage || agent.source !== receipt.agentImageSource) {
    throw new Error('candidate receipt agent image does not match podTemplate')
  }
}

export function assertReceiptShapeForRecovery(receipt: DeploymentReceipt): void {
  assertReceiptShape(receipt)
}

export function assertRecoveryStateBinding(
  baseline: DeploymentSnapshot,
  receipt: DeploymentReceipt,
): void {
  assertSnapshotShape(baseline)
  assertReceiptShape(receipt)
  if (baseline.uid !== receipt.uid || baseline.name !== receipt.name || baseline.namespace !== receipt.namespace) {
    throw new Error('protected recovery receipt is bound to a different Deployment')
  }
}

function cleanContainerForJob(container: JsonObject): JsonObject {
  const result = cloneJson(container)
  for (const field of [
    'ports',
    'startupProbe',
    'readinessProbe',
    'livenessProbe',
    'lifecycle',
    'terminationMessagePath',
    'terminationMessagePolicy',
  ]) delete result[field]
  return result
}

/**
 * A probe copied out of the Deployment may target a *named* port, but the Job's
 * containers carry no `ports` list (cleanContainerForJob strips it), so the name
 * has nothing to resolve against.  kubelet then falls back to parsing the name
 * as a number, the probe errors permanently ("strconv.Atoi: parsing ..."), the
 * native sidecar never reports `started`, and the migrate container is never
 * launched at all — the Job silently burns its whole activeDeadlineSeconds and
 * dies as DeadlineExceeded with no logs.  Resolve the name to its number here,
 * while the source container's port list is still in hand, and return null when
 * it cannot be resolved so the caller uses its own numeric probe rather than one
 * that can never pass.
 */
function resolveProbePorts(probe: unknown, ports: unknown): JsonValue | null {
  if (!isRecord(probe) || !isJsonValue(probe)) return null
  const resolved = cloneJson(probe) as JsonObject
  for (const key of ['httpGet', 'tcpSocket']) {
    const target = resolved[key]
    if (!isRecord(target) || typeof target.port !== 'string') continue
    const declared = Array.isArray(ports)
      ? ports.find((entry) => isRecord(entry) && entry.name === target.port && typeof entry.containerPort === 'number')
      : undefined
    if (!isRecord(declared)) return null
    target.port = declared.containerPort
  }
  return resolved
}

function podSpecForJob(template: JsonObject): { pod: JsonObject; server: JsonObject; proxy: JsonObject; proxyStartupProbe: JsonValue } {
  const spec = asJsonObject(getObjectPath(template, ['spec'], 'pod template'), 'pod template spec')
  const allInit = Array.isArray(spec.initContainers) ? spec.initContainers : []
  const allContainers = getArrayPath(spec, ['containers'], 'pod template')
  const proxySource = [...allInit, ...allContainers].find((item) => isRecord(item) && item.name === 'cloud-sql-proxy')
  if (!isRecord(proxySource)) throw new Error('Deployment pod template has no cloud-sql-proxy container')
  const server = findNamedObject(allContainers, 'server', 'pod template containers')
  const pod = cloneJson(spec)
  delete pod.containers
  delete pod.initContainers
  delete pod.ephemeralContainers
  const proxyStartupProbe = resolveProbePorts(proxySource.startupProbe, proxySource.ports) ?? {
    httpGet: { path: '/readiness', port: 9090 },
    periodSeconds: 1,
    failureThreshold: 60,
  }
  return { pod, server: cleanContainerForJob(server), proxy: cleanContainerForJob(proxySource), proxyStartupProbe }
}

function upsertEnv(container: JsonObject, name: string, value: string): void {
  const env = Array.isArray(container.env) ? container.env.filter((entry) => !(isRecord(entry) && entry.name === name)) : []
  env.push({ name, value })
  container.env = env as JsonValue
}

function jobBase(
  template: JsonObject,
  options: { name: string; image: string; command: string[]; args?: string[] },
): { metadata: JsonObject; spec: JsonObject; pod: JsonObject; server: JsonObject; proxy: JsonObject } {
  const { pod, server, proxy, proxyStartupProbe } = podSpecForJob(template)
  server.name = 'migrate'
  server.image = options.image
  server.imagePullPolicy = 'IfNotPresent'
  server.command = options.command as unknown as JsonValue
  delete server.args
  if (options.args) server.args = options.args as unknown as JsonValue
  server.workingDir = '/app'
  pod.restartPolicy = 'Never'
  pod.initContainers = [
    {
      ...proxy,
      name: 'cloud-sql-proxy',
      restartPolicy: 'Always',
      startupProbe: proxyStartupProbe,
    },
  ] as unknown as JsonValue
  pod.containers = [server] as unknown as JsonValue
  return {
    metadata: {
      name: options.name,
      labels: { app: 'cumora-schema-recovery' },
    },
    spec: {
      backoffLimit: 0,
      activeDeadlineSeconds: 600,
      ttlSecondsAfterFinished: 600,
      template: {
        metadata: { labels: { app: 'cumora-schema-recovery', 'recovery.cumora.ai/managed': 'true' } },
        spec: pod,
      },
    },
    pod,
    server,
    proxy,
  }
}

export function buildMigrationJob(
  baseline: DeploymentSnapshot,
  options: MigrationJobOptions,
): JsonObject {
  assertSnapshotShape(baseline)
  assertDigestImage(options.image, 'migration image')
  if (!['off', 'archive-detach'].includes(options.repairMode)) throw new Error('invalid migration repair mode')
  const result = jobBase(baseline.podTemplate, {
    name: options.name,
    image: options.image,
    command: ['npm', 'run', 'migrate'],
  })
  result.spec.activeDeadlineSeconds = options.activeDeadlineSeconds ?? 600
  result.spec.ttlSecondsAfterFinished = options.ttlSecondsAfterFinished ?? 600
  upsertEnv(result.server, 'MIGRATION_0002_REPAIR', options.repairMode)
  return {
    apiVersion: 'batch/v1',
    kind: 'Job',
    metadata: result.metadata,
    spec: result.spec,
  }
}

/**
 * This is injected into the old server image during recovery.  It imports the
 * old image's own pool, schema gate, and manifest, so a newer checkout cannot
 * accidentally decide that an old image is compatible.  The program writes a
 * single marker line and nothing else; the parser below fails closed on any
 * extra or malformed output.
 */
export function buildSchemaVerifierScript(): string {
  const marker = JSON.stringify(SCHEMA_VERIFIER_MARKER)
  return `
const marker = ${marker};
let result = { status: "unknown", reasonCode: "module_unavailable", currentVersion: null, minSupported: null, maxSupported: null };
let client;
let pool;
let MigrationHistoryError;
let manifest;
let began = false;
let committed = false;
try {
  const [poolModule, schemaModule, manifestModule] = await Promise.all([
    import("./server/src/db/pool.ts"),
    import("./server/src/db/schema-version.ts"),
    import("./server/src/db/migrations/manifest.ts"),
  ]);
  pool = poolModule.pool;
  const verifySchemaCompatibility = schemaModule.verifySchemaCompatibility;
  MigrationHistoryError = manifestModule.MigrationHistoryError;
  manifest = manifestModule;
  if (!pool || typeof pool.connect !== "function" || typeof verifySchemaCompatibility !== "function" || typeof MigrationHistoryError !== "function") throw new Error("verifier modules unavailable");
  if (!Number.isInteger(manifest.MIN_SUPPORTED_SCHEMA_VERSION) || !Number.isInteger(manifest.MAX_SUPPORTED_SCHEMA_VERSION)) throw new Error("verifier manifest unavailable");
  client = await pool.connect();
  result.minSupported = manifest.MIN_SUPPORTED_SCHEMA_VERSION;
  result.maxSupported = manifest.MAX_SUPPORTED_SCHEMA_VERSION;
  await client.query("BEGIN READ ONLY");
  began = true;
  await client.query("SET LOCAL statement_timeout = '30000ms'");
  const version = await verifySchemaCompatibility(client);
  if (!Number.isInteger(version) || version < result.minSupported || version > result.maxSupported) throw new Error("invalid schema version");
  await client.query("COMMIT");
  committed = true;
  result = { status: "compatible", currentVersion: version, minSupported: result.minSupported, maxSupported: result.maxSupported };
} catch (error) {
  result = typeof MigrationHistoryError === "function" && error instanceof MigrationHistoryError
    ? { status: "incompatible", code: ["schema_uninitialized", "schema_behind", "schema_ahead", "migration_history_invalid"].includes(error.code) ? error.code : "migration_history_invalid", currentVersion: null, minSupported: result.minSupported, maxSupported: result.maxSupported }
    : { status: "unknown", reasonCode: /timeout|deadline/i.test(String(error)) ? "timeout" : (pool ? "connection_failed" : "module_unavailable"), currentVersion: null, minSupported: result.minSupported, maxSupported: result.maxSupported };
  if (began && !committed && client) {
    try { await client.query("ROLLBACK"); } catch {}
  }
} finally {
  try { if (client) client.release(); } catch { result = { status: "unknown", reasonCode: "cleanup_failed", currentVersion: null, minSupported: result.minSupported, maxSupported: result.maxSupported }; }
  try { if (pool) await pool.end(); } catch { result = { status: "unknown", reasonCode: "cleanup_failed", currentVersion: null, minSupported: result.minSupported, maxSupported: result.maxSupported }; }
}
process.stdout.write(marker + " " + JSON.stringify({ protocol: 1, kind: "schema-verifier", ...result }) + "\\n");
`
}

export function buildSchemaVerifierJob(
  baseline: DeploymentSnapshot,
  options: SchemaVerifierJobOptions,
): JsonObject {
  assertSnapshotShape(baseline)
  if (options.image !== baseline.serverImage) {
    throw new Error('schema verifier image must be the captured baseline server image')
  }
  const result = jobBase(baseline.podTemplate, {
    name: options.name,
    image: options.image,
    command: ['node', '--import', 'tsx', '--input-type=module', '-e'],
    args: [buildSchemaVerifierScript()],
  })
  // Cloud SQL proxy startup (up to 60s), image scheduling/pull, the bounded
  // five-second pool connection, and the 30s read-only query all need room
  // before the Job is considered a verifier timeout.
  result.spec.activeDeadlineSeconds = options.activeDeadlineSeconds ?? 180
  result.spec.ttlSecondsAfterFinished = options.ttlSecondsAfterFinished ?? 600
  result.metadata.labels = {
    app: 'cumora-schema-recovery',
    'recovery.cumora.ai/managed': 'true',
    'recovery.cumora.ai/read-only': 'true',
  }
  return {
    apiVersion: 'batch/v1',
    kind: 'Job',
    metadata: result.metadata,
    spec: result.spec,
  }
}

function parseMarkerValue(value: unknown): SchemaVerifierResult | null {
  if (!isRecord(value)) return null
  if (value.protocol !== 1 || value.kind !== 'schema-verifier') return null
  const currentVersion = value.currentVersion === undefined ? null : value.currentVersion
  const minSupported = value.minSupported === undefined ? null : value.minSupported
  const maxSupported = value.maxSupported === undefined ? null : value.maxSupported
  if (currentVersion !== null && !Number.isInteger(currentVersion)) return null
  if (minSupported !== null && !Number.isInteger(minSupported)) return null
  if (maxSupported !== null && !Number.isInteger(maxSupported)) return null
  if (value.status === 'compatible') {
    if (
      !Number.isInteger(currentVersion) ||
      !Number.isInteger(minSupported) ||
      !Number.isInteger(maxSupported) ||
      (currentVersion as number) < (minSupported as number) ||
      (currentVersion as number) > (maxSupported as number)
    ) return null
    return {
      protocol: 1,
      kind: 'schema-verifier',
      status: 'compatible',
      currentVersion: currentVersion as number,
      minSupported: minSupported as number,
      maxSupported: maxSupported as number,
    }
  }
  if (value.status === 'incompatible' && typeof value.code === 'string' && SCHEMA_INCOMPATIBLE_CODES.includes(value.code as typeof SCHEMA_INCOMPATIBLE_CODES[number])) {
    return {
      protocol: 1,
      kind: 'schema-verifier',
      status: 'incompatible',
      currentVersion: Number.isInteger(currentVersion) ? currentVersion as number : null,
      minSupported: Number.isInteger(minSupported) ? minSupported as number : null,
      maxSupported: Number.isInteger(maxSupported) ? maxSupported as number : null,
      code: value.code as SchemaVerifierResult['code'],
    }
  }
  if (value.status === 'unknown' && typeof value.reasonCode === 'string' && SCHEMA_UNKNOWN_REASON_CODES.includes(value.reasonCode as typeof SCHEMA_UNKNOWN_REASON_CODES[number])) {
    return {
      protocol: 1,
      kind: 'schema-verifier',
      status: 'unknown',
      currentVersion: Number.isInteger(currentVersion) ? currentVersion as number : null,
      minSupported: Number.isInteger(minSupported) ? minSupported as number : null,
      maxSupported: Number.isInteger(maxSupported) ? maxSupported as number : null,
      reasonCode: value.reasonCode as string,
    }
  }
  return null
}

export function parseSchemaVerifierOutput(stdout: string, stderr = ''): SchemaVerifierResult {
  const combined = `${stdout}${stderr.length > 0 ? `\n${stderr}` : ''}`
  const lines = combined.split(/\r?\n/).map((line) => line.trim()).filter((line) => line.length > 0)
  const markerLines = lines.filter((line) => line.startsWith(SCHEMA_VERIFIER_MARKER))
  if (markerLines.length !== 1) return unknownVerifierResult('bad_output')
  const markerLine = markerLines[0]
  if (!markerLine.startsWith(`${SCHEMA_VERIFIER_MARKER} `)) return unknownVerifierResult('bad_output')
  const payload = markerLine.slice(SCHEMA_VERIFIER_MARKER.length + 1)
  const nonMarkerLines = combined
    .split(/\r?\n/)
    .map((line) => line.trim())
    .filter((line) => line.length > 0 && !line.startsWith(`${SCHEMA_VERIFIER_MARKER} `))
  if (nonMarkerLines.length > 0) return unknownVerifierResult('bad_output')
  try {
    const value = JSON.parse(payload) as unknown
    return parseMarkerValue(value) ?? unknownVerifierResult('protocol_error')
  } catch {
    return unknownVerifierResult('bad_output')
  }
}

function unknownVerifierResult(reasonCode: SchemaVerifierResult['reasonCode']): SchemaVerifierResult {
  return {
    protocol: 1,
    kind: 'schema-verifier',
    status: 'unknown',
    currentVersion: null,
    minSupported: null,
    maxSupported: null,
    reasonCode,
  }
}

export function assertCompatibleVerifierResult(result: SchemaVerifierResult): asserts result is SchemaVerifierResult & { status: 'compatible'; currentVersion: number; minSupported: number; maxSupported: number } {
  if (
    result.protocol !== 1 ||
    result.kind !== 'schema-verifier' ||
    result.status !== 'compatible' ||
    !Number.isInteger(result.currentVersion) ||
    !Number.isInteger(result.minSupported) ||
    !Number.isInteger(result.maxSupported) ||
    (result.currentVersion as number) < (result.minSupported as number) ||
    (result.currentVersion as number) > (result.maxSupported as number)
  ) {
    throw new Error(`old image schema verifier did not prove compatibility (${result.status})`)
  }
}

export function assertDeploymentReceiptMatches(
  baseline: DeploymentSnapshot,
  current: DeploymentReceipt,
  expectedCandidateHash: string,
): void {
  assertSnapshotShape(baseline)
  assertReceiptShape(current)
  if (current.uid !== baseline.uid) throw new Error('Deployment UID changed since baseline capture')
  if (current.podTemplateHash !== expectedCandidateHash) {
    throw new Error('Deployment template changed since candidate patch')
  }
}

export function assertCandidateReceiptMatches(
  baseline: DeploymentSnapshot,
  receipt: DeploymentReceipt,
  images: CandidateImages,
): void {
  assertSnapshotShape(baseline)
  assertReceiptShape(receipt)
  if (receipt.uid !== baseline.uid || receipt.name !== baseline.name || receipt.namespace !== baseline.namespace) {
    throw new Error('candidate receipt Deployment identity changed')
  }
  if (receipt.serverImage !== images.server) throw new Error('candidate receipt server image does not match selected digest')
  const expectedAgent = images.agent ?? baseline.agentImage
  if (receipt.agentImage !== expectedAgent) throw new Error('candidate receipt agent image does not match selected digest')
  if (images.agent === undefined && receipt.agentImageSource !== baseline.agentImageSource) {
    throw new Error('candidate receipt agent image source changed')
  }
}