Skip to content

Where agent teams gather. Cross-platform team chat where AI agents are first-class teammates — with cloud or bring-your-own (Claude Code / Codex) brains.

TypeScript
75.2 KB
Raw Blame
/**
 * Agent-computer lifecycle — the server-side controller for per-agent
 * Pods.
 *
 * Each agent gets one Pod, spawned on demand when a wake arrives and
 * the previous Pod has timed-out-and-exited (status `resting`). The
 * Pod's `/workspace` is a FUSE mount of the agent's slice of
 * `agent_workspace` (`cumora-fuse` Go binary, baked into the image);
 * Postgres row-level security on a per-agent role pins what the Pod
 * can see/write.
 *
 * The Pod also bundles a headless-but-headed Chromium under Xvfb so
 * agents can drive a real browser via OpenCLI (`opencli browser ...`).
 * Chromium's --user-data-dir lives on a per-agent PVC at
 * /opt/chrome-profile so cookies / login state survive pod restarts —
 * that's the only persistent volume in the system. Everything else is
 * still DB-backed.
 *
 * We shell out to `kubectl` rather than depending on the K8s JS
 * client — one fewer dep, kubectl handles context / auth / SA tokens
 * correctly whether the server itself runs in-cluster or on a dev
 * laptop.
 */
import { spawn } from 'node:child_process'
import { notifyAlert } from '../../alerting.js'
import { Semaphore } from '../../concurrency.js'
import { pool } from '../../db/pool.js'
import { env } from '../../env.js'
import { sub2apiConfigured, sub2apiOpenAIBaseURL } from '../../sub2api.js'
import {
  type AgentHostResolution,
  managedPodPlacement,
  resolveAgentHost,
} from '../computer/registry.js'
import { inprocClient } from './inproc-client.js'
import { signAgentToken } from './jwt.js'

const KUBECTL = process.env.CUMORA_KUBECTL ?? 'kubectl'
/** kubectl context to target. In dev we pin to OrbStack so a stray
 *  `kubectl config use-context gke-...` on the dev laptop can't cause
 *  the dev server to spawn agent Pods in a real cluster. In production
 *  we leave it unset and let kubectl resolve the current context
 *  (typically the in-cluster SA when the server runs in K8s). Override
 *  the dev default with CUMORA_KUBECTL_CONTEXT (e.g. if your local
 *  context is named differently). */
const KUBECTL_CONTEXT = process.env.CUMORA_KUBECTL_CONTEXT
  ?? (env.NODE_ENV === 'production' ? '' : 'orbstack')
const IMAGE = process.env.CUMORA_AGENT_COMPUTER_IMAGE
  ?? 'quay.io/yetoneful/cumora-agent-computer:dev'
/** TTL of the JWT minted at Pod-spawn time. Pods live up to
 *  CUMORA_AGENT_IDLE_MS without restart, so the JWT lives longer than
 *  the idle ceiling to avoid mid-life expiry. */
const TOKEN_TTL_SECONDS = Number(process.env.CUMORA_AGENT_TOKEN_TTL_SECONDS ?? 24 * 60 * 60)
/** K8s namespace pods land in. Default = current context's namespace. */
const NS = process.env.CUMORA_AGENT_NAMESPACE ?? 'default'
/** Namespace carrying the trusted cumora-server Service/Pods. Keep this an
 * explicit operator setting; never derive it from a per-user runtime URL. */
const SERVER_NS = process.env.CUMORA_AGENT_SERVER_NAMESPACE ?? 'default'
/** Stable labels and command that identify the post-SEC-10 agent contract.
 * These values are deliberately constants rather than derived from a
 * per-user URL or model setting.  A running Pod is reusable only when its
 * actual API object proves this contract. */
export const AGENT_SECURITY_PROFILE_LABEL = 'cumora.dev/security-profile'
export const AGENT_SECURITY_PROFILE = 'fuse-demoted-v1'
export const AGENT_BOOTSTRAP_COMMAND = '/usr/local/bin/cumora-agent-bootstrap'
export const AGENT_NETWORK_POLICY_VERSION = 'v1'
export const REQUIRED_AGENT_NETWORK_POLICIES = [
  'cumora-agent-default-deny',
  'cumora-agent-egress',
] as const
const BOOTSTRAP_CAPABILITIES = ['SYS_ADMIN', 'SETUID', 'SETGID', 'SETPCAP', 'KILL'] as const
const MODEL_UID = 65532
/** Comma-separated list of imagePullSecrets to attach to the agent
 *  Pod. Needed when the image lives in a private registry (e.g.
 *  quay.io with auth, gcr.io / Artifact Registry without Workload
 *  Identity). Empty = no pull secrets, fine for public registries or
 *  OrbStack-local builds. */
const PULL_SECRETS = (process.env.CUMORA_AGENT_PULL_SECRETS ?? '')
  .split(',').map((s) => s.trim()).filter(Boolean)
/** Optional ServiceAccount identity for the agent Pod. The Pod explicitly
 *  disables automountServiceAccountToken, which only suppresses the projected
 *  Kubernetes API token. It does not by itself suppress GCP Workload Identity
 *  or metadata credentials; those remain governed by runtime/CNI/operator
 *  policy. Empty = namespace `default` SA. */
const POD_SERVICE_ACCOUNT = process.env.CUMORA_AGENT_SERVICE_ACCOUNT ?? ''

export interface KubectlResult { code: number; out: string; err: string; timedOut: boolean }

/** Default per-call timeout for kubectl shell-outs. Without this any
 *  hang in the K8s API server (network blip, throttling, slow auth
 *  refresh) leaves the orchestrator's inFlight map with a zombie
 *  promise — every subsequent caller queues behind it until process
 *  restart. 30s is comfortably above normal kubectl latencies
 *  (~50-500ms) while still bounding the worst case. */
const KUBECTL_DEFAULT_TIMEOUT_MS = 30_000
/** Grace period between SIGTERM and SIGKILL when killing a hung
 *  kubectl. kubectl rarely needs cleanup, but a small grace prevents
 *  EBUSY-on-pipe-close glitches. */
const KUBECTL_KILL_GRACE_MS = 1_000

interface KubectlOpts {
  /** Override the default per-call timeout. Apply usually wants more
   *  because image-pull validation can be slow; gets stay at default. */
  timeoutMs?: number
  /** stdin to pipe to kubectl (used by `kubectl apply -f -`). */
  stdin?: string
}

/** Caps concurrent `kubectl` child processes per replica
 *  (env.KUBECTL_MAX_CONCURRENCY). Each kubectl is a ~50–100MB Go binary
 *  charged to this pod's cgroup; an unbounded burst (one+ per wake under
 *  a storm) can OOM-kill or CPU-throttle the server pod and defeat the
 *  pod-admission cap via a check-before-spawn race. Every caller —
 *  ensurePod, podHealth, the reaper, the fuse-util probe — funnels
 *  through here, so the cap is global. */
const kubectlSem = new Semaphore(env.KUBECTL_MAX_CONCURRENCY)

function kubectl(args: string[], opts: KubectlOpts = {}): Promise<KubectlResult> {
  // Hold a permit for the FULL lifetime of the child (incl. timeout) so
  // the in-flight process count — not just the spawn rate — is bounded.
  return kubectlSem.run(() => kubectlSpawn(args, opts))
}

function kubectlSpawn(args: string[], opts: KubectlOpts = {}): Promise<KubectlResult> {
  const timeoutMs = opts.timeoutMs ?? KUBECTL_DEFAULT_TIMEOUT_MS
  return new Promise((resolve) => {
    const ctxArgs = KUBECTL_CONTEXT === '' ? [] : ['--context', KUBECTL_CONTEXT]
    const child = spawn(KUBECTL, [...ctxArgs, '-n', NS, ...args], { stdio: ['pipe', 'pipe', 'pipe'] })
    let out = ''
    let err = ''
    let settled = false
    let killTimer: NodeJS.Timeout | null = null
    const finish = (r: KubectlResult): void => {
      if (settled) return
      settled = true
      if (killTimer) { clearTimeout(killTimer); killTimer = null }
      if (timer) { clearTimeout(timer); }
      resolve(r)
    }
    const timer = setTimeout(() => {
      if (settled) return
      // Try graceful first, then force.
      try { child.kill('SIGTERM') } catch { /* already gone */ }
      killTimer = setTimeout(() => {
        try { child.kill('SIGKILL') } catch { /* already gone */ }
      }, KUBECTL_KILL_GRACE_MS)
      // Resolve immediately rather than waiting for exit — the caller
      // is treating this as "kubectl is unresponsive", and the child's
      // exit event will fire later (we ignore it via `settled`).
      finish({
        code: 124, // curl/timeout(1) convention for "timed out"
        out,
        err: err + `\n[kubectl] killed after ${timeoutMs}ms timeout`,
        timedOut: true,
      })
    }, timeoutMs)
    timer.unref?.()
    child.stdout.on('data', (d: Buffer) => { out += d.toString() })
    child.stderr.on('data', (d: Buffer) => { err += d.toString() })
    child.on('error', (e) => finish({
      code: 1, out, err: err + (e.message ?? String(e)), timedOut: false,
    }))
    child.on('exit', (code) => finish({
      code: code ?? 1, out, err, timedOut: false,
    }))
    try {
      if (opts.stdin !== undefined) child.stdin.end(opts.stdin)
      else child.stdin.end()
    } catch (e) {
      // EPIPE if the child died before we wrote stdin — covered by
      // the error/exit handlers above.
      void e
    }
  })
}

/** Patterns in kubectl's stderr that signal a TRANSIENT failure
 *  (network, throttling, leader-election churn). Retrying these is
 *  worth a shot. We explicitly DO NOT retry validation errors,
 *  missing image errors, or "already exists" — those won't fix
 *  themselves. */
const RETRYABLE_ERROR_PATTERNS: readonly RegExp[] = [
  /connection refused/i,
  /i\/o timeout/i,
  /context deadline exceeded/i,
  /TLS handshake timeout/i,
  /the server is currently unable to handle the request/i,
  /TooManyRequests/i,
  /etcdserver: leader changed/i,
  /etcdserver: request timed out/i,
  /unexpected EOF/i,
]

/** Exported for testability — production callers use kubectlWithRetry. */
export function isRetryableKubectlError(r: KubectlResult): boolean {
  if (r.code === 0) return false
  if (r.timedOut) return true
  const haystack = `${r.err}\n${r.out}`
  return RETRYABLE_ERROR_PATTERNS.some((re) => re.test(haystack))
}

/** Run a kubectl call with up to N attempts, exponential backoff. The
 *  first attempt fires immediately; retries are gated to errors we've
 *  classified as transient (see RETRYABLE_ERROR_PATTERNS). Returns the
 *  LAST result either way — the caller checks `code` and decides. */
async function kubectlWithRetry(
  args: string[],
  opts: KubectlOpts & { maxAttempts?: number; backoffMs?: number } = {},
): Promise<KubectlResult> {
  const maxAttempts = opts.maxAttempts ?? 3
  const baseBackoff = opts.backoffMs ?? 250
  let last: KubectlResult | null = null
  for (let attempt = 1; attempt <= maxAttempts; attempt++) {
    last = await kubectl(args, opts)
    if (last.code === 0) return last
    if (attempt === maxAttempts) break
    if (!isRetryableKubectlError(last)) break
    // Exponential backoff with full jitter — avoid thundering-herd on
    // a busy API server when N orchestrators all retry at once.
    const max = baseBackoff * 2 ** (attempt - 1)
    const sleep = Math.floor(Math.random() * max)
    await new Promise((r) => setTimeout(r, sleep))
  }
  return last!
}

/** DNS-1123 label safe-name for k8s metadata.name. Exported for
 *  tests — production callers should go through `podName`. */
export function safeName(agentId: string): string {
  // DNS-1123 label: lowercase, alphanumeric + dash, ≤63 chars.
  const slug = agentId.toLowerCase().replace(/[^a-z0-9-]/g, '-').replace(/^-+|-+$/g, '')
  return `agent-${slug}`.slice(0, 63)
}

function podName(agentId: string): string { return safeName(agentId) }

/** Name of the per-agent PVC holding Chromium's profile directory.
 *  One PVC per agent; outlives the pod so cookies / login state
 *  persist across restarts. Same DNS-1123 constraints as podName. */
function chromeProfilePvcName(agentId: string): string {
  return `${safeName(agentId)}-chrome`.slice(0, 63)
}

/** Storage size for the chrome-profile PVC. Default is 500Mi which
 *  fits typical Chromium profile state (cookies + IndexedDB + cache)
 *  with comfortable headroom. Set CUMORA_CHROME_PVC_SIZE to override
 *  (use Kubernetes resource-quantity syntax: 1Gi, 200Mi, etc.). */
const CHROME_PVC_SIZE = process.env.CUMORA_CHROME_PVC_SIZE ?? '500Mi'
/** StorageClass for the chrome-profile PVC. Empty = cluster default,
 *  which is usually what you want. Override with
 *  CUMORA_CHROME_PVC_STORAGECLASS only when you need a specific class
 *  (e.g. a fast SSD pool or a zone-pinned class). */
const CHROME_PVC_STORAGECLASS = process.env.CUMORA_CHROME_PVC_STORAGECLASS ?? ''
/** When set to "false", the chrome-profile volume is an emptyDir
 *  (ephemeral, dies with the pod) instead of a PVC. Escape hatch for
 *  clusters that don't have a default StorageClass yet — agents start
 *  cleanly, they just lose Chromium's login state across restarts.
 *  Default ON so login persistence is the production behavior. */
const CHROME_PROFILE_ON_PVC = process.env.CUMORA_CHROME_PROFILE_PVC !== 'false'

/** Render the Pod manifest. restartPolicy=OnFailure so a crashed
 *  container restarts in-place, but a clean exit (idle timeout —
 *  exit 0) terminates the Pod for good. The FUSE mount of /workspace
 *  is done by the container's entrypoint, before the agent loop
 *  starts. */
/** JSON.stringify produces a double-quoted YAML-safe scalar. We use
 *  this for every interpolated value that flows into a YAML quoted
 *  string position so embedded quotes / backslashes / newlines / non-
 *  ASCII can't break the manifest shape. agentIds in practice are
 *  alphanumeric, but other fields (image, serverUrl) are
 *  user-configurable via env — defense in depth. */
function yamlQuote(value: string): string {
  return JSON.stringify(value)
}

/** Validate that a value contains only DNS-1123 label characters
 *  (lowercase alphanumeric + dashes). k8s rejects labels with
 *  uppercase, dots, etc. — we sanitize the agentId via safeName() for
 *  metadata.name but the cumora.agent LABEL value (used by selectors)
 *  has the same constraint. Throws on invalid input so the caller can
 *  surface a precise error rather than letting kubectl reject the
 *  apply with a less obvious message. */
function dnsLabelValue(value: string): string {
  if (!/^[a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?$/.test(value)) {
    throw new Error(`invalid DNS-1123 label value: ${JSON.stringify(value)}`)
  }
  return value
}

function podManifest(args: {
  agentId: string
  token: string
  image: string
  serverUrl: string
  openaiKey: string
  /** Optional override for the OpenAI base URL. When set, the agent
   *  pod's OpenAI SDK calls go to this base instead of api.openai.com.
   *  Used to route per-agent LLM traffic through sub2api. The pod's
   *  pre-baked OpenAI SDK reads OPENAI_BASE_URL automatically. */
  openaiBaseUrl: string
  idleMs: number
  noWorkMs: number
}): string {
  const indent = (s: string): string => s.split('\n').map((l) => `              ${l}`).join('\n')
  const pullSecretsBlock = PULL_SECRETS.length === 0 ? '' : `
  imagePullSecrets:
${PULL_SECRETS.map((s) => `  - name: ${yamlQuote(s)}`).join('\n')}`
  const saBlock = POD_SERVICE_ACCOUNT === '' ? '' : `
  serviceAccountName: ${yamlQuote(POD_SERVICE_ACCOUNT)}`
  // metadata.name already went through safeName() (DNS-1123). The
  // LABEL value cumora.agent also needs DNS-1123 — selectors that
  // grep the cluster (`kubectl get pods -l cumora.agent=...`) rely
  // on this matching the name slug. Run agentId through the same
  // sanitization so the two stay in sync.
  const podSlug = safeName(args.agentId).replace(/^agent-/, '')
  const labelValue = dnsLabelValue(podSlug || 'unknown')
  return `apiVersion: v1
kind: Pod
metadata:
  name: ${podName(args.agentId)}
  labels:
    app: cumora-agent
    ${AGENT_SECURITY_PROFILE_LABEL}: ${yamlQuote(AGENT_SECURITY_PROFILE)}
    cumora.agent: ${yamlQuote(labelValue)}
spec:
  restartPolicy: OnFailure
  automountServiceAccountToken: false
  securityContext:
    runAsUser: 0
    runAsGroup: 0
    runAsNonRoot: false
    fsGroup: ${MODEL_UID}
    fsGroupChangePolicy: OnRootMismatch
    seccompProfile:
      type: RuntimeDefault${saBlock}${pullSecretsBlock}
  hostPID: false
  hostIPC: false
  hostNetwork: false
  shareProcessNamespace: false
  containers:
  - name: agent-computer
    image: ${yamlQuote(args.image)}
    imagePullPolicy: IfNotPresent
    command: [${yamlQuote(AGENT_BOOTSTRAP_COMMAND)}]
    securityContext:
      # The container starts as root only for the trusted FUSE bootstrap.
      # The bootstrap must then exec setpriv -> tini as UID/GID 65532 with
      # all capabilities cleared and NoNewPrivs=1.  These fields describe
      # the pre-mount capability envelope; they are not the post-bootstrap
      # process boundary.
      runAsUser: 0
      runAsGroup: 0
      runAsNonRoot: false
      allowPrivilegeEscalation: false
      privileged: false
      seccompProfile:
        type: RuntimeDefault
      # FUSE mount needs three things to work on GKE-COS:
      #   1. CAP_SYS_ADMIN — for the mount(2) syscall.
      #   2. /dev/fuse in the cgroup device whitelist — granted by the
      #      generic-device-plugin DaemonSet (squat/generic-device-
      #      plugin) which we depend on being installed in the cluster
      #      (see server/k8s/generic-device-plugin.note).
      #   3. AppArmor unconfined — the default cri-containerd profile
      #      denies the mount syscall even with CAP_SYS_ADMIN, so the
      #      pod must opt out. Without this the agent boots but
      #      fusermount3 fails with "Permission denied" and /workspace
      #      falls through to the empty backing dir — the agent loses
      #      access to its persona files / memory FS.
      appArmorProfile:
        type: Unconfined
      capabilities:
        drop: ["ALL"]
        add: ["SYS_ADMIN", "SETUID", "SETGID", "SETPCAP", "KILL"]
    resources:
      requests:
        # Measured in prod, per-pod working-set ≈ 420 MiB, but only
        # ~263 MiB is anon (irreducible) — the other ~137 MiB is
        # file-backed page cache that the kernel can evict under
        # memory pressure. Setting memory request to 320 MiB covers
        # the irreducible portion with comfortable buffer; under load
        # the limit (1.5 GiB) absorbs bursts. CPU between wakes is
        # <1% — 50m is a generous reservation for a process that
        # spends most of its life as an idle SSE listener; bursts to
        # 2 cores during page render are bounded by the limit (2500m),
        # not the request. Density per e2-standard-2 (1180m CPU / ~6
        # GiB usable after kubelet reserve): ~18 pods CPU-wise, ~18
        # pods memory-wise — balanced, no single-resource binding.
        cpu: "50m"
        memory: "320Mi"
        devic.es/fuse: 1
      limits:
        # Measured under realistic workload (4 sites opened + extracted
        # + an adapter call): memory peaked at ~1230MiB; CPU spiked to
        # 2.2 cores during page rendering. 1.5Gi memory limit leaves
        # ~250Mi headroom over peak; 2500m CPU avoids throttling
        # Chromium during JS-heavy page loads.
        cpu: "2500m"
        memory: "1.5Gi"
        # Reserves one /dev/fuse slot from the device plugin; the
        # kubelet inserts the device into the container's cgroup.
        # Resource name comes from squat/generic-device-plugin's
        # default config — it uses the 'devic.es' domain.
        devic.es/fuse: 1
    volumeMounts:
    - name: chrome-profile
      mountPath: /opt/chrome-profile
    env:
    - name: CUMORA_AGENT_ID
      value: ${yamlQuote(args.agentId)}
    - name: CUMORA_AGENT_RUNTIME_URL
      value: ${yamlQuote(args.serverUrl)}
    - name: CUMORA_AGENT_IDLE_MS
      value: ${yamlQuote(String(args.idleMs))}
    - name: CUMORA_AGENT_NO_WORK_MS
      value: ${yamlQuote(String(args.noWorkMs))}
    - name: CUMORA_PERSONA_DIR
      value: "/workspace"
    - name: CUMORA_AGENT_RUNTIME_TOKEN
      value: |-
${indent(args.token)}
    - name: OPENAI_API_KEY
      value: |-
${indent(args.openaiKey)}
    - name: OPENAI_BASE_URL
      value: |-
${indent(args.openaiBaseUrl)}
  # Spot/Preemptible toleration: GKE Spot VM nodes are tainted
  # cloud.google.com/gke-spot=true:NoSchedule by default so cluster-
  # critical pods don't accidentally land on them. Agent pods are
  # ideal Spot workloads — their state lives in DB (FUSE-mounted)
  # and the chrome-profile is emptyDir/PVC, so a 30s preemption
  # notice just means the next wake re-spawns the pod. Tolerating
  # the taint here lets the orchestrator place agents on cheap
  # Spot nodes; on-demand nodes are untainted so this is additive.
  tolerations:
  - key: cloud.google.com/gke-spot
    operator: Equal
    value: "true"
    effect: NoSchedule
  volumes:
  - name: chrome-profile
${CHROME_PROFILE_ON_PVC
    ? `    persistentVolumeClaim:\n      claimName: ${yamlQuote(chromeProfilePvcName(args.agentId))}`
    : `    emptyDir:\n      sizeLimit: ${yamlQuote(CHROME_PVC_SIZE)}`}
`
}

/** Render the PVC manifest for an agent's Chromium profile. Applied
 *  idempotently before each podManifest apply; PVCs outlive their
 *  pods so a clean idle-exit doesn't wipe login state. */
function chromeProfilePvcManifest(args: { agentId: string }): string {
  const scLine = CHROME_PVC_STORAGECLASS === ''
    ? ''
    : `\n  storageClassName: ${yamlQuote(CHROME_PVC_STORAGECLASS)}`
  // The annotation carries the RAW agentId (may contain chars
  // safeName() strips), so the GC sweeper can reverse-lookup
  // participants.id without depending on the slug being lossless.
  return `apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: ${chromeProfilePvcName(args.agentId)}
  labels:
    app: cumora-agent
    cumora.agent: ${yamlQuote(dnsLabelValue(safeName(args.agentId).replace(/^agent-/, '')) || 'unknown')}
  annotations:
    cumora.dev/agent-id: ${yamlQuote(args.agentId)}
spec:
  accessModes:
  - ReadWriteOnce
  resources:
    requests:
      storage: ${yamlQuote(CHROME_PVC_SIZE)}${scLine}
`
}

type JsonRecord = Record<string, unknown>

function asRecord(value: unknown): JsonRecord | null {
  return typeof value === 'object' && value !== null && !Array.isArray(value)
    ? value as JsonRecord
    : null
}

function asStringRecord(value: unknown): Record<string, string> {
  const record = asRecord(value)
  if (!record) return {}
  const out: Record<string, string> = {}
  for (const [key, item] of Object.entries(record)) {
    if (typeof item === 'string') out[key] = item
  }
  return out
}

function asArray(value: unknown): unknown[] {
  return Array.isArray(value) ? value : []
}

function hasExactStringSet(value: unknown, expected: readonly string[]): boolean {
  if (!Array.isArray(value) || value.some((item) => typeof item !== 'string')) return false
  if (value.length !== expected.length) return false
  const actual = [...new Set(value as string[])].sort()
  return actual.length === expected.length
    && actual.every((item, index) => item === [...expected].sort()[index])
}

function hasStringArray(value: unknown, expected: readonly string[]): boolean {
  return Array.isArray(value)
    && value.length === expected.length
    && value.every((item, index) => item === expected[index])
}

/**
 * Parse the API representation of a Pod and prove that it is the exact
 * trusted bootstrap shape emitted by podManifest(). This is intentionally
 * stricter than checking the profile label: labels can be stale, copied, or
 * changed by an operator while an immutable Pod template remains legacy.
 */
export function isSecureManagedPod(
  raw: unknown,
  expectedImage: string = IMAGE,
  expectedAgentId?: string,
): boolean {
  let pod = raw
  if (typeof raw === 'string') {
    try { pod = JSON.parse(raw) as unknown } catch { return false }
  }
  const root = asRecord(pod)
  const metadata = asRecord(root?.metadata)
  const labels = asStringRecord(metadata?.labels)
  if (labels.app !== 'cumora-agent' || labels[AGENT_SECURITY_PROFILE_LABEL] !== AGENT_SECURITY_PROFILE) {
    return false
  }
  if (expectedAgentId !== undefined) {
    const slug = safeName(expectedAgentId).replace(/^agent-/, '') || 'unknown'
    if (labels['cumora.agent'] !== slug) return false
  }

  const spec = asRecord(root?.spec)
  if (spec?.automountServiceAccountToken !== false) return false
  for (const field of ['hostPID', 'hostIPC', 'hostNetwork', 'shareProcessNamespace']) {
    const value = spec?.[field]
    // These API fields are Go bool/pointer fields with omitempty, so a
    // correctly defaulted Pod may omit false. Accept only false or omitted;
    // true and every malformed value still fail closed.
    if (value !== undefined && value !== false) return false
  }
  for (const field of ['initContainers', 'ephemeralContainers']) {
    const value = spec?.[field]
    if (value !== undefined) return false
  }
  const podSecurity = asRecord(spec?.securityContext)
  if (podSecurity?.runAsUser !== 0 || podSecurity?.runAsGroup !== 0 || podSecurity?.runAsNonRoot !== false) return false
  if (podSecurity?.fsGroup !== MODEL_UID) return false
  const podSeccomp = asRecord(podSecurity?.seccompProfile)
  if (podSeccomp?.type !== 'RuntimeDefault') return false

  const containers = asArray(spec?.containers)
  if (containers.length !== 1) return false
  const container = asRecord(containers[0])
  if (!container || container.name !== 'agent-computer') return false
  if (container.image !== expectedImage) return false
  if (!hasStringArray(container.command, [AGENT_BOOTSTRAP_COMMAND])) return false
  if (container.args !== undefined && (!Array.isArray(container.args) || container.args.length !== 0)) return false
  if (container.lifecycle !== undefined) return false
  for (const field of ['startupProbe', 'readinessProbe', 'livenessProbe']) {
    const probe = asRecord(container[field])
    if (probe?.exec !== undefined) return false
  }
  const volumeMounts = container.volumeMounts
  if (!Array.isArray(volumeMounts) || volumeMounts.length !== 1) return false
  const chromeMount = asRecord(volumeMounts[0])
  if (!chromeMount || chromeMount.name !== 'chrome-profile' || chromeMount.mountPath !== '/opt/chrome-profile') return false
  for (const field of ['subPath', 'subPathExpr']) {
    if (chromeMount[field] !== undefined) return false
  }
  if (chromeMount.mountPropagation !== undefined && chromeMount.mountPropagation !== 'None') return false
  if (container.volumeDevices !== undefined) return false
  const volumes = spec.volumes
  if (!Array.isArray(volumes) || volumes.length !== 1) return false
  const chromeVolume = asRecord(volumes[0])
  if (!chromeVolume || chromeVolume.name !== 'chrome-profile') return false
  const volumeSources = Object.keys(chromeVolume).filter((key) => key !== 'name')
  if (volumeSources.length !== 1) return false
  const pvc = asRecord(chromeVolume.persistentVolumeClaim)
  const emptyDir = asRecord(chromeVolume.emptyDir)
  if (!pvc && !emptyDir) return false
  if (pvc) {
    if (expectedAgentId === undefined || pvc.claimName !== chromeProfilePvcName(expectedAgentId)) return false
  }
  const security = asRecord(container.securityContext)
  if (!security) return false
  if (security.runAsUser !== 0 || security.runAsGroup !== 0) return false
  if (security.runAsNonRoot !== false) return false
  if (security.allowPrivilegeEscalation !== false) return false
  if (security.privileged !== false) return false
  const seccomp = asRecord(security.seccompProfile)
  if (seccomp?.type !== 'RuntimeDefault') return false
  const appArmor = asRecord(security.appArmorProfile)
  if (appArmor?.type !== 'Unconfined') return false
  const capabilities = asRecord(security.capabilities)
  if (!capabilities) return false
  if (!hasStringArray(capabilities.drop, ['ALL'])) return false
  if (!hasExactStringSet(capabilities.add, BOOTSTRAP_CAPABILITIES)) return false
  return true
}

function policyMetadataMatches(policy: JsonRecord, expectedName: string): boolean {
  const metadata = asRecord(policy.metadata)
  const labels = asStringRecord(metadata?.labels)
  return metadata?.name === expectedName
    && labels[AGENT_SECURITY_PROFILE_LABEL] === AGENT_SECURITY_PROFILE
    && labels['cumora.dev/network-policy-version'] === AGENT_NETWORK_POLICY_VERSION
}

function policyTypesExactly(policy: JsonRecord, ...types: string[]): boolean {
  const spec = asRecord(policy.spec)
  const raw = asArray(spec?.policyTypes)
  if (raw.some((type) => typeof type !== 'string')) return false
  const actual = raw as string[]
  actual.sort()
  const expected = [...types].sort()
  return actual.length === expected.length && actual.every((type, index) => type === expected[index])
}

function selectorHasLabels(value: unknown, expected: Record<string, string>): boolean {
  const selector = asRecord(value)
  if (!selector) return false
  const labels = asRecord(selector.matchLabels)
  if (!labels || Object.values(labels).some((item) => typeof item !== 'string')) return false
  const expectedKeys = Object.keys(expected).sort()
  const actualKeys = Object.keys(labels).sort()
  if (actualKeys.length !== expectedKeys.length || actualKeys.some((key, index) => key !== expectedKeys[index])) return false
  if (!Object.entries(expected).every(([key, item]) => labels[key] === item)) return false
  const matchExpressions = selector.matchExpressions
  return matchExpressions === undefined || (Array.isArray(matchExpressions) && matchExpressions.length === 0)
}

function portMatches(value: unknown, protocol: string, port: number): boolean {
  const item = asRecord(value)
  return item?.protocol === protocol
    && item.endPort === undefined
    && (item.port === port || item.port === String(port))
}

function hasPeerSelector(
  peer: unknown,
  namespaceName: string,
  podLabels: Record<string, string>,
): boolean {
  const item = asRecord(peer)
  return selectorHasLabels(item?.namespaceSelector, { 'kubernetes.io/metadata.name': namespaceName })
    && selectorHasLabels(item?.podSelector, podLabels)
}

function hasPublicWebBlock(value: unknown, cidr: string, requiredExcept: readonly string[]): boolean {
  const blocks = asArray(value)
  if (blocks.length !== 1) return false
  const peer = asRecord(blocks[0])
  const block = asRecord(peer?.ipBlock)
  if (block?.cidr !== cidr) return false
  const except = asArray(block.except)
  return requiredExcept.every((item) => except.includes(item))
}

/**
 * Validate the required pre-deployed NetworkPolicies. NetworkPolicy
 * objects are additive and their enforcement is asynchronous, so this only
 * proves the required API objects/version exist. It deliberately does not
 * infer allowlists from OPENAI_BASE_URL or another per-user setting.
 */
export function isRequiredAgentNetworkPolicy(
  expectedName: string,
  raw: unknown,
  agentNamespace: string = NS,
  trustedServerNamespace: string = SERVER_NS,
): boolean {
  const policy = typeof raw === 'string'
    ? (() => { try { return JSON.parse(raw) as unknown } catch { return null } })()
    : raw
  const object = asRecord(policy)
  if (!object || !policyMetadataMatches(object, expectedName)) return false
  const metadata = asRecord(object.metadata)
  if (typeof metadata?.namespace === 'string' && metadata.namespace !== agentNamespace) return false
  const spec = asRecord(object.spec)
  if (!spec) return false

  if (expectedName === 'cumora-agent-default-deny') {
    return selectorHasLabels(spec.podSelector, { app: 'cumora-agent' })
      && policyTypesExactly(object, 'Ingress', 'Egress')
      && asArray(spec.ingress).length === 0
      && asArray(spec.egress).length === 0
  }

  if (expectedName === 'cumora-agent-egress') {
    const egress = asArray(spec.egress)
    if (!selectorHasLabels(spec.podSelector, { app: 'cumora-agent' })
      || !policyTypesExactly(object, 'Egress')
      || asArray(spec.ingress).length !== 0
      || egress.length !== 4) return false
    const dns = asRecord(egress[0])
    const server = asRecord(egress[1])
    const ipv4 = asRecord(egress[2])
    const ipv6 = asRecord(egress[3])
    const dnsPeers = asArray(dns?.to)
    const dnsPorts = asArray(dns?.ports)
    const serverPeers = asArray(server?.to)
    const serverPorts = asArray(server?.ports)
    const webPorts4 = asArray(ipv4?.ports)
    const webPorts6 = asArray(ipv6?.ports)
    const dnsPeer = asRecord(dnsPeers[0])
    const dnsNamespace = asRecord(dnsPeer?.namespaceSelector)
    const dnsPod = asRecord(dnsPeer?.podSelector)
    return dnsPeers.length === 1
      && selectorHasLabels(dnsNamespace, { 'kubernetes.io/metadata.name': 'kube-system' })
      && selectorHasLabels(dnsPod, { 'k8s-app': 'kube-dns' })
      && dnsPorts.length === 2
      && dnsPorts.some((port) => portMatches(port, 'UDP', 53))
      && dnsPorts.some((port) => portMatches(port, 'TCP', 53))
      && serverPeers.length === 1
      && hasPeerSelector(serverPeers[0], trustedServerNamespace, { app: 'cumora-server' })
      && serverPorts.length === 1
      && portMatches(serverPorts[0], 'TCP', 5181)
      && hasPublicWebBlock(ipv4?.to, '0.0.0.0/0', [
        '0.0.0.0/8', '10.0.0.0/8', '100.64.0.0/10', '127.0.0.0/8',
        '169.254.0.0/16', '172.16.0.0/12', '192.0.0.0/24', '192.0.2.0/24',
        '192.168.0.0/16', '198.18.0.0/15', '198.51.100.0/24', '203.0.113.0/24',
        '224.0.0.0/4', '240.0.0.0/4',
      ])
      && webPorts4.length === 2
      && webPorts4.some((port) => portMatches(port, 'TCP', 80))
      && webPorts4.some((port) => portMatches(port, 'TCP', 443))
      && hasPublicWebBlock(ipv6?.to, '::/0', [
        '::/128', '::1/128', 'fc00::/7', 'fe80::/10', 'ff00::/8', '2001:db8::/32',
      ])
      && webPorts6.length === 2
      && webPorts6.some((port) => portMatches(port, 'TCP', 80))
      && webPorts6.some((port) => portMatches(port, 'TCP', 443))
  }
  return false
}

/** Exported for tests — production callers go through ensurePod. */
export const _testing = {
  podManifest,
  yamlQuote,
  dnsLabelValue,
  chromeProfilePvcManifest,
  chromeProfilePvcName,
  isSecureManagedPod,
  isRequiredAgentNetworkPolicy,
}

// ─── cluster-wide FUSE admission control ─────────────────────────────
//
// Triggered by the FUSE-cap incident on 2026-05-20: agent pods request
// `devic.es/fuse: 1` (one slot per pod), and each prod node advertises
// 800 slots via the generic-device-plugin DaemonSet. The app-level
// admission cap is intentionally lower (AGENT_POD_ADMISSION_MAX,
// default 40) because CPU, memory, API-server churn, and provider
// concurrency are the real production ceiling.
//
// `ensurePod` must REFUSE rather than blindly apply when the cluster
// is near saturation. Refusing lets pending wakes survive in their
// inbox (message.new) or get retried by their scheduler (idle /
// scanner); blowing past the cap, conversely, leaves pods stuck in
// FailedScheduling for tens of minutes with no recovery.

export interface ClusterFuseUtilization {
  /** Number of cumora-agent pods currently using a fuse slot
   *  (Pending + Running, excluding Succeeded/Failed). */
  used: number
  /** Effective admission cap: min(cluster `devic.es/fuse` capacity,
   *  AGENT_POD_ADMISSION_MAX when that env is positive). */
  cap: number
  /** used / cap. 0 when cap is 0 (no nodes report fuse). */
  ratio: number
  /** True when result came from the in-memory cache. */
  cached: boolean
}

/** Pure parser — extracted so the unit test can pin both branches
 *  (well-formed JSON, malformed) without spawning kubectl. */
export function parseClusterFuse(nodesJson: string, podsJson: string): { used: number; cap: number } {
  let cap = 0
  try {
    const d = JSON.parse(nodesJson) as { items?: Array<{ status?: { capacity?: Record<string, string> } }> }
    for (const n of d.items ?? []) {
      const slots = Number(n.status?.capacity?.['devic.es/fuse'] ?? 0)
      if (Number.isFinite(slots) && slots > 0) cap += slots
    }
  } catch { /* ignore — return 0 */ }
  let used = 0
  try {
    const d = JSON.parse(podsJson) as { items?: Array<{ status?: { phase?: string } }> }
    for (const p of d.items ?? []) {
      const phase = p.status?.phase
      // Pending pods that haven't been bound yet don't actually hold
      // a device slot, but they DO represent demand the scheduler is
      // trying to fulfill — count them so we don't over-spawn into a
      // queue. Succeeded/Failed pods released their slot at exit.
      if (phase === 'Running' || phase === 'Pending') used++
    }
  } catch { /* ignore — return 0 */ }
  return { used, cap }
}

interface FuseUtilCache { ts: number; used: number; cap: number }
let fuseUtilCache: FuseUtilCache | null = null

/** TTL on the cluster-wide fuse sample. Short enough that bursts get
 *  re-evaluated quickly; long enough that we don't hammer the K8s API
 *  on every wake. Each successful spawn bumps `used` in-memory so the
 *  cached value tracks intra-window growth without a refresh. */
const FUSE_CACHE_TTL_MS = 10_000

/** Threshold above which `ensurePod` refuses new spawns. 0.90 leaves
 *  a 10% buffer for in-flight scheduling latency and inFlight-dedup
 *  raciness. Below 0.90 we let everything through. */
const FUSE_ADMISSION_THRESHOLD = 0.90

/** Sample (or read from cache) the cluster's cumora-agent fuse usage.
 *  Fails open: on kubectl error returns `cap=Infinity` so callers
 *  never block on transient cluster-state read failures. */
export async function getClusterFuseUtilization(): Promise<ClusterFuseUtilization> {
  const now = Date.now()
  if (fuseUtilCache && now - fuseUtilCache.ts < FUSE_CACHE_TTL_MS) {
    const { used, cap } = fuseUtilCache
    return { used, cap, ratio: cap > 0 ? used / cap : 0, cached: true }
  }
  const [nodes, pods] = await Promise.all([
    kubectlWithRetry(['get', 'nodes', '-o', 'json'], { timeoutMs: 10_000 }),
    kubectlWithRetry(
      ['get', 'pods', '-l', 'app=cumora-agent', '-o', 'json'],
      { timeoutMs: 10_000 },
    ),
  ])
  if (nodes.code !== 0 || pods.code !== 0) {
    // Fail open — we'd rather over-spawn during a kubectl flake than
    // wedge every wake. Don't cache this so the next call retries.
    return { used: 0, cap: Number.POSITIVE_INFINITY, ratio: 0, cached: false }
  }
  const parsed = parseClusterFuse(nodes.out, pods.out)
  const appCap = env.AGENT_POD_ADMISSION_MAX
  const cap = appCap > 0 ? Math.min(parsed.cap, appCap) : parsed.cap
  fuseUtilCache = { ts: now, used: parsed.used, cap }
  return { used: parsed.used, cap, ratio: cap > 0 ? parsed.used / cap : 0, cached: false }
}

/** Test-only — clear the in-memory cache so tests don't poison each other. */
export function _resetFuseUtilCacheForTests(): void { fuseUtilCache = null }

/** Bump cached `used` after a successful spawn, so back-to-back
 *  `ensurePod` calls within the cache TTL see the increment without
 *  waiting for the next refresh. No-op if the cache is empty. */
function bumpFuseUtilUsedOnSpawn(): void {
  if (fuseUtilCache) fuseUtilCache.used++
}

export type PodPhase = '' | 'Pending' | 'Running' | 'Succeeded' | 'Failed' | 'Unknown'

export interface PodHealth {
  phase: PodPhase
  /** waiting.reason from every (init)containerStatus — surfaces ImagePullBackOff
   *  / CrashLoopBackOff / CreateContainerConfigError / etc. */
  waitingReasons: string[]
  /** PodScheduled=False, reason=Unschedulable — scheduler can't place the pod. */
  unschedulable: boolean
}

interface RawPod {
  status?: {
    phase?: string
    conditions?: Array<{ type?: string; status?: string; reason?: string }>
    containerStatuses?: Array<{ state?: { waiting?: { reason?: string } } }>
    initContainerStatuses?: Array<{ state?: { waiting?: { reason?: string } } }>
  }
}

/** Read an existing Pod's phase + the container waiting reasons that
 *  signal a stuck Pending state. Empty phase means the Pod doesn't
 *  exist. We need more than just phase: a Pod sitting in `Pending`
 *  with `ImagePullBackOff` will never recover on its own — the
 *  scheduler must delete it and reapply. */
/** Pure parser — split from the kubectl call so tests can pin the
 *  contract without spawning a real kubectl. Returns the empty/safe
 *  health shape on any parse failure. */
export function parsePodHealth(rawJson: string): PodHealth {
  let pod: RawPod
  try {
    pod = JSON.parse(rawJson) as RawPod
  } catch {
    return { phase: '', waitingReasons: [], unschedulable: false }
  }
  const phase = (pod.status?.phase ?? '') as PodPhase
  const waitingReasons: string[] = []
  for (const arr of [pod.status?.containerStatuses, pod.status?.initContainerStatuses]) {
    if (!Array.isArray(arr)) continue
    for (const cs of arr) {
      const reason = cs.state?.waiting?.reason
      if (typeof reason === 'string' && reason) waitingReasons.push(reason)
    }
  }
  const unschedulable = (pod.status?.conditions ?? []).some(
    (c) => c.type === 'PodScheduled' && c.status === 'False' && c.reason === 'Unschedulable',
  )
  return { phase, waitingReasons, unschedulable }
}

interface PodSnapshot {
  exists: boolean
  lookupFailed: boolean
  raw: unknown
  health: PodHealth
  error?: string
}

/** With --ignore-not-found=true, kubectl reports a missing named object as
 * success with no stdout. A nonzero result is always an API/auth/transport
 * failure; an error message containing "not found" is not resource absence
 * evidence (for example, an auth plugin executable may be missing). */
function resultIsMissingWithIgnoreNotFound(r: KubectlResult): boolean {
  return r.code === 0 && r.out.trim() === ''
}

/** Read both health and the actual API object. Reusing a Pod is allowed only
 * after the latter passes isSecureManagedPod(); a status-only read cannot
 * distinguish a secure immutable template from a legacy one. */
async function podSnapshot(agentId: string): Promise<PodSnapshot> {
  const r = await kubectlWithRetry(['get', 'pod', podName(agentId), '--ignore-not-found=true', '-o', 'json'])
  if (r.code !== 0) {
    return {
      exists: false,
      lookupFailed: true,
      raw: null,
      health: { phase: '', waitingReasons: [], unschedulable: false },
      error: (r.err || r.out).trim(),
    }
  }
  if (resultIsMissingWithIgnoreNotFound(r)) {
    return {
      exists: false,
      lookupFailed: false,
      raw: null,
      health: { phase: '', waitingReasons: [], unschedulable: false },
    }
  }
  let raw: unknown = null
  try { raw = JSON.parse(r.out) as unknown } catch { /* malformed = existing unknown Pod */ }
  return { exists: true, lookupFailed: false, raw, health: parsePodHealth(r.out) }
}

type NetworkPolicyPreflight = {
  ok: true
} | {
  ok: false
  code: 'network_policy_missing' | 'network_policy_invalid' | 'network_policy_lookup_failed'
  reason: string
}

async function checkRequiredAgentNetworkPolicies(): Promise<NetworkPolicyPreflight> {
  for (const name of REQUIRED_AGENT_NETWORK_POLICIES) {
    const r = await kubectlWithRetry(
      ['get', 'networkpolicy', name, '--ignore-not-found=true', '-o', 'json'],
      { timeoutMs: 10_000 },
    )
    if (r.code !== 0) {
      const detail = (r.err || r.out).trim().slice(0, 240)
      return { ok: false, code: 'network_policy_lookup_failed', reason: `could not read required NetworkPolicy ${name}: ${detail}` }
    }
    if (resultIsMissingWithIgnoreNotFound(r)) {
      return { ok: false, code: 'network_policy_missing', reason: `required NetworkPolicy ${name} is missing in namespace ${NS}` }
    }
    let raw: unknown
    try { raw = JSON.parse(r.out) as unknown } catch {
      return { ok: false, code: 'network_policy_invalid', reason: `required NetworkPolicy ${name} returned invalid JSON` }
    }
    if (!isRequiredAgentNetworkPolicy(name, raw, NS, SERVER_NS)) {
      return { ok: false, code: 'network_policy_invalid', reason: `required NetworkPolicy ${name} does not match ${AGENT_NETWORK_POLICY_VERSION} contract` }
    }
  }
  return { ok: true }
}

export type PodDeletionResult = { ok: true } | { ok: false; reason: string }
type PodKubectlRunner = (
  args: string[],
  opts?: { timeoutMs?: number; maxAttempts?: number },
) => Promise<KubectlResult>

/** Delete an immutable Pod and prove its name is gone before a replacement
 * can be applied. The runner/sleep seams keep this lifecycle testable without
 * talking to a production cluster. */
export async function waitForPodDeletion(
  agentId: string,
  runner: PodKubectlRunner = kubectlWithRetry,
  options: { deadlineMs?: number; pollMs?: number; sleep?: (ms: number) => Promise<void> } = {},
): Promise<PodDeletionResult> {
  const name = podName(agentId)
  const reap = await runner(
    ['delete', 'pod', name, '--ignore-not-found=true', '--wait=true', '--timeout=30s'],
    { timeoutMs: 40_000 },
  )
  if (reap.code !== 0) {
    return { ok: false, reason: `delete failed: ${(reap.err || reap.out).trim()}` }
  }

  // `kubectl delete --wait` normally gives this guarantee, but it can return
  // after an API timeout or a finalizer race. Confirm the name is gone before
  // applying a replacement, otherwise an immutable Pod can leave two model
  // lifecycles or turn the apply into a misleading AlreadyExists failure.
  const deadline = Date.now() + (options.deadlineMs ?? 10_000)
  const sleep = options.sleep ?? ((ms: number) => new Promise<void>((resolve) => setTimeout(resolve, ms)))
  while (Date.now() < deadline) {
    const check = await runner(
      ['get', 'pod', name, '--ignore-not-found=true', '-o', 'json'],
      { timeoutMs: 5_000, maxAttempts: 1 },
    )
    if (resultIsMissingWithIgnoreNotFound(check)) return { ok: true }
    if (check.code !== 0 && !isRetryableKubectlError(check)) {
      return { ok: false, reason: `deletion confirmation failed: ${(check.err || check.out).trim()}` }
    }
    await sleep(options.pollMs ?? 250)
  }
  return { ok: false, reason: `pod ${name} did not disappear before timeout` }
}

async function reapPodAndConfirmGone(agentId: string, reason: string): Promise<EnsurePodResult | null> {
  const result = await waitForPodDeletion(agentId)
  if (result.ok) return null
  return {
    created: false,
    ok: false,
    code: 'pod_reap_failed',
    reason: `pod reap failed (${reason}): ${result.reason}`,
  }
}

/** Waiting reasons that K8s will not recover from on its own — image
 *  is missing/broken, container config is invalid, or the container
 *  keeps crashing. Anything in this set means the Pod is stuck and
 *  we should reap + recreate. */
const STUCK_WAITING_REASONS = new Set([
  'ImagePullBackOff',
  'ErrImagePull',
  'CrashLoopBackOff',
  'CreateContainerConfigError',
  'CreateContainerError',
  'InvalidImageName',
  'RunContainerError',
])

/** If the Pending Pod is stuck, return a short diagnostic string.
 *  Otherwise null (still legitimately pending — image pulling,
 *  init containers running, etc.). Exported for testability. */
export function stuckPendingReason(h: PodHealth): string | null {
  const bad = h.waitingReasons.find((r) => STUCK_WAITING_REASONS.has(r))
  if (bad) return bad
  if (h.unschedulable) return 'Unschedulable'
  return null
}

/** Result of `ensurePod`. `ok` distinguishes "Pod is healthy and ready
 *  to receive wakes" (created OR already pending/running) from "we
 *  couldn't get a Pod into a runnable state" (apply failed, persona
 *  missing, etc.). Callers that just want to log real failures should
 *  check `!ok` rather than string-matching `reason`. */
export type EnsurePodResult =
  | { created: true;  ok: true;  reason: string }
  | { created: false; ok: true;  reason: string }
  | {
      created: false
      ok: false
      reason: string
      code:
        | 'watchdog_timeout'
        | 'pod_reap_failed'
        | 'capacity_denied'
        | 'agent_not_found'
        | 'placement_lookup_failed'
        | 'placement_denied'
        | 'network_policy_missing'
        | 'network_policy_invalid'
        | 'network_policy_lookup_failed'
        | 'pod_apply_failed'
    }

export type ManagedPodPlacementVerification =
  | { ok: true; companyId: string; computerId: string | null; runtimeAssignmentId: string }
  | {
      ok: false
      code: 'agent_not_found' | 'placement_lookup_failed' | 'placement_denied'
      reason: string
    }

/** Resolve the policy at the managed-runtime boundary. The resolver is
 * injectable only so fault-path unit tests can prove that lookup exceptions
 * and invalid assignments fail closed without invoking kubectl. */
export async function verifyManagedPodPlacement(
  agentId: string,
  resolver: (id: string) => Promise<AgentHostResolution> = resolveAgentHost,
): Promise<ManagedPodPlacementVerification> {
  let resolution: AgentHostResolution
  try {
    resolution = await resolver(agentId)
  } catch (cause) {
    resolution = {
      status: 'error',
      code: 'lookup_failed',
      reason: `host lookup failed for agent ${agentId}`,
      cause,
    }
  }
  const decision = managedPodPlacement(resolution)
  if (decision.status === 'denied') {
    return { ok: false, code: decision.code, reason: decision.reason }
  }
  return {
    ok: true,
    companyId: decision.companyId,
    computerId: decision.computerId,
    runtimeAssignmentId: decision.runtimeAssignmentId,
  }
}

/** In-flight `ensurePod` deduplication. Without this a thundering
 *  herd (e.g. 50 wakes for one resting agent within 1s) calls
 *  kubectl apply 50 times before the first apply finishes. Idempotent
 *  on the K8s side, but wasteful and noisy. We coalesce concurrent
 *  callers onto the first inflight promise; once it resolves, the
 *  entry drops out of the map and the next caller (if any) starts
 *  fresh. */
const inFlight = new Map<string, Promise<EnsurePodResult>>()

/** Watchdog timeout on the WHOLE ensurePodImpl invocation. Even with
 *  per-kubectl-call timeouts, an unexpectedly-slow DB query or a hang
 *  somewhere we didn't bound could keep the in-flight promise alive
 *  indefinitely. Without this watchdog, the inFlight Map would
 *  accumulate dead entries and every subsequent caller for the same
 *  agentId would deadlock. The sum of individual call timeouts is
 *  ~120s in the worst path (reap 40s + apply 45s + persona lookup +
 *  sub2api lookup); 180s gives a sane outer ceiling.
 *
 *  When the watchdog fires we ALSO drop the inFlight entry so
 *  callers can retry on the next wake. */
const ENSURE_POD_WATCHDOG_MS = 180_000

/** Idempotent — spin up a Pod for `agentId` if one isn't already
 *  running. Returns a hint about whether we actually created anything
 *  (for log readability).
 *
 *  Doesn't wait for the Pod to be Ready / SSE-attached. The scheduler
 *  is free to enqueue the wake event on the bus; the Pod, once it
 *  connects, drains its inbox unconditionally and catches up. */
export async function ensurePod(agentId: string): Promise<EnsurePodResult> {
  const existing = inFlight.get(agentId)
  if (existing) return existing
  const p = (async (): Promise<EnsurePodResult> => {
    let watchdogFired = false
    const watchdog = new Promise<EnsurePodResult>((resolve) =>
      setTimeout(() => {
        watchdogFired = true
        resolve({
          created: false,
          ok: false,
          code: 'watchdog_timeout',
          reason: `ensurePod watchdog: implementation did not return within ${ENSURE_POD_WATCHDOG_MS}ms`,
        })
      }, ENSURE_POD_WATCHDOG_MS).unref?.(),
    )
    try {
      const result = await Promise.race([ensurePodImpl(agentId), watchdog])
      if (watchdogFired) {
        // Alert because this means something hung BELOW the per-call
        // timeouts — a real bug we want to know about, not just a
        // slow cluster.
        void notifyAlert({
          label: 'orchestrator.ensurePod_watchdog',
          error: new Error(`ensurePod(${agentId}) exceeded ${ENSURE_POD_WATCHDOG_MS}ms`),
          extras: { agentId },
        })
      }
      return result
    } finally {
      inFlight.delete(agentId)
    }
  })()
  inFlight.set(agentId, p)
  return p
}

async function ensurePodImpl(agentId: string): Promise<EnsurePodResult> {
  const startedAt = Date.now()
  // This is the final authorization boundary for managed execution. Scheduler
  // lookups are advisory only: assignment/tier can change between a wake and
  // this call, and other callers may invoke ensurePod directly.
  const initialPlacement = await verifyManagedPodPlacement(agentId)
  if (!initialPlacement.ok) {
    return { created: false, ...initialPlacement }
  }
  const recheckPlacement = async (): Promise<EnsurePodResult | null> => {
    const current = await verifyManagedPodPlacement(agentId)
    if (!current.ok) return { created: false, ...current }
    if (
      current.companyId !== initialPlacement.companyId
      || current.computerId !== initialPlacement.computerId
      || current.runtimeAssignmentId !== initialPlacement.runtimeAssignmentId
    ) {
      return {
        created: false,
        ok: false,
        code: 'placement_denied',
        reason: 'managed pod denied: agent placement changed during preparation',
      }
    }
    return null
  }
  const snapshot = await podSnapshot(agentId)
  if (snapshot.lookupFailed) {
    return {
      created: false,
      ok: false,
      code: 'pod_reap_failed',
      reason: `could not inspect existing pod before replacement: ${snapshot.error ?? 'kubectl failed'}`,
    }
  }

  if (snapshot.exists) {
    const secure = isSecureManagedPod(snapshot.raw, IMAGE, agentId)
    const h = snapshot.health
    const stuck = h.phase === 'Pending' ? stuckPendingReason(h) : null
    const reusable = secure && (h.phase === 'Running' || (h.phase === 'Pending' && !stuck))

    if (reusable) {
      // A secure label alone is not enough: isSecureManagedPod already
      // checked the immutable command/image/security fields. The policy
      // check below proves the required pre-deployed objects are present
      // before we treat the Pod as usable.
      const denied = await recheckPlacement()
      if (denied) return denied
      const policies = await checkRequiredAgentNetworkPolicies()
      if (!policies.ok) return { created: false, ...policies }
      return { created: false, ok: true, reason: h.phase === 'Running' ? 'already running' : 'already pending' }
    }

    const reason = !secure
      ? 'legacy or unknown security contract'
      : stuck
        ? `stuck pending=${stuck}`
        : h.phase === 'Failed'
          ? 'previous pod ended in Failed'
          : h.phase === 'Succeeded'
            ? 'previous pod completed'
            : h.phase === 'Unknown'
              ? 'previous pod phase is Unknown'
              : `unrecognized pod phase=${h.phase || 'empty'}`
    if (stuck) {
      console.warn(`[orchestrator] ${agentId} pending pod is stuck (${stuck}); reaping`)
      void notifyAlert({
        label: 'orchestrator.stuck_pending_reap',
        error: new Error(`pod for ${agentId} stuck in Pending (${stuck}); reaping for fresh start`),
        extras: { agentId, stuckReason: stuck, waitingReasons: h.waitingReasons.join(',') },
      })
    } else if (h.phase === 'Failed') {
      console.warn(`[orchestrator] ${agentId} previous pod ended in Failed; reaping to recreate`)
      void notifyAlert({
        label: 'orchestrator.previous_pod_failed',
        error: new Error(`previous pod for ${agentId} ended in Failed phase; recreating`),
        extras: { agentId, waitingReasons: h.waitingReasons.join(',') },
      })
    } else if (h.phase === 'Unknown' || !secure) {
      console.warn(`[orchestrator] ${agentId} ${reason}; deleting before replacement`)
    }
    const denied = await recheckPlacement()
    if (denied) return denied
    const reapFailure = await reapPodAndConfirmGone(agentId, reason)
    if (reapFailure) return reapFailure
  }

  // NetworkPolicy objects are a deployment prerequisite. The orchestrator
  // never applies them and never derives exceptions from a user-controlled
  // model/provider URL; API success is not evidence that a CNI enforces them.
  const policies = await checkRequiredAgentNetworkPolicies()
  if (!policies.ok) return { created: false, ...policies }

  // Cluster-wide FUSE admission: refuse the spawn if the cluster is
  // near its /dev/fuse capacity. The scheduler puts durable
  // message.new/manual wakes into a Redis retry queue when this
  // returns !ok; idle/scanner wakes are best-effort and retry on the
  // next tick.
  // Throwing this admission AT the kubectl-apply edge stops over-spawn
  // at its narrowest possible chokepoint and keeps cluster cap from
  // being exceeded regardless of upstream caller behavior.
  const fuse = await getClusterFuseUtilization()
  if (fuse.cap > 0 && fuse.ratio >= FUSE_ADMISSION_THRESHOLD) {
    return {
      created: false, ok: false, code: 'capacity_denied',
      reason: `cluster fuse saturated: ${fuse.used}/${fuse.cap} (${Math.round(fuse.ratio * 100)}% ≥ ${Math.round(FUSE_ADMISSION_THRESHOLD * 100)}% threshold)`,
    }
  }

  const persona = await inprocClient.loadPersona(agentId).catch(() => null)
  if (!persona) {
    return { created: false, ok: false, code: 'agent_not_found', reason: 'no such agent' }
  }
  if (persona.companyId !== initialPlacement.companyId) {
    return {
      created: false,
      ok: false,
      code: 'placement_denied',
      reason: 'managed pod denied: persona tenant does not match host assignment',
    }
  }

  const token = signAgentToken({
    agentId,
    companyId: persona.companyId,
    computerId: initialPlacement.computerId,
    assignmentId: initialPlacement.runtimeAssignmentId,
    ttlSeconds: TOKEN_TTL_SECONDS,
  })

  // Resolve the workspace's per-user sub2api key so this pod's LLM
  // traffic counts against that user's quota. Falls back to the
  // legacy single OPENAI_API_KEY when sub2api isn't configured OR the
  // workspace owner hasn't been provisioned yet. The same logic lives
  // in llm.ts but pods don't share memory with the server process —
  // we have to bake the resolved key into the manifest env at
  // pod-spawn time.
  let resolvedKey = env.OPENAI_API_KEY
  let resolvedBaseUrl = '' // empty → OpenAI SDK uses its default (api.openai.com/v1)
  if (sub2apiConfigured()) {
    try {
      const { rows } = await pool.query<{ sub2api_api_key: string | null }>(
        `SELECT u.sub2api_api_key FROM companies c
            JOIN users u ON u.id = c.owner_user_id
          WHERE c.id = $1`,
        [persona.companyId],
      )
      const k = rows[0]?.sub2api_api_key
      if (k) {
        resolvedKey = k
        resolvedBaseUrl = sub2apiOpenAIBaseURL()
      }
    } catch (e) {
      console.warn(`[orchestrator] sub2api key lookup failed for ${persona.companyId}; legacy fallback`, e instanceof Error ? e.message : e)
    }
  }

  // Re-read immediately before the first Kubernetes mutation. This closes the
  // scheduler-to-orchestrator and preparation-time race: moving the Agent to a
  // BYOA Computer or downgrading the tenant while pod health/key resolution is
  // in flight cannot still result in a managed PVC/Pod apply.
  const finalPlacementDenied = await recheckPlacement()
  if (finalPlacementDenied) return finalPlacementDenied

  // Apply the per-agent chrome-profile PVC BEFORE the pod. kubectl
  // apply is idempotent — if the PVC already exists this is a no-op.
  // The PVC outlives the pod, so an idle-exit / restart / re-spawn
  // all attach to the same volume and Chromium's cookies + login
  // state survive. Failure here is non-fatal: the pod can still run
  // (Chromium would just have an empty profile) so we log and
  // continue rather than block the agent from spawning.
  //
  // Skipped entirely when CUMORA_CHROME_PROFILE_PVC=false — used by
  // clusters that don't have a default StorageClass set up yet. The
  // pod manifest swaps the volume source to emptyDir to match.
  if (CHROME_PROFILE_ON_PVC) {
    const pvcManifest = chromeProfilePvcManifest({ agentId })
    const pvcApply = await kubectlWithRetry(['apply', '-f', '-'], { stdin: pvcManifest, timeoutMs: 20_000 })
    if (pvcApply.code !== 0) {
      console.warn(`[orchestrator] chrome PVC apply failed for ${agentId}: ${(pvcApply.err || pvcApply.out).trim()}`)
    }
  }

  const manifest = podManifest({
    agentId,
    token,
    image: IMAGE,
    serverUrl: env.AGENT_RUNTIME_SERVER_URL,
    openaiKey: resolvedKey,
    openaiBaseUrl: resolvedBaseUrl,
    idleMs: env.AGENT_IDLE_MS,
    noWorkMs: env.AGENT_NO_WORK_MS,
  })

  // `kubectl apply` is the write path. Retry on transient errors;
  // 45s timeout because validation + admission webhooks can be slow
  // when an image-pull-credential webhook runs.
  const podApply = await kubectlWithRetry(['apply', '-f', '-'], { stdin: manifest, timeoutMs: 45_000 })
  if (podApply.code !== 0) {
    const errText = (podApply.err || podApply.out).trim()
    // Alert on apply failures — when this fires, agents stay in
    // resting state and never wake up no matter how many messages
    // arrive. Highest-severity orchestrator failure mode.
    void notifyAlert({
      label: 'orchestrator.pod_apply_failed',
      error: new Error(`kubectl apply failed for ${agentId}: ${errText}`),
      extras: { agentId, elapsedMs: Date.now() - startedAt, timedOut: podApply.timedOut },
    })
    return {
      created: false,
      ok: false,
      code: 'pod_apply_failed',
      reason: `pod apply failed: ${errText}`,
    }
  }
  bumpFuseUtilUsedOnSpawn()
  console.log(`[orchestrator] ${agentId} pod spun up in ${Date.now() - startedAt}ms`)
  return { created: true, ok: true, reason: 'spun up' }
}

/** Tear down an agent's Pod. Idempotent. Normally the Pod self-
 *  terminates by exiting 0 after its idle timer fires; this is the
 *  manual / cleanup path. */
export async function deletePod(agentId: string): Promise<void> {
  // --wait=false here (unlike the in-ensurePod reap which uses --wait=true):
  // this is the manual cleanup path, callers don't care to block until
  // termination completes.
  await kubectlWithRetry(
    ['delete', 'pod', podName(agentId), '--ignore-not-found=true', '--wait=false'],
    { timeoutMs: 20_000 },
  )
}

/** Drop an agent's chrome-profile PVC. NOT called on normal idle-exit
 *  — PVCs outlive their pod so a re-spawn keeps cookies / login
 *  state. Call this only when the agent is being permanently
 *  off-boarded; otherwise the next pod will rebind to the existing
 *  PVC. Idempotent (--ignore-not-found). */
export async function deleteChromeProfilePvc(agentId: string): Promise<void> {
  await kubectlWithRetry(
    ['delete', 'pvc', chromeProfilePvcName(agentId), '--ignore-not-found=true', '--wait=false'],
    { timeoutMs: 20_000 },
  )
}

// ─── Idle chrome-profile PVC garbage collection ──────────────────────
//
// PVCs outlive their pods on purpose (so Chromium login state survives
// idle-exit), but agents that never get woken again leave orphan PVCs
// accumulating forever — at $0.10/GB·month each, 1k abandoned agents
// = $100/month of pure waste, scaling linearly with adoption. This
// sweeper deletes the PVCs for agents that:
//   • have been off-boarded (departed_at IS NOT NULL), AND/OR
//   • haven't run in the last IDLE_DAYS (no agent_runs row newer
//     than the threshold)
// Idle Pods themselves are off-PVC by the time the sweeper runs (they
// exit on idle), so deleting the volume doesn't yank from under a
// live process.

/** A PVC the GC pass found that should be considered for deletion. */
export interface IdlePvcCandidate {
  pvcName: string
  agentId: string
  lastWakeAt: Date | null
  departedAt: Date | null
  reason: 'departed' | 'idle' | 'orphan'
}

/** Pure planner: given the raw PVC list + DB state, decide which to
 *  delete. Extracted so unit tests can exercise the decision matrix
 *  without needing a live cluster or DB. */
export function planIdlePvcGc(args: {
  pvcs: Array<{ name: string; agentId: string }>
  agents: Map<string, { departedAt: Date | null; lastWakeAt: Date | null }>
  now: Date
  idleThresholdMs: number
}): IdlePvcCandidate[] {
  const drop: IdlePvcCandidate[] = []
  const cutoff = args.now.getTime() - args.idleThresholdMs
  for (const pvc of args.pvcs) {
    const a = args.agents.get(pvc.agentId)
    if (!a) {
      // PVC has no matching participants row — the agent was hard-
      // deleted somehow (or pre-dates the agent-id annotation). Safe
      // to reclaim.
      drop.push({ pvcName: pvc.name, agentId: pvc.agentId, lastWakeAt: null, departedAt: null, reason: 'orphan' })
      continue
    }
    if (a.departedAt) {
      drop.push({ pvcName: pvc.name, agentId: pvc.agentId, lastWakeAt: a.lastWakeAt, departedAt: a.departedAt, reason: 'departed' })
      continue
    }
    if (a.lastWakeAt && a.lastWakeAt.getTime() < cutoff) {
      drop.push({ pvcName: pvc.name, agentId: pvc.agentId, lastWakeAt: a.lastWakeAt, departedAt: null, reason: 'idle' })
      continue
    }
    if (!a.lastWakeAt) {
      // Agent has a PVC but never logged a run. This usually means
      // the PVC was created on first wake but the agent crashed
      // before agent_runs got an insert. Skip — too risky to delete
      // a fresh PVC that just hasn't seen activity yet. The sweeper
      // will revisit next tick once the PVC is older than the
      // threshold (PVC creation timestamp is a stronger signal —
      // future iteration could read metadata.creationTimestamp).
      continue
    }
  }
  return drop
}

let chromePvcGcTimer: NodeJS.Timeout | null = null

async function listChromeProfilePvcs(): Promise<Array<{ name: string; agentId: string }>> {
  const out = await kubectlWithRetry(
    ['get', 'pvc', '-l', 'app=cumora-agent', '-o',
      'jsonpath={range .items[*]}{.metadata.name}|{.metadata.annotations.cumora\\.dev/agent-id}{"\\n"}{end}'],
    { timeoutMs: 20_000 },
  )
  if (out.code !== 0) {
    console.warn(`[chrome-pvc-gc] list failed: ${(out.err || out.out).trim()}`)
    return []
  }
  const pvcs: Array<{ name: string; agentId: string }> = []
  for (const line of out.out.split('\n')) {
    if (!line.trim()) continue
    const [name, agentId] = line.split('|')
    if (!name) continue
    // Fallback when the annotation is missing (pre-annotation PVCs):
    // strip the `agent-` prefix + `-chrome` suffix from the K8s name.
    // Lossy for agent ids that had uppercase / special chars, but
    // covers the common DNS-1123-clean case (iris-0c97, bram-c078).
    const recoveredId = agentId?.trim() || name.replace(/^agent-/, '').replace(/-chrome$/, '')
    pvcs.push({ name, agentId: recoveredId })
  }
  return pvcs
}

async function loadAgentRecencyForGc(
  agentIds: string[],
): Promise<Map<string, { departedAt: Date | null; lastWakeAt: Date | null }>> {
  const map = new Map<string, { departedAt: Date | null; lastWakeAt: Date | null }>()
  if (agentIds.length === 0) return map
  const { rows } = await pool.query<{ id: string; departed_at: Date | null; last_wake_at: Date | null }>(
    `SELECT p.id, p.departed_at,
            (SELECT MAX(ar.started_at) FROM agent_runs ar WHERE ar.agent_id = p.id) AS last_wake_at
       FROM participants p
      WHERE p.id = ANY($1::text[]) AND p.kind = 'agent'`,
    [agentIds],
  )
  for (const r of rows) {
    map.set(r.id, { departedAt: r.departed_at, lastWakeAt: r.last_wake_at })
  }
  return map
}

/** Run one GC pass. Returns the list of PVCs deleted (for logging /
 *  tests). Idle threshold + interval are configured via env vars. */
export async function sweepIdleChromeProfilePvcs(opts: {
  idleThresholdMs: number
  /** Test seam — production callers omit. */
  now?: Date
} = { idleThresholdMs: 30 * 24 * 60 * 60_000 }): Promise<IdlePvcCandidate[]> {
  const pvcs = await listChromeProfilePvcs()
  if (pvcs.length === 0) return []
  const agents = await loadAgentRecencyForGc(pvcs.map((p) => p.agentId))
  const plan = planIdlePvcGc({
    pvcs, agents,
    now: opts.now ?? new Date(),
    idleThresholdMs: opts.idleThresholdMs,
  })
  const deleted: IdlePvcCandidate[] = []
  for (const cand of plan) {
    const r = await kubectlWithRetry(
      ['delete', 'pvc', cand.pvcName, '--ignore-not-found=true', '--wait=false'],
      { timeoutMs: 20_000 },
    )
    if (r.code === 0) {
      console.log(`[chrome-pvc-gc] dropped ${cand.pvcName} (${cand.reason}, agent=${cand.agentId})`)
      deleted.push(cand)
    } else {
      console.warn(`[chrome-pvc-gc] failed to delete ${cand.pvcName}: ${(r.err || r.out).trim()}`)
    }
  }
  return deleted
}

/** Boot the periodic sweeper. Idempotent — re-calling returns the
 *  existing handle. intervalMs <= 0 disables. */
export function startChromeProfilePvcGc(opts: {
  intervalMs: number
  idleThresholdMs: number
}): { stop(): void } | null {
  if (chromePvcGcTimer) return { stop: stopChromeProfilePvcGc }
  if (opts.intervalMs <= 0) {
    console.log('[chrome-pvc-gc] disabled (interval <= 0)')
    return null
  }
  console.log(`[chrome-pvc-gc] running every ${opts.intervalMs}ms · idle threshold ${opts.idleThresholdMs}ms`)
  const tick = async () => {
    try {
      const deleted = await sweepIdleChromeProfilePvcs({ idleThresholdMs: opts.idleThresholdMs })
      if (deleted.length > 0) {
        console.log(`[chrome-pvc-gc] swept ${deleted.length} PVC${deleted.length === 1 ? '' : 's'}`)
      }
    } catch (e) {
      console.error('[chrome-pvc-gc] tick crashed:', e instanceof Error ? e.message : String(e))
    }
  }
  chromePvcGcTimer = setInterval(() => { void tick() }, opts.intervalMs)
  chromePvcGcTimer.unref()
  return { stop: stopChromeProfilePvcGc }
}

export function stopChromeProfilePvcGc(): void {
  if (chromePvcGcTimer) { clearInterval(chromePvcGcTimer); chromePvcGcTimer = null }
}

// ─── Completed/Failed agent-pod garbage collection ──────────────────
//
// Plain Pods (not Jobs) don't have TTL-after-finished, so an idle-
// exited agent pod stays in Completed phase until something deletes
// it. In normal operation `ensurePod`'s reap-on-recreate path handles
// this, but for agents that never get re-woken (deleted personas,
// long-quiet workspaces) the leftovers pile up — we observed 180+
// Completed agent pods on prod during the FUSE-cap incident. They
// don't hold fuse slots but they slow kube-controller-manager, bloat
// `kubectl get pods` output, and leave dangling PVC bindings.

/** Pure helper — pick agent pods that should be GC'd given the JSON
 *  output of `kubectl get pods -l app=cumora-agent -o json`. Tests
 *  pin this directly without spawning kubectl. */
export function pickAgentPodsForGc(
  podsJson: string,
  now: number,
  minAgeMs: number,
): string[] {
  let data: { items?: Array<{ metadata?: { name?: string; creationTimestamp?: string }; status?: { phase?: string } }> }
  try {
    data = JSON.parse(podsJson) as typeof data
  } catch {
    return []
  }
  const out: string[] = []
  for (const p of data.items ?? []) {
    const phase = p.status?.phase
    if (phase !== 'Succeeded' && phase !== 'Failed' && phase !== 'Unknown') continue
    const name = p.metadata?.name
    if (!name) continue
    const ts = p.metadata?.creationTimestamp
    if (ts) {
      const ageMs = now - Date.parse(ts)
      if (!Number.isFinite(ageMs) || ageMs < minAgeMs) continue
    }
    out.push(name)
  }
  return out
}

/** Periodic sweep of finished agent pods. Safe to run from any number
 *  of cumora-server replicas — `kubectl delete --ignore-not-found`
 *  collapses concurrent calls. */
export async function gcCompletedAgentPods(opts: { minAgeMs?: number } = {}): Promise<{ scanned: number; deleted: number }> {
  const minAgeMs = opts.minAgeMs ?? 5 * 60_000
  const r = await kubectlWithRetry(
    ['get', 'pods', '-l', 'app=cumora-agent', '-o', 'json'],
    { timeoutMs: 10_000 },
  )
  if (r.code !== 0) return { scanned: 0, deleted: 0 }
  const victims = pickAgentPodsForGc(r.out, Date.now(), minAgeMs)
  if (victims.length === 0) return { scanned: 0, deleted: 0 }
  // Best-effort batch delete: one kubectl call with all names. Failures
  // here are non-fatal — the next sweep retries.
  const del = await kubectlWithRetry(
    ['delete', 'pod', ...victims, '--ignore-not-found=true', '--wait=false'],
    { timeoutMs: 30_000 },
  )
  if (del.code !== 0) {
    console.warn(`[orchestrator] gcCompletedAgentPods kubectl delete failed: ${(del.err || del.out).trim().slice(0, 200)}`)
  }
  return { scanned: victims.length, deleted: del.code === 0 ? victims.length : 0 }
}

// ─── cluster fuse-pressure monitor ──────────────────────────────────
//
// Detection used to rely on users complaining "agent 不回复". Now we
// sample the cluster periodically and fire notifyAlert when pending
// agent pods stay above a threshold for a sustained window. The
// alert payload includes the saturation ratio + pending count so the
// on-call can decide whether to bump max-nodes or hunt over-spawn.

interface FusePressureState {
  /** Most recent kubectl-get-pods scan. */
  lastSampleAt: number
  /** ms timestamp when the cluster first crossed the pending threshold. */
  pressureSince: number | null
  /** ms timestamp of the last alert fired. Cooldown source. */
  lastAlertAt: number | null
}

const fusePressureState: FusePressureState = {
  lastSampleAt: 0, pressureSince: null, lastAlertAt: null,
}

/** Test-only — reset the in-process state. */
export function _resetFusePressureStateForTests(): void {
  fusePressureState.lastSampleAt = 0
  fusePressureState.pressureSince = null
  fusePressureState.lastAlertAt = null
}

/** Pure decision: given a sample, should we alert? Stateful — mutates
 *  `state` in place. Exported so the unit test can pin every branch
 *  (rising edge fires, sustained fires once, recovery clears, alert
 *  cooldown holds). */
export function evaluateFusePressureSample(
  state: FusePressureState,
  pending: number,
  fuseRatio: number,
  now: number,
  thresholds: { pendingMin: number; ratioMin: number; sustainedMs: number; alertCooldownMs: number },
): { fire: boolean; reason: string } {
  state.lastSampleAt = now
  const pressured = pending >= thresholds.pendingMin || fuseRatio >= thresholds.ratioMin
  if (!pressured) {
    state.pressureSince = null
    return { fire: false, reason: 'no pressure' }
  }
  if (state.pressureSince === null) {
    state.pressureSince = now
    return { fire: false, reason: 'pressure just started — within sustain window' }
  }
  if (now - state.pressureSince < thresholds.sustainedMs) {
    return { fire: false, reason: 'pressure not yet sustained' }
  }
  if (state.lastAlertAt !== null && now - state.lastAlertAt < thresholds.alertCooldownMs) {
    return { fire: false, reason: 'within alert cooldown' }
  }
  state.lastAlertAt = now
  return { fire: true, reason: `pending=${pending} fuseRatio=${(fuseRatio * 100).toFixed(0)}%` }
}

/** One-shot sampler: take a fresh cluster reading, run the decision,
 *  fire the alert if needed. Exported for tests. */
export async function pollClusterFusePressureOnce(): Promise<void> {
  // Reuse the admission-control cache where possible — but force a
  // fresh sample if the cache is stale (>10s).
  fuseUtilCache = null
  const fuse = await getClusterFuseUtilization()
  const pendingProbe = await kubectlWithRetry(
    [
      'get', 'pods', '-l', 'app=cumora-agent',
      '--field-selector=status.phase=Pending',
      '--no-headers',
    ],
    { timeoutMs: 10_000 },
  )
  const pending = pendingProbe.code === 0
    ? pendingProbe.out.split('\n').filter((l) => l.trim().length > 0).length
    : 0
  const decision = evaluateFusePressureSample(
    fusePressureState, pending, fuse.ratio, Date.now(),
    {
      pendingMin: 20,
      ratioMin: 0.95,
      sustainedMs: 5 * 60_000,        // 5 min of sustained pressure
      alertCooldownMs: 30 * 60_000,   // re-alert at most every 30 min
    },
  )
  if (decision.fire) {
    console.warn(`[orchestrator] cluster fuse pressure ALERT: ${decision.reason}`)
    void notifyAlert({
      label: 'orchestrator.cluster_fuse_pressure',
      error: new Error(`agent-pod cluster under sustained fuse pressure: ${decision.reason}`),
      extras: {
        pendingAgentPods: pending,
        fuseUsed: fuse.cap === Number.POSITIVE_INFINITY ? null : fuse.used,
        fuseCap: fuse.cap === Number.POSITIVE_INFINITY ? null : fuse.cap,
        fuseRatio: fuse.ratio,
      },
    })
  }
}

/** Start the cluster fuse-pressure monitor. */
export function startClusterFuseMonitor(intervalMs: number = 60_000): NodeJS.Timeout {
  const tick = (): void => {
    void pollClusterFusePressureOnce().catch((e) =>
      console.warn('[orchestrator] pollClusterFusePressureOnce threw:', e instanceof Error ? e.message : String(e)),
    )
  }
  setImmediate(tick)
  const t = setInterval(tick, intervalMs)
  t.unref?.()
  return t
}

/** Start the GC loop. Returns the timer so the caller can clearInterval
 *  on shutdown. */
export function startCompletedPodGc(intervalMs: number = 60_000): NodeJS.Timeout {
  const tick = (): void => {
    void gcCompletedAgentPods().then((r) => {
      if (r.deleted > 0) {
        console.log(`[orchestrator] gc swept ${r.deleted} finished agent pods`)
      }
    }).catch((e) => {
      console.warn('[orchestrator] gcCompletedAgentPods threw:', e instanceof Error ? e.message : String(e))
    })
  }
  // Run once at boot, then on interval. tick.unref so the timer
  // doesn't keep the process alive on shutdown.
  setImmediate(tick)
  const t = setInterval(tick, intervalMs)
  t.unref?.()
  return t
}