Skip to content

Where agent teams gather. Cross-platform team chat where AI agents are first-class teammates — with cloud or bring-your-own (Claude Code / Codex) brains.

TypeScript
36.3 KB
Raw Blame
/**
 * Unit tests for the pure helpers in agents/runtime/orchestrator.ts.
 *
 * The full ensurePod path shells out to kubectl and writes to a real
 * k8s cluster — not testable as a unit. But the parsing /
 * classification logic that ensurePod relies on IS pure and is where
 * subtle bugs hide (e.g. did we correctly classify ImagePullBackOff
 * as stuck? did Unschedulable surface? did we coalesce
 * containerStatuses with initContainerStatuses?).
 *
 * This file pins:
 *   - parsePodHealth: every shape of kubectl get pod -o json the
 *     production code might encounter (Running, Pending+stuck,
 *     Unschedulable, missing fields, corrupted JSON)
 *   - stuckPendingReason: every stuck-vs-not classification
 *   - safeName: DNS-1123 sanitization + length cap
 *   - isRetryableKubectlError: each retryable + non-retryable pattern
 *
 * Run: node --import tsx --test server/src/__tests__/agents-orchestrator.test.ts
 */
import assert from 'node:assert/strict'
import { readFile } from 'node:fs/promises'
import { dirname, resolve } from 'node:path'
import { after, test } from 'node:test'
import { fileURLToPath } from 'node:url'
import { load, loadAll } from 'js-yaml'
import {
  _testing,
  isRetryableKubectlError,
  type KubectlResult,
  parsePodHealth,
  planIdlePvcGc,
  safeName,
  stuckPendingReason,
  waitForPodDeletion,
} from '../agents/runtime/orchestrator.js'
import { pool } from '../db/pool.js'

const {
  yamlQuote,
  dnsLabelValue,
  podManifest,
  isSecureManagedPod,
  isRequiredAgentNetworkPolicy,
} = _testing

const repoRoot = resolve(dirname(fileURLToPath(import.meta.url)), '../../..')
const readRepo = (path: string): Promise<string> => readFile(resolve(repoRoot, path), 'utf8')

after(async () => {
  // orchestrator.ts transitively imports inproc-client.ts → redis.ts
  // which opens Redis connections at module load. Disconnect on test
  // exit so the test process can shut down instead of hanging 60s on
  // dangling handles.
  try { await pool.end() } catch { /* ignore */ }
  try {
    const { redis, sub } = await import('../redis.js')
    redis.disconnect()
    sub.disconnect()
  } catch { /* ignore */ }
})

// ── parsePodHealth ─────────────────────────────────────────────────────

test('parsePodHealth: Running pod with no waiting reasons', () => {
  const h = parsePodHealth(JSON.stringify({
    status: {
      phase: 'Running',
      containerStatuses: [{ state: { running: { startedAt: '...' } } }],
    },
  }))
  assert.equal(h.phase, 'Running')
  assert.deepEqual(h.waitingReasons, [])
  assert.equal(h.unschedulable, false)
})

test('parsePodHealth: Pending pod with ImagePullBackOff surfaces the waiting reason', () => {
  const h = parsePodHealth(JSON.stringify({
    status: {
      phase: 'Pending',
      containerStatuses: [
        { state: { waiting: { reason: 'ImagePullBackOff', message: '...' } } },
      ],
    },
  }))
  assert.equal(h.phase, 'Pending')
  assert.deepEqual(h.waitingReasons, ['ImagePullBackOff'])
})

test('parsePodHealth: coalesces containerStatuses + initContainerStatuses', () => {
  const h = parsePodHealth(JSON.stringify({
    status: {
      phase: 'Pending',
      initContainerStatuses: [{ state: { waiting: { reason: 'CrashLoopBackOff' } } }],
      containerStatuses: [{ state: { waiting: { reason: 'PodInitializing' } } }],
    },
  }))
  assert.equal(h.waitingReasons.length, 2)
  assert.ok(h.waitingReasons.includes('CrashLoopBackOff'))
  assert.ok(h.waitingReasons.includes('PodInitializing'))
})

test('parsePodHealth: Unschedulable surfaces via the PodScheduled condition', () => {
  const h = parsePodHealth(JSON.stringify({
    status: {
      phase: 'Pending',
      conditions: [
        { type: 'PodScheduled', status: 'False', reason: 'Unschedulable' },
      ],
    },
  }))
  assert.equal(h.unschedulable, true)
})

test('parsePodHealth: PodScheduled=True does NOT flag unschedulable', () => {
  const h = parsePodHealth(JSON.stringify({
    status: {
      phase: 'Running',
      conditions: [{ type: 'PodScheduled', status: 'True' }],
    },
  }))
  assert.equal(h.unschedulable, false)
})

test('parsePodHealth: malformed JSON returns the empty/safe shape', () => {
  const h = parsePodHealth('not json {{{')
  assert.equal(h.phase, '')
  assert.deepEqual(h.waitingReasons, [])
  assert.equal(h.unschedulable, false)
})

test('parsePodHealth: empty object returns the empty/safe shape', () => {
  const h = parsePodHealth('{}')
  assert.equal(h.phase, '')
  assert.deepEqual(h.waitingReasons, [])
})

test('parsePodHealth: missing containerStatuses (just-created pod) does not crash', () => {
  const h = parsePodHealth(JSON.stringify({ status: { phase: 'Pending' } }))
  assert.equal(h.phase, 'Pending')
  assert.deepEqual(h.waitingReasons, [])
})

test('parsePodHealth: containerStatuses not an array (malformed kubectl output) is ignored', () => {
  // Defense in depth: if k8s ever changes the shape, we don't crash.
  const h = parsePodHealth(JSON.stringify({
    status: { phase: 'Running', containerStatuses: 'oops' },
  }))
  assert.equal(h.phase, 'Running')
  assert.deepEqual(h.waitingReasons, [])
})

test('parsePodHealth: container status without waiting reason is skipped', () => {
  const h = parsePodHealth(JSON.stringify({
    status: {
      phase: 'Running',
      containerStatuses: [
        { state: { running: {} } },                          // no waiting
        { state: { waiting: { reason: 'ContainerCreating' } } }, // valid
        { state: { waiting: {} } },                          // no reason
      ],
    },
  }))
  assert.deepEqual(h.waitingReasons, ['ContainerCreating'])
})

// ── stuckPendingReason ─────────────────────────────────────────────────

test('stuckPendingReason: ImagePullBackOff is stuck', () => {
  assert.equal(
    stuckPendingReason({ phase: 'Pending', waitingReasons: ['ImagePullBackOff'], unschedulable: false }),
    'ImagePullBackOff',
  )
})

test('stuckPendingReason: CrashLoopBackOff is stuck', () => {
  assert.equal(
    stuckPendingReason({ phase: 'Pending', waitingReasons: ['CrashLoopBackOff'], unschedulable: false }),
    'CrashLoopBackOff',
  )
})

test('stuckPendingReason: Unschedulable is stuck even with no waiting reason', () => {
  assert.equal(
    stuckPendingReason({ phase: 'Pending', waitingReasons: [], unschedulable: true }),
    'Unschedulable',
  )
})

test('stuckPendingReason: ContainerCreating is NOT stuck (legitimate pending)', () => {
  assert.equal(
    stuckPendingReason({ phase: 'Pending', waitingReasons: ['ContainerCreating'], unschedulable: false }),
    null,
  )
})

test('stuckPendingReason: PodInitializing is NOT stuck (init containers running)', () => {
  assert.equal(
    stuckPendingReason({ phase: 'Pending', waitingReasons: ['PodInitializing'], unschedulable: false }),
    null,
  )
})

test('stuckPendingReason: mixed waiting reasons returns the FIRST stuck one', () => {
  // A pod with init containers still running + main container stuck
  // — the orchestrator should reap.
  const r = stuckPendingReason({
    phase: 'Pending',
    waitingReasons: ['PodInitializing', 'ImagePullBackOff'],
    unschedulable: false,
  })
  assert.equal(r, 'ImagePullBackOff')
})

test('stuckPendingReason: empty waiting reasons + schedulable → not stuck', () => {
  assert.equal(
    stuckPendingReason({ phase: 'Pending', waitingReasons: [], unschedulable: false }),
    null,
  )
})

test('stuckPendingReason: each member of STUCK_WAITING_REASONS classified correctly', () => {
  const known = [
    'ImagePullBackOff', 'ErrImagePull', 'CrashLoopBackOff',
    'CreateContainerConfigError', 'CreateContainerError',
    'InvalidImageName', 'RunContainerError',
  ]
  for (const reason of known) {
    assert.equal(
      stuckPendingReason({ phase: 'Pending', waitingReasons: [reason], unschedulable: false }),
      reason,
      `${reason} should classify as stuck`,
    )
  }
})

// ── safeName ───────────────────────────────────────────────────────────

test('safeName: alphanumeric agentId passes through (lowercased)', () => {
  assert.equal(safeName('iris0c97'), 'agent-iris0c97')
})

test('safeName: hyphens in id are preserved', () => {
  assert.equal(safeName('atlas-7860'), 'agent-atlas-7860')
})

test('safeName: uppercase letters are lowercased (DNS-1123 requires)', () => {
  assert.equal(safeName('Nova-9BC6'), 'agent-nova-9bc6')
})

test('safeName: special chars are replaced with dashes', () => {
  assert.equal(safeName('user.name@example'), 'agent-user-name-example')
})

test('safeName: leading/trailing dashes are stripped (DNS-1123 forbids them)', () => {
  // The slug stripper applies BEFORE the agent- prefix, so we test
  // the inner behavior: invalid leading chars don't sneak through.
  assert.equal(safeName('-foo-'), 'agent-foo')
})

test('safeName: extremely long ids are truncated to 63 chars', () => {
  const longId = 'a'.repeat(200)
  const out = safeName(longId)
  assert.ok(out.length <= 63, `result is ${out.length} chars, max 63`)
  assert.ok(out.startsWith('agent-'), 'still has the agent- prefix')
})

test('safeName: empty string still produces a valid prefix', () => {
  assert.equal(safeName(''), 'agent-')
  // The empty slug is technically not a great pod name but the
  // caller is responsible for not passing empty agentIds; this just
  // confirms we don't crash.
})

test('safeName: all-invalid-chars id collapses to dashes then stripped', () => {
  // '___' has no alphanumerics → replaced to '---' → stripped → ''
  assert.equal(safeName('___'), 'agent-')
})

// ── isRetryableKubectlError ────────────────────────────────────────────

function mkRes(over: Partial<KubectlResult>): KubectlResult {
  return { code: 1, out: '', err: '', timedOut: false, ...over }
}

test('isRetryableKubectlError: code=0 is never retryable (success)', () => {
  assert.equal(isRetryableKubectlError(mkRes({ code: 0 })), false)
})

test('isRetryableKubectlError: our timeout (code=124) IS retryable', () => {
  assert.equal(isRetryableKubectlError(mkRes({ code: 124, timedOut: true })), true)
})

test('isRetryableKubectlError: "connection refused" is retryable', () => {
  assert.equal(isRetryableKubectlError(mkRes({
    err: 'Unable to connect to the server: dial tcp 1.2.3.4:443: connection refused',
  })), true)
})

test('isRetryableKubectlError: "i/o timeout" is retryable', () => {
  assert.equal(isRetryableKubectlError(mkRes({
    err: 'net/http: TLS handshake timeout',
  })), true)
})

test('isRetryableKubectlError: "the server is currently unable" is retryable (5xx)', () => {
  assert.equal(isRetryableKubectlError(mkRes({
    err: 'Error from server (ServiceUnavailable): the server is currently unable to handle the request',
  })), true)
})

test('isRetryableKubectlError: TooManyRequests is retryable (rate-limit)', () => {
  assert.equal(isRetryableKubectlError(mkRes({
    err: 'Error from server (TooManyRequests): rate limit exceeded',
  })), true)
})

test('isRetryableKubectlError: validation errors are NOT retryable', () => {
  // A malformed manifest won't fix itself by retrying.
  assert.equal(isRetryableKubectlError(mkRes({
    err: 'error validating data: ValidationError(Pod.spec): unknown field "foo"',
  })), false)
})

test('isRetryableKubectlError: "already exists" is NOT retryable', () => {
  // Idempotency conflict — retrying would just hit the same error.
  assert.equal(isRetryableKubectlError(mkRes({
    err: 'Error from server (AlreadyExists): pods "agent-iris" already exists',
  })), false)
})

test('isRetryableKubectlError: "not found" is NOT retryable', () => {
  assert.equal(isRetryableKubectlError(mkRes({
    err: 'Error from server (NotFound): pods "agent-x" not found',
  })), false)
})

test('isRetryableKubectlError: patterns also matched against stdout (some kubectl errors land there)', () => {
  // `kubectl apply` puts some errors on stdout in older versions.
  assert.equal(isRetryableKubectlError(mkRes({
    out: 'Unable to connect to the server: i/o timeout',
  })), true)
})

// ── yamlQuote / dnsLabelValue / podManifest ───────────────────────────

test('yamlQuote: escapes a value with embedded double quote', () => {
  const out = yamlQuote('hello "world"')
  // JSON.stringify yields a quoted YAML scalar — embedded quotes get
  // backslash-escaped, which YAML's double-quote scalar accepts.
  assert.equal(out, '"hello \\"world\\""')
})

test('yamlQuote: escapes embedded newline (multi-line agentIds, hypothetically)', () => {
  const out = yamlQuote('line-one\nline-two')
  assert.ok(!out.includes('\n'), 'no raw newline in output')
  assert.equal(out, '"line-one\\nline-two"')
})

test('yamlQuote: escapes embedded backslash', () => {
  const out = yamlQuote('a\\b')
  assert.equal(out, '"a\\\\b"')
})

test('yamlQuote: empty string → empty quoted scalar', () => {
  assert.equal(yamlQuote(''), '""')
})

test('dnsLabelValue: alphanumeric+dash passes through', () => {
  assert.equal(dnsLabelValue('iris-0c97'), 'iris-0c97')
})

test('dnsLabelValue: uppercase REJECTED (must throw)', () => {
  assert.throws(() => dnsLabelValue('Iris-0c97'), /invalid DNS-1123/)
})

test('dnsLabelValue: leading dash REJECTED', () => {
  assert.throws(() => dnsLabelValue('-iris'), /invalid DNS-1123/)
})

test('dnsLabelValue: trailing dash REJECTED', () => {
  assert.throws(() => dnsLabelValue('iris-'), /invalid DNS-1123/)
})

test('dnsLabelValue: dots REJECTED (k8s would 422)', () => {
  assert.throws(() => dnsLabelValue('iris.local'), /invalid DNS-1123/)
})

test('dnsLabelValue: empty string REJECTED', () => {
  assert.throws(() => dnsLabelValue(''), /invalid DNS-1123/)
})

test('dnsLabelValue: 63 chars OK, 64 REJECTED', () => {
  const ok = 'a' + 'b'.repeat(62)
  assert.equal(ok.length, 63)
  assert.equal(dnsLabelValue(ok), ok)
  const bad = 'a' + 'b'.repeat(63)
  assert.equal(bad.length, 64)
  assert.throws(() => dnsLabelValue(bad), /invalid DNS-1123/)
})

test('podManifest: malicious agentId is sanitized AND quoted — cannot escape the YAML', () => {
  // The historical bug shape this defends against: an agentId
  // containing `" maliciousField:` could escape the label/value
  // strings and inject arbitrary YAML keys. The layered defense:
  //   1. safeName collapses special chars to '-' before they reach
  //      the label position (DNS-1123 sanitization).
  //   2. yamlQuote wraps every CUMORA_AGENT_ID env value in a quoted
  //      scalar, so even if the sanitizer missed something, JSON
  //      escaping would block YAML key injection.
  const m = podManifest({
    agentId: 'iris" injected: "yes',
    token: 'jwt-token',
    image: 'quay.io/x/y:z',
    serverUrl: 'http://example/runtime',
    openaiKey: 'sk-test',
    openaiBaseUrl: '',
    idleMs: 600_000,
    noWorkMs: 90_000,
  })
  // The sanitized form lands as the pod name and label slug.
  assert.match(m, /name: agent-iris--injected---yes/)
  // CRITICAL: the RAW agentId, if it appears at all, is fully
  // escaped — no unescaped `"` followed by a YAML-keyish thing.
  // We grep for the spoofed top-level injection: `injected: "yes`
  // outside a quoted string. The whole raw agentId only appears
  // inside the CUMORA_AGENT_ID env value (which uses yamlQuote).
  const rawAppearances = m.match(/iris" injected: "yes/g) ?? []
  // It DOES appear inside the escaped env value — but always
  // surrounded by JSON-escaped quotes (\")
  for (const idx of rawAppearances.map((_, i) => m.indexOf('iris" injected: "yes', i))) {
    // 2 chars before the match should be a backslash + quote pair
    // (the JSON-escape pattern `\"`).
    const prefix = m.slice(Math.max(0, idx - 2), idx)
    assert.equal(prefix, '\\"',
      `raw agentId at index ${idx} must be preceded by \\" (JSON-escaped quote)`)
  }
})

test('podManifest: well-formed inputs produce a parseable manifest with the slug label', () => {
  const m = podManifest({
    agentId: 'iris-0c97',
    token: 'jwt.token.part',
    image: 'quay.io/img:tag',
    serverUrl: 'http://server/runtime',
    openaiKey: 'sk-abc',
    openaiBaseUrl: 'https://sub2api/v1',
    idleMs: 600_000,
    noWorkMs: 90_000,
  })
  assert.match(m, /name: agent-iris-0c97/)
  assert.match(m, /cumora.agent: "iris-0c97"/)
  assert.match(m, /CUMORA_AGENT_ID/)
  assert.match(m, /value: "iris-0c97"/)
  assert.match(m, /CUMORA_AGENT_IDLE_MS/)
  assert.match(m, /value: "600000"/)
  assert.match(m, /CUMORA_AGENT_NO_WORK_MS/)
  assert.match(m, /value: "90000"/)
  assert.match(m, /CUMORA_AGENT_RUNTIME_TOKEN/)
  assert.match(m, /jwt\.token\.part/)
  // Resource targets are pinned against measured in-container working-
  // set + CPU figures: idle ≈ 380MiB, peak ≈ 1230MiB across 4 sites,
  // CPU bursts to ~2.2 cores during page render. A regression that
  // quietly downgrades Chromium's headroom should surface as a test
  // failure rather than as OOM-loops / throttling in prod.
  assert.match(m, /requests:\n\s+# Measured in prod, per-pod working-set ≈ 420 MiB/)
  assert.match(m, /cpu: "50m"/)
  assert.match(m, /memory: "320Mi"/)
  assert.match(m, /limits:\n\s+# Measured under realistic workload/)
  assert.match(m, /cpu: "2500m"/)
  assert.match(m, /memory: "1\.5Gi"/)
  assert.match(m, /devic\.es\/fuse: 1/)
  // Chrome profile volume — default ON-PVC. Off-PVC variant is the
  // emptyDir escape hatch tested separately.
  assert.match(m, /volumeMounts:\n\s+- name: chrome-profile\n\s+mountPath: \/opt\/chrome-profile/)
  assert.match(m, /volumes:\n\s+- name: chrome-profile\n\s+persistentVolumeClaim:\n\s+claimName: "agent-iris-0c97-chrome"/)
})

test('podManifest: parsed API shape contains the complete bootstrap security contract', () => {
  const image = 'quay.io/img:security-test'
  const manifest = podManifest({
    agentId: 'iris-0c97',
    token: 'jwt.token.part',
    image,
    serverUrl: 'http://server/runtime',
    openaiKey: 'sk-test',
    openaiBaseUrl: '',
    idleMs: 600_000,
    noWorkMs: 90_000,
  })
  const pod = load(manifest) as unknown
  assert.equal(isSecureManagedPod(pod, image, 'iris-0c97'), true)
  const root = pod as {
    metadata?: { labels?: Record<string, string> }
    spec?: {
      automountServiceAccountToken?: boolean
      securityContext?: {
        runAsUser?: number
        runAsGroup?: number
        runAsNonRoot?: boolean
        fsGroup?: number
        seccompProfile?: { type?: string }
      }
      containers?: Array<{
        command?: string[]
        securityContext?: {
          runAsUser?: number
          runAsGroup?: number
          capabilities?: { drop?: string[]; add?: string[] }
        }
      }>
    }
  }
  assert.equal(root.metadata?.labels?.['cumora.dev/security-profile'], 'fuse-demoted-v1')
  assert.equal(root.spec?.automountServiceAccountToken, false)
  assert.equal(root.spec?.securityContext?.runAsUser, 0)
  assert.equal(root.spec?.securityContext?.runAsGroup, 0)
  assert.equal(root.spec?.securityContext?.fsGroup, 65532)
  assert.equal(root.spec?.securityContext?.seccompProfile?.type, 'RuntimeDefault')
  const security = root.spec?.containers?.[0]
  assert.deepEqual(security?.command, ['/usr/local/bin/cumora-agent-bootstrap'])
  assert.equal(security?.securityContext?.runAsUser, 0)
  assert.equal(security?.securityContext?.runAsGroup, 0)
  assert.deepEqual(security?.securityContext?.capabilities?.drop, ['ALL'])
  assert.deepEqual(
    [...(security?.securityContext?.capabilities?.add ?? [])].sort(),
    ['KILL', 'SETGID', 'SETPCAP', 'SETUID', 'SYS_ADMIN'],
  )
})

test('isSecureManagedPod: stale label and legacy command cannot be reused', () => {
  const image = 'quay.io/img:security-test'
  const pod = load(podManifest({
    agentId: 'iris-0c97',
    token: 'jwt',
    image,
    serverUrl: 'http://server/runtime',
    openaiKey: 'sk-test',
    openaiBaseUrl: '',
    idleMs: 600_000,
    noWorkMs: 90_000,
  })) as Record<string, unknown>
  const spec = pod.spec as Record<string, unknown>
  const container = (spec.containers as Array<Record<string, unknown>>)[0]
  container.command = ['/usr/local/bin/agent-computer-entrypoint.sh']
  assert.equal(isSecureManagedPod(pod, image, 'iris-0c97'), false)
})

test('isSecureManagedPod: namespace/process/mount hooks cannot shadow the trusted bootstrap', () => {
  const image = 'quay.io/img:security-test'
  const secure = load(podManifest({
    agentId: 'iris-0c97',
    token: 'jwt',
    image,
    serverUrl: 'http://server/runtime',
    openaiKey: 'sk-test',
    openaiBaseUrl: '',
    idleMs: 600_000,
    noWorkMs: 90_000,
  })) as Record<string, unknown>
  const reject = (label: string, mutate: (pod: Record<string, unknown>) => void) => {
    const candidate = structuredClone(secure) as Record<string, unknown>
    mutate(candidate)
    assert.equal(isSecureManagedPod(candidate, image, 'iris-0c97'), false, label)
  }
  const apiDefaulted = structuredClone(secure) as Record<string, unknown>
  const apiDefaultedSpec = apiDefaulted.spec as Record<string, unknown>
  for (const field of ['hostPID', 'hostIPC', 'hostNetwork', 'shareProcessNamespace']) delete apiDefaultedSpec[field]
  assert.equal(isSecureManagedPod(apiDefaulted, image, 'iris-0c97'), true, 'API omission of default-false host fields remains secure')
  const spec = (pod: Record<string, unknown>) => pod.spec as Record<string, unknown>
  const container = (pod: Record<string, unknown>) => (spec(pod).containers as Array<Record<string, unknown>>)[0]
  reject('host PID namespace is rejected', (pod) => { spec(pod).hostPID = true })
  reject('host IPC namespace is rejected', (pod) => { spec(pod).hostIPC = true })
  reject('host network namespace is rejected', (pod) => { spec(pod).hostNetwork = true })
  reject('shared process namespace is rejected', (pod) => { spec(pod).shareProcessNamespace = true })
  reject('init containers are rejected', (pod) => { spec(pod).initContainers = [{ name: 'shadow' }] })
  reject('ephemeral containers are rejected', (pod) => { spec(pod).ephemeralContainers = [{ name: 'shadow' }] })
  reject('container lifecycle hooks are rejected', (pod) => { container(pod).lifecycle = { postStart: { exec: { command: ['sh'] } } } })
  reject('extra args are rejected', (pod) => { container(pod).args = ['--legacy'] })
  reject('exec probes are rejected', (pod) => { container(pod).livenessProbe = { exec: { command: ['sh'] } } })
  reject('extra volume mounts are rejected', (pod) => {
    const mounts = container(pod).volumeMounts as Array<Record<string, unknown>>
    mounts.push({ name: 'shadow', mountPath: '/shadow' })
  })
  reject('hostPath volumes are rejected', (pod) => {
    const volume = (spec(pod).volumes as Array<Record<string, unknown>>)[0]
    delete volume.persistentVolumeClaim
    volume.hostPath = { path: '/' }
  })
  reject('a PVC for another agent is rejected', (pod) => {
    const volume = (spec(pod).volumes as Array<Record<string, unknown>>)[0]
    const pvc = volume.persistentVolumeClaim as Record<string, unknown>
    pvc.claimName = 'agent-other-chrome'
  })
})

test('secure reuse, legacy/unknown replacement, and bounded deletion are fail-closed', async () => {
  const image = 'quay.io/img:security-test'
  const secure = load(podManifest({
    agentId: 'iris-0c97',
    token: 'jwt',
    image,
    serverUrl: 'http://server/runtime',
    openaiKey: 'sk-test',
    openaiBaseUrl: '',
    idleMs: 600_000,
    noWorkMs: 90_000,
  })) as Record<string, unknown>
  assert.equal(isSecureManagedPod(secure, image, 'iris-0c97'), true, 'secure live object is reusable')
  assert.equal(isSecureManagedPod({ metadata: { labels: { app: 'cumora-agent' } } }, image, 'iris-0c97'), false, 'unknown object is not reusable')
  assert.equal(isRequiredAgentNetworkPolicy('cumora-agent-egress', null, 'default'), false, 'missing policy fails closed')

  const calls: string[][] = []
  let getCount = 0
  const eventuallyGone = await waitForPodDeletion('iris-0c97', async (args) => {
    calls.push(args)
    if (args[0] === 'delete') return { code: 0, out: '', err: '', timedOut: false }
    getCount++
    return getCount === 1
      ? { code: 0, out: JSON.stringify(secure), err: '', timedOut: false }
      : { code: 0, out: '', err: '', timedOut: false }
  }, { deadlineMs: 100, pollMs: 0, sleep: async () => {} })
  assert.deepEqual(eventuallyGone, { ok: true })
  assert.equal(calls[0][0], 'delete')
  assert.ok(calls.filter((args) => args[0] === 'get').length >= 2, 'waits for the old name to disappear')

  const stuck = await waitForPodDeletion('iris-0c97', async (args) => (
    args[0] === 'delete'
      ? { code: 0, out: '', err: '', timedOut: false }
      : { code: 0, out: JSON.stringify(secure), err: '', timedOut: false }
  ), { deadlineMs: 0, pollMs: 0, sleep: async () => {} })
  assert.equal(stuck.ok, false, 'a Pod that never disappears blocks replacement')

  const authFailure = await waitForPodDeletion('iris-0c97', async (args) => (
    args[0] === 'delete'
      ? { code: 0, out: '', err: '', timedOut: false }
      : { code: 1, out: '', err: 'exec: gke-gcloud-auth-plugin: not found', timedOut: false }
  ), { deadlineMs: 100, pollMs: 0, sleep: async () => {} })
  assert.equal(authFailure.ok, false, 'auth/plugin failure is not resource absence')
})

test('checked-in NetworkPolicies parse and match the orchestrator contract', async () => {
  const docs = loadAll(await readRepo('server/k8s/cumora-agent-network-policy.yaml')) as unknown[]
  assert.equal(docs.length, 2, 'policy bundle must contain only scoped deny and egress policies')
  const byName = new Map<string, unknown>()
  for (const doc of docs) {
    const metadata = (doc as { metadata?: { name?: unknown } } | null)?.metadata
    if (typeof metadata?.name === 'string') byName.set(metadata.name, doc)
  }
  for (const name of ['cumora-agent-default-deny', 'cumora-agent-egress']) {
    assert.equal(byName.has(name), true, `${name} must be present in the checked-in policy bundle`)
    assert.equal(isRequiredAgentNetworkPolicy(name, byName.get(name), 'default'), true, `${name} must satisfy v1`)
  }
  const deny = byName.get('cumora-agent-default-deny') as Record<string, unknown>
  const denySelector = ((deny.spec as Record<string, unknown>).podSelector as Record<string, unknown>).matchLabels as Record<string, string>
  const selected = (labels: Record<string, string>) => Object.entries(denySelector).every(([key, value]) => labels[key] === value)
  assert.equal(selected({ app: 'cumora-agent' }), true)
  assert.equal(selected({ app: 'cumora-server' }), false, 'default namespace server is not selected')
  assert.equal(selected({ app: 'redis' }), false, 'default namespace Redis is not selected')
  assert.equal(selected({ app: 'other-workload' }), false, 'other workloads are not selected')
  const broad = structuredClone(deny)
  ;(broad.spec as Record<string, unknown>).podSelector = {}
  assert.equal(isRequiredAgentNetworkPolicy('cumora-agent-default-deny', broad, 'default'), false, 'namespace-wide selector is rejected')
  const extraLabel = structuredClone(deny)
  const extraSelector = ((extraLabel.spec as Record<string, unknown>).podSelector as Record<string, unknown>).matchLabels as Record<string, string>
  extraSelector.team = 'agents'
  assert.equal(isRequiredAgentNetworkPolicy('cumora-agent-default-deny', extraLabel, 'default'), false, 'extra selector labels are rejected')
  const extraExpression = structuredClone(deny)
  const expressionSelector = (extraExpression.spec as Record<string, unknown>).podSelector as Record<string, unknown>
  expressionSelector.matchExpressions = [{ key: 'team', operator: 'Exists' }]
  assert.equal(isRequiredAgentNetworkPolicy('cumora-agent-default-deny', extraExpression, 'default'), false, 'selector expressions are rejected')
  const egress = byName.get('cumora-agent-egress') as Record<string, unknown>
  const egressSpec = egress.spec as Record<string, unknown>
  assert.deepEqual((egressSpec.podSelector as Record<string, unknown>).matchLabels, { app: 'cumora-agent' })
  const egressRules = egressSpec.egress as Array<Record<string, unknown>>
  const publicIpv4 = egressRules[2].to as Array<Record<string, unknown>>
  const block = publicIpv4[0].ipBlock as { except?: string[] }
  assert.ok(block.except?.includes('169.254.0.0/16'), 'metadata/link-local range must be excluded')
  assert.ok(block.except?.includes('100.64.0.0/10'), 'CGNAT range must be excluded')
  const tampered = structuredClone(egress)
  const tamperedRules = (tampered.spec as Record<string, unknown>).egress as Array<Record<string, unknown>>
  const tamperedBlock = ((tamperedRules[2].to as Array<Record<string, unknown>>)[0].ipBlock) as { except?: string[] }
  tamperedBlock.except = tamperedBlock.except?.filter((cidr) => cidr !== '100.64.0.0/10')
  assert.equal(isRequiredAgentNetworkPolicy('cumora-agent-egress', tampered, 'default'), false, 'a broad private-range exception is rejected')

  const wideDns = structuredClone(egress)
  const wideDnsRules = (wideDns.spec as Record<string, unknown>).egress as Array<Record<string, unknown>>
  const wideDnsPorts = wideDnsRules[0].ports as Array<Record<string, unknown>>
  wideDnsPorts[0].endPort = 65535
  assert.equal(isRequiredAgentNetworkPolicy('cumora-agent-egress', wideDns, 'default'), false, 'DNS port ranges are rejected')

  const wideServer = structuredClone(egress)
  const wideServerRules = (wideServer.spec as Record<string, unknown>).egress as Array<Record<string, unknown>>
  const wideServerPorts = wideServerRules[1].ports as Array<Record<string, unknown>>
  wideServerPorts[0].endPort = 65535
  assert.equal(isRequiredAgentNetworkPolicy('cumora-agent-egress', wideServer, 'default'), false, 'server port ranges are rejected')

  const wideWeb = structuredClone(egress)
  const wideWebRules = (wideWeb.spec as Record<string, unknown>).egress as Array<Record<string, unknown>>
  const wideWebPorts = wideWebRules[2].ports as Array<Record<string, unknown>>
  wideWebPorts[0].endPort = 65535
  assert.equal(isRequiredAgentNetworkPolicy('cumora-agent-egress', wideWeb, 'default'), false, 'public Web port ranges are rejected')
})

test('server RBAC manifests grant policy reads only and disable agent SA tokens', async () => {
  for (const path of ['server/k8s/cumora-server.gke.yaml', 'server/k8s/cumora-server.orbstack.yaml']) {
    const docs = loadAll(await readRepo(path)) as unknown[]
    const role = docs.find((doc) => {
      const object = doc as { kind?: unknown; metadata?: { name?: unknown } } | null
      return object?.kind === 'Role' && object.metadata?.name === 'cumora-server-agent-pods'
    }) as { rules?: Array<{ apiGroups?: string[]; resources?: string[]; verbs?: string[] }> } | undefined
    assert.ok(role, `${path} must define the server Role`)
    const policyRule = role.rules?.find((rule) => rule.resources?.includes('networkpolicies'))
    assert.deepEqual(policyRule?.apiGroups, ['networking.k8s.io'], `${path} policy API group`)
    assert.deepEqual(policyRule?.verbs, ['get', 'list', 'watch'], `${path} policy verbs are read-only`)
    const deployment = docs.find((doc) => (doc as { kind?: unknown } | null)?.kind === 'Deployment') as {
      spec?: { template?: { spec?: { containers?: Array<{ name?: string; automountServiceAccountToken?: boolean }> } } }
    } | undefined
    const server = deployment?.spec?.template?.spec?.containers?.find((container) => container.name === 'server')
    assert.ok(server, `${path} must contain the server container`)
    // Agent token mounting is controlled by the generated agent Pod, not the
    // server container; ensure the checked-in Role has no policy mutation path.
    assert.equal(policyRule?.verbs?.includes('create'), false)
  }
})

test('podManifest: image with embedded quote is escaped (CUMORA_AGENT_COMPUTER_IMAGE env is user-set)', () => {
  // An attacker who controlled CUMORA_AGENT_COMPUTER_IMAGE could
  // historically have tried `legit-image"\ninjected: yaml` to break
  // the manifest. yamlQuote wrapping makes it a literal string.
  const m = podManifest({
    agentId: 'iris-0c97',
    token: 't',
    image: 'evil"\ninjected: yaml',
    serverUrl: 'http://x',
    openaiKey: 'k',
    openaiBaseUrl: '',
    idleMs: 1000,
    noWorkMs: 90_000,
  })
  // The injected text appears ONCE, as part of the escaped image
  // value — NOT as a top-level YAML key.
  assert.ok(m.includes('"evil\\"\\ninjected: yaml"'),
    'image string is quoted-and-escaped, not interpolated raw')
})

test('isRetryableKubectlError: image-pull errors are NOT retryable at the kubectl level', () => {
  // The retry classification is for KUBECTL itself failing — image
  // pulls happen async inside k8s after apply succeeds. So a kubectl
  // apply with code=1 because the image doesn't exist gets reported
  // via "ImagePullBackOff" later, not at the apply step. We don't
  // retry kubectl-level "image not found" because it's the same
  // every time.
  assert.equal(isRetryableKubectlError(mkRes({
    err: 'failed to pull image "broken:latest": not found',
  })), false)
})

// ─── planIdlePvcGc ───────────────────────────────────────────────────
// Tests cover the decision matrix without touching kubectl or the DB —
// the planner is the only piece of the GC sweeper with non-trivial
// logic, and any future tweak (e.g. "also delete if PVC older than X
// AND no runs") should be reflected here first.

const NOW = new Date('2026-06-15T00:00:00Z')
const DAY_MS = 24 * 60 * 60_000
const IDLE_30D = 30 * DAY_MS

test('planIdlePvcGc: departed agent → drop', () => {
  const drop = planIdlePvcGc({
    pvcs: [{ name: 'agent-iris-chrome', agentId: 'iris' }],
    agents: new Map([['iris', { departedAt: new Date(NOW.getTime() - DAY_MS), lastWakeAt: new Date(NOW.getTime() - 2 * DAY_MS) }]]),
    now: NOW,
    idleThresholdMs: IDLE_30D,
  })
  assert.equal(drop.length, 1)
  assert.equal(drop[0].reason, 'departed')
})

test('planIdlePvcGc: recently woken active agent → keep', () => {
  const drop = planIdlePvcGc({
    pvcs: [{ name: 'agent-iris-chrome', agentId: 'iris' }],
    agents: new Map([['iris', { departedAt: null, lastWakeAt: new Date(NOW.getTime() - DAY_MS) }]]),
    now: NOW,
    idleThresholdMs: IDLE_30D,
  })
  assert.deepEqual(drop, [])
})

test('planIdlePvcGc: idle past threshold → drop', () => {
  const drop = planIdlePvcGc({
    pvcs: [{ name: 'agent-iris-chrome', agentId: 'iris' }],
    agents: new Map([['iris', { departedAt: null, lastWakeAt: new Date(NOW.getTime() - 45 * DAY_MS) }]]),
    now: NOW,
    idleThresholdMs: IDLE_30D,
  })
  assert.equal(drop.length, 1)
  assert.equal(drop[0].reason, 'idle')
})

test('planIdlePvcGc: orphan PVC (no participants row) → drop', () => {
  const drop = planIdlePvcGc({
    pvcs: [{ name: 'agent-ghost-chrome', agentId: 'ghost' }],
    agents: new Map(),
    now: NOW,
    idleThresholdMs: IDLE_30D,
  })
  assert.equal(drop.length, 1)
  assert.equal(drop[0].reason, 'orphan')
})

test('planIdlePvcGc: agent exists but never ran → keep (defer to next pass)', () => {
  // PVC was created just-in-time when the pod was about to spawn,
  // but the pod crashed before agent_runs got an insert. Deleting
  // a fresh PVC would lose a profile that just hasn't had a chance
  // to log activity yet. The PVC will be picked up next pass if
  // the agent still hasn't run (idle threshold catches it
  // eventually); for now, leave alone.
  const drop = planIdlePvcGc({
    pvcs: [{ name: 'agent-iris-chrome', agentId: 'iris' }],
    agents: new Map([['iris', { departedAt: null, lastWakeAt: null }]]),
    now: NOW,
    idleThresholdMs: IDLE_30D,
  })
  assert.deepEqual(drop, [])
})

test('planIdlePvcGc: mixed batch picks only the right ones', () => {
  const drop = planIdlePvcGc({
    pvcs: [
      { name: 'agent-iris-chrome', agentId: 'iris' },
      { name: 'agent-bram-chrome', agentId: 'bram' },
      { name: 'agent-nova-chrome', agentId: 'nova' },
      { name: 'agent-ghost-chrome', agentId: 'ghost' },
    ],
    agents: new Map([
      ['iris', { departedAt: null, lastWakeAt: new Date(NOW.getTime() - DAY_MS) }],        // active
      ['bram', { departedAt: new Date(NOW.getTime() - 5 * DAY_MS), lastWakeAt: null }],    // off-boarded
      ['nova', { departedAt: null, lastWakeAt: new Date(NOW.getTime() - 60 * DAY_MS) }],   // idle
      // ghost not in map → orphan
    ]),
    now: NOW,
    idleThresholdMs: IDLE_30D,
  })
  const byAgent = new Map(drop.map((d) => [d.agentId, d.reason]))
  assert.equal(drop.length, 3)
  assert.equal(byAgent.get('iris'), undefined)
  assert.equal(byAgent.get('bram'), 'departed')
  assert.equal(byAgent.get('nova'), 'idle')
  assert.equal(byAgent.get('ghost'), 'orphan')
})