Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript
44.0 KB
Raw Blame
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
import * as k8s from '@kubernetes/client-node'
import {
  asAppsApi,
  asCoreApi,
  asCustomApi,
  createMockAppsApi,
  createMockCoreApi,
  createMockCustomApi,
} from '../../test/__fixtures__/testMocks'
import { McpServerReconciler, RUNTIME_NOT_DESIRED_FAIL_CLOSED_MESSAGE } from '../reconciler'
import { McpServerCRD } from '../types'

vi.mock('../config', () => ({
  config: {
    devMode: false,
    port: 8081,
    namespace: 'mcp-server',
    hostNamespace: 'mcp-host',
    mcpServerImagePullPolicy: 'IfNotPresent',
    egressProxyImage: 'clerum/nginx-egress-proxy:0.1.0',
    stdioBridgeImage: 'clerum/stdio-bridge:test',
    stdioBridgeResources: {
      requests: { cpu: '50m', memory: '64Mi' },
      limits: { cpu: '200m', memory: '128Mi' },
    },
    devMcpServers: [],
    devContexts: [],
    devAuthTokens: new Map(),
  },
}))

const CREDENTIALS_REVISION_ANNOTATION = 'clerum.io/credentials-revision'

function makeServer(withEnvSecret = true): McpServerCRD {
  return {
    name: 'linear',
    namespace: 'mcp-server',
    uid: 'uid-linear-1234',
    spec: {
      contextRef: 'ctx1',
      image: 'linear-mcp:v1',
      transport: { type: 'streamableHttp', port: 3000, url: 'http://linear.mcp-server.svc:3000' },
      ...(withEnvSecret && {
        envSecret: {
          name: 'linear-credentials',
          keys: [{ secretKey: 'api-key', envVar: 'LINEAR_API_KEY' }],
        },
      }),
    },
  }
}

/** base64 of `value`, the form a Secret's `data` actually carries. */
function stored(value: string): string {
  return Buffer.from(value, 'utf8').toString('base64')
}

function createdDeployment(appsApi: ReturnType<typeof createMockAppsApi>): k8s.V1Deployment {
  expect(appsApi.createNamespacedDeployment).toHaveBeenCalledTimes(1)
  expect(appsApi.replaceNamespacedDeployment).not.toHaveBeenCalled()
  return (appsApi.createNamespacedDeployment.mock.calls[0][0] as { body: k8s.V1Deployment }).body
}

function replacedDeployment(appsApi: ReturnType<typeof createMockAppsApi>): k8s.V1Deployment {
  expect(appsApi.replaceNamespacedDeployment).toHaveBeenCalledTimes(1)
  expect(appsApi.createNamespacedDeployment).not.toHaveBeenCalled()
  return (appsApi.replaceNamespacedDeployment.mock.calls[0][0] as { body: k8s.V1Deployment }).body
}

function credentialsRevisionOf(deployment: k8s.V1Deployment): string | undefined {
  return deployment.spec?.template?.metadata?.annotations?.[CREDENTIALS_REVISION_ANNOTATION]
}

function readinessPollsOf(reconciler: McpServerReconciler): Map<string, unknown> {
  return (reconciler as unknown as { readinessPolls: Map<string, unknown> }).readinessPolls
}

function notConvergedDeployment(generation: number, name = 'linear') {
  return {
    metadata: {
      name,
      namespace: 'mcp-server',
      resourceVersion: '1',
      uid: 'linear-deployment',
      generation,
      labels: {
        'clerum.io/managed-by': 'host-context-controller',
        'clerum.io/mcpserver': 'linear',
      },
    },
    spec: { replicas: 1 },
    status: {
      observedGeneration: generation,
      replicas: 2,
      updatedReplicas: 1,
      readyReplicas: 1,
      availableReplicas: 1,
      unavailableReplicas: 0,
    },
  }
}

function holdNextStatusRead(customApi: ReturnType<typeof createMockCustomApi>): {
  release: (value?: unknown) => void
  resolved: () => number
  isHeld: () => boolean
} {
  let pending: ((value: unknown) => void) | undefined
  let resolved = 0
  let holdNext = true
  customApi.getNamespacedCustomObjectStatus.mockImplementation(
    () =>
      new Promise(resolve => {
        if (holdNext) {
          holdNext = false
          pending = value => {
            resolved += 1
            resolve(value)
          }
          return
        }
        resolve({ metadata: { resourceVersion: '1' }, status: { conditions: [] } })
      })
  )
  return {
    isHeld: () => pending !== undefined,
    resolved: () => resolved,
    release: (value = { metadata: { resourceVersion: '1' }, status: { conditions: [] } }) => {
      if (pending === undefined) {
        throw new Error('status GET was never held')
      }
      pending(value)
    },
  }
}

function laterConditionWrites(
  customApi: ReturnType<typeof createMockCustomApi>,
  afterCallCount: number
): Array<{ type: string; status: string; reason: string }> {
  return customApi.patchNamespacedCustomObjectStatus.mock.calls
    .slice(afterCallCount)
    .flatMap(call => {
      const body = (call[0] as { body?: unknown }).body
      return Array.isArray(body) ? body : []
    })
    .flatMap((op: unknown) => {
      const value = (op as { value?: unknown }).value
      const conditions = Array.isArray(value)
        ? value
        : ((value as { conditions?: unknown[] })?.conditions ?? [])
      return Array.isArray(conditions) ? conditions : []
    })
    .map((c: unknown) => c as { type: string; status: string; reason: string })
}

async function reconcileFailClosed(
  reconciler: McpServerReconciler,
  coreApi: ReturnType<typeof createMockCoreApi>
) {
  coreApi.readNamespacedSecret.mockRejectedValue(
    Object.assign(new Error('not found'), { code: 404 })
  )
  await reconciler.reconcile(makeServer())
}

/**
 * Every DeploymentReady condition written to the CRD, oldest first. Conditions
 * travel as a JSON Patch on the status subresource, so this digs the condition
 * out of the patch document.
 */
function deploymentReadyWrites(customApi: ReturnType<typeof createMockCustomApi>): Array<{
  status: string
  reason: string
  message: string
}> {
  return customApi.patchNamespacedCustomObjectStatus.mock.calls
    .flatMap(call => {
      const body = (call[0] as { body?: unknown }).body
      return Array.isArray(body) ? body : []
    })
    .flatMap((op: unknown) => {
      const value = (op as { value?: unknown }).value
      const conditions = Array.isArray(value)
        ? value
        : ((value as { conditions?: unknown[] })?.conditions ?? [])
      return Array.isArray(conditions) ? conditions : []
    })
    .filter((c: unknown) => (c as { type?: string }).type === 'DeploymentReady')
    .map((c: unknown) => c as { status: string; reason: string; message: string })
}

/** A Deployment whose rollout has fully converged. */
function convergedDeployment(name = 'linear') {
  return {
    metadata: {
      name,
      resourceVersion: '1',
      generation: 2,
      labels: {
        'clerum.io/managed-by': 'host-context-controller',
        'clerum.io/mcpserver': 'linear',
      },
    },
    spec: { replicas: 1 },
    status: {
      observedGeneration: 2,
      replicas: 1,
      updatedReplicas: 1,
      readyReplicas: 1,
      unavailableReplicas: 0,
    },
  }
}

describe('connector credentials revision (issue #223)', () => {
  let appsApi: ReturnType<typeof createMockAppsApi>
  let coreApi: ReturnType<typeof createMockCoreApi>
  let customApi: ReturnType<typeof createMockCustomApi>
  let reconciler: McpServerReconciler

  beforeEach(() => {
    appsApi = createMockAppsApi()
    coreApi = createMockCoreApi()
    customApi = createMockCustomApi()
    let live: k8s.V1Deployment | undefined
    // This suite creates a connector, then observes the same API object on rotation.
    appsApi.readNamespacedDeployment.mockImplementation(async () => {
      if (!live) throw { code: 404 }
      return structuredClone(live)
    })
    const persist = async ({ body }: { body: k8s.V1Deployment }) => {
      live = {
        ...structuredClone(body),
        metadata: { ...body.metadata, resourceVersion: '7', generation: 1 },
        status: { observedGeneration: 1, replicas: 1, updatedReplicas: 1, readyReplicas: 1 },
      }
      return live
    }
    appsApi.createNamespacedDeployment.mockImplementation(persist)
    appsApi.replaceNamespacedDeployment.mockImplementation(persist)
    coreApi.readNamespacedService.mockRejectedValueOnce({ code: 404 })
    coreApi.readNamespacedSecret.mockResolvedValue({
      metadata: { name: 'linear-credentials' },
      data: { 'api-key': stored('old-key') },
    })
    reconciler = new McpServerReconciler({} as k8s.KubeConfig, {
      assumeInventoryAuthorityWhenUnconfigured: true,
      appsApi: asAppsApi(appsApi),
      coreApi: asCoreApi(coreApi),
      customApi: asCustomApi(customApi),
    })
  })

  it('stamps the pod template with a revision derived from the secret content', async () => {
    await reconciler.reconcile(makeServer())

    const revision = credentialsRevisionOf(createdDeployment(appsApi))
    expect(revision).toMatch(/^[0-9a-f]{64}$/)
  })

  it('produces the same revision when only the key ORDER differs', async () => {
    coreApi.readNamespacedSecret.mockResolvedValue({
      data: { 'api-key': stored('k1'), 'webhook-secret': stored('k2') },
    })
    await reconciler.reconcile(makeServer())
    const first = credentialsRevisionOf(createdDeployment(appsApi))
    expect(first).toMatch(/^[0-9a-f]{64}$/)

    appsApi.readNamespacedDeployment.mockClear()
    coreApi.readNamespacedSecret.mockClear()
    appsApi.createNamespacedDeployment.mockClear()
    appsApi.replaceNamespacedDeployment.mockClear()
    coreApi.readNamespacedSecret.mockResolvedValue({
      data: { 'webhook-secret': stored('k2'), 'api-key': stored('k1') },
    })
    await reconciler.reconcile(makeServer())

    // Key order is an artifact of how the API server answered, not a change in
    // credentials: it must never roll a pod.
    expect(coreApi.readNamespacedSecret).toHaveBeenCalled()
    expect(appsApi.readNamespacedDeployment).toHaveBeenCalled()
    expect(appsApi.createNamespacedDeployment).not.toHaveBeenCalled()
    expect(appsApi.replaceNamespacedDeployment).not.toHaveBeenCalled()
  })

  it('produces a different revision when a credential VALUE changes', async () => {
    await reconciler.reconcile(makeServer())
    const before = credentialsRevisionOf(createdDeployment(appsApi))

    appsApi.createNamespacedDeployment.mockClear()
    appsApi.replaceNamespacedDeployment.mockClear()
    coreApi.readNamespacedSecret.mockResolvedValue({
      data: { 'api-key': stored('rotated-key') },
    })
    await reconciler.reconcile(makeServer())
    const after = credentialsRevisionOf(replacedDeployment(appsApi))

    // This difference IS the rollout: same pod template otherwise.
    expect(after).not.toBe(before)
  })

  it('writes no annotation for a connector without envSecret', async () => {
    await reconciler.reconcile(makeServer(false))

    const annotations = createdDeployment(appsApi).spec?.template?.metadata?.annotations
    expect(annotations?.[CREDENTIALS_REVISION_ANNOTATION]).toBeUndefined()
  })

  it('keeps the new revision when replacing over an existing Deployment', async () => {
    // The update path: read finds the existing Deployment, then replace runs the desired
    // object through preserveDeploymentAnnotations. The stale revision on the
    // live object must not win — otherwise a rotation would write the OLD
    // digest back and the pod would never roll.
    appsApi.readNamespacedDeployment.mockResolvedValue({
      metadata: {
        name: 'linear',
        resourceVersion: '7',
        generation: 1,
        labels: {
          'clerum.io/managed-by': 'host-context-controller',
          'clerum.io/mcpserver': 'linear',
        },
      },
      spec: {
        replicas: 1,
        template: {
          metadata: {
            annotations: {
              [CREDENTIALS_REVISION_ANNOTATION]: 'stale-revision',
              'operator.io/keep-me': 'yes',
            },
          },
        },
      },
      status: { observedGeneration: 1, replicas: 1, updatedReplicas: 1, readyReplicas: 1 },
    })

    await reconciler.reconcile(makeServer())

    const annotations = replacedDeployment(appsApi).spec?.template?.metadata?.annotations
    expect(annotations?.[CREDENTIALS_REVISION_ANNOTATION]).toMatch(/^[0-9a-f]{64}$/)
    expect(annotations?.[CREDENTIALS_REVISION_ANNOTATION]).not.toBe('stale-revision')
    // Unrelated operator annotations still survive the replace.
    expect(annotations?.['operator.io/keep-me']).toBe('yes')
  })
})

describe('generation-aware rollout readiness (issue #223)', () => {
  const appsApi = createMockAppsApi()
  const coreApi = createMockCoreApi()
  const customApi = createMockCustomApi()
  let reconciler: McpServerReconciler

  beforeEach(() => {
    vi.clearAllMocks()
    // Initial create reads are absent; preserve the queued rollout observations after it.
    appsApi.readNamespacedDeployment.mockRejectedValueOnce({ code: 404 })
    coreApi.readNamespacedService.mockRejectedValueOnce({ code: 404 })
    coreApi.readNamespacedSecret.mockResolvedValue({ data: { 'api-key': stored('old-key') } })
    reconciler = new McpServerReconciler({} as k8s.KubeConfig, {
      assumeInventoryAuthorityWhenUnconfigured: true,
      appsApi: asAppsApi(appsApi),
      coreApi: asCoreApi(coreApi),
      customApi: asCustomApi(customApi),
    })
  })

  it('is NOT ready while the controller has not observed the current generation', async () => {
    appsApi.readNamespacedDeployment.mockResolvedValue({
      metadata: {
        name: 'linear',
        namespace: 'mcp-server',
        resourceVersion: '1',
        uid: 'linear-deployment',
        generation: 3,
        labels: {
          'clerum.io/managed-by': 'host-context-controller',
          'clerum.io/mcpserver': 'linear',
        },
      },
      spec: { replicas: 1 },
      // The old pod is still Ready — the trap the pre-#223 check fell into.
      status: { observedGeneration: 2, updatedReplicas: 1, readyReplicas: 1 },
    })

    await reconciler.reconcile(makeServer())

    expect(deploymentReadyWrites(customApi).at(-1)?.status).toBe('False')
  })

  it('is NOT ready while replicas are not yet updated', async () => {
    appsApi.readNamespacedDeployment.mockResolvedValue({
      metadata: {
        name: 'linear',
        namespace: 'mcp-server',
        resourceVersion: '1',
        uid: 'linear-deployment',
        generation: 2,
        labels: {
          'clerum.io/managed-by': 'host-context-controller',
          'clerum.io/mcpserver': 'linear',
        },
      },
      spec: { replicas: 2 },
      status: { observedGeneration: 2, updatedReplicas: 1, readyReplicas: 2 },
    })

    await reconciler.reconcile(makeServer())

    expect(deploymentReadyWrites(customApi).at(-1)?.status).toBe('False')
  })

  it('is NOT ready while a replica is unavailable', async () => {
    appsApi.readNamespacedDeployment.mockResolvedValue({
      metadata: {
        name: 'linear',
        namespace: 'mcp-server',
        resourceVersion: '1',
        uid: 'linear-deployment',
        generation: 2,
        labels: {
          'clerum.io/managed-by': 'host-context-controller',
          'clerum.io/mcpserver': 'linear',
        },
      },
      spec: { replicas: 1 },
      status: {
        observedGeneration: 2,
        updatedReplicas: 1,
        readyReplicas: 1,
        unavailableReplicas: 1,
      },
    })

    await reconciler.reconcile(makeServer())

    expect(deploymentReadyWrites(customApi).at(-1)?.status).toBe('False')
  })

  it('is ready only once the rollout has fully converged', async () => {
    appsApi.readNamespacedDeployment.mockResolvedValue(convergedDeployment('linear'))

    await reconciler.reconcile(makeServer())

    expect(deploymentReadyWrites(customApi).at(-1)?.status).toBe('True')
  })

  it('is NOT ready while a crash-looping new pod leaves the OLD pod behind (failed rollout, E2)', async () => {
    // The trap that a bare readyReplicas/unavailableReplicas check misses: with
    // replicas=1 + RollingUpdate(maxSurge=1, maxUnavailable=0), a new pod that
    // crash-loops on a bad credential never goes Ready, so K8s keeps the OLD
    // pod. That old pod holds readyReplicas=1 and availableReplicas=1, so
    // unavailableReplicas=0 and updatedReplicas=1 — everything looks converged
    // EXCEPT status.replicas, which stays at 2 (old + crash-looping new). This
    // is exactly the e2e E2 scenario; without `totalReplicas === updatedReplicas`
    // the poll reports the failed rotation as healthy.
    appsApi.readNamespacedDeployment.mockResolvedValue({
      metadata: {
        name: 'linear',
        namespace: 'mcp-server',
        resourceVersion: '1',
        uid: 'linear-deployment',
        generation: 2,
        labels: {
          'clerum.io/managed-by': 'host-context-controller',
          'clerum.io/mcpserver': 'linear',
        },
      },
      spec: { replicas: 1 },
      status: {
        observedGeneration: 2,
        replicas: 2, // old pod still present alongside the crash-looping new one
        updatedReplicas: 1,
        readyReplicas: 1, // the OLD pod masks the failure
        availableReplicas: 1,
        unavailableReplicas: 0,
      },
    })

    await reconciler.reconcile(makeServer())

    expect(deploymentReadyWrites(customApi).at(-1)?.status).toBe('False')
  })
})

/**
 * Composition tests. Each half works in isolation — the readiness check reads
 * the Deployment correctly, and updateStatusConditions writes the CRD
 * correctly — but before #223 nothing called the second after the first
 * converged, so a successful rotation never reached the CRD. These two tests
 * fail against that arrangement; they are the regression guard.
 */
describe('the readiness poll must publish its verdict on the CRD (issue #223)', () => {
  const appsApi = createMockAppsApi()
  const coreApi = createMockCoreApi()
  const customApi = createMockCustomApi()
  let reconciler: McpServerReconciler

  beforeEach(() => {
    vi.clearAllMocks()
    // Initial create reads are absent; preserve the queued rollout observations after it.
    appsApi.readNamespacedDeployment.mockRejectedValueOnce({ code: 404 })
    coreApi.readNamespacedService.mockRejectedValueOnce({ code: 404 })
    vi.useFakeTimers()
    coreApi.readNamespacedSecret.mockResolvedValue({ data: { 'api-key': stored('old-key') } })
    reconciler = new McpServerReconciler({} as k8s.KubeConfig, {
      assumeInventoryAuthorityWhenUnconfigured: true,
      appsApi: asAppsApi(appsApi),
      coreApi: asCoreApi(coreApi),
      customApi: asCustomApi(customApi),
    })
  })

  afterEach(() => {
    vi.useRealTimers()
  })

  it('writes DeploymentReady=True once the poll sees the rollout converge', async () => {
    // Mid-rollout at reconcile time (the normal state right after a write),
    // converged by the time the poll runs.
    appsApi.readNamespacedDeployment
      .mockResolvedValueOnce({
        metadata: {
          name: 'linear',
          namespace: 'mcp-server',
          resourceVersion: '1',
          uid: 'linear-deployment',
          generation: 2,
          labels: {
            'clerum.io/managed-by': 'host-context-controller',
            'clerum.io/mcpserver': 'linear',
          },
        },
        spec: { replicas: 1 },
        status: { observedGeneration: 1, updatedReplicas: 0, readyReplicas: 1 },
      })
      .mockResolvedValue(convergedDeployment('linear'))

    await reconciler.reconcile(makeServer())
    await vi.advanceTimersByTimeAsync(5000)

    expect(deploymentReadyWrites(customApi).at(-1)?.status).toBe('True')
  })

  it('converges from the immediate False to a later True, instead of staying False', async () => {
    appsApi.readNamespacedDeployment
      .mockResolvedValueOnce({
        metadata: {
          name: 'linear',
          namespace: 'mcp-server',
          resourceVersion: '1',
          uid: 'linear-deployment',
          generation: 2,
          labels: {
            'clerum.io/managed-by': 'host-context-controller',
            'clerum.io/mcpserver': 'linear',
          },
        },
        spec: { replicas: 1 },
        status: { observedGeneration: 1, updatedReplicas: 0, readyReplicas: 1 },
      })
      .mockResolvedValue(convergedDeployment('linear'))

    await reconciler.reconcile(makeServer())
    const beforePoll = deploymentReadyWrites(customApi).map(c => c.status)
    await vi.advanceTimersByTimeAsync(5000)
    const afterPoll = deploymentReadyWrites(customApi).map(c => c.status)

    // Generation-aware readiness makes the immediate verdict False by
    // construction, so a terminal False here would mean every successful
    // rotation is reported as a failure.
    expect(beforePoll).toContain('False')
    expect(afterPoll.at(-1)).toBe('True')
  })

  it('gives up loudly on the CRD, with the rollout numbers, when it never converges', async () => {
    appsApi.readNamespacedDeployment.mockResolvedValue({
      metadata: {
        name: 'linear',
        namespace: 'mcp-server',
        resourceVersion: '1',
        uid: 'linear-deployment',
        generation: 2,
        labels: {
          'clerum.io/managed-by': 'host-context-controller',
          'clerum.io/mcpserver': 'linear',
        },
      },
      spec: { replicas: 1 },
      status: {
        observedGeneration: 2,
        updatedReplicas: 1,
        readyReplicas: 0,
        unavailableReplicas: 1,
      },
    })

    await reconciler.reconcile(makeServer())
    // 24 polls × 5s, the full budget.
    await vi.advanceTimersByTimeAsync(24 * 5000)

    const last = deploymentReadyWrites(customApi).at(-1)
    expect(last?.status).toBe('False')
    expect(last?.reason).toBe('RolloutIncomplete')
    // The message must carry evidence, not just "not ready".
    expect(last?.message).toContain('ready 0/1')
    expect(last?.message).toContain('unavailable 1')
  })

  it('still reaches the terminal RolloutIncomplete verdict when repeated reconciles fire during the poll window (status-update-loop, E2)', async () => {
    // Failed credential rotation: the crash-looping new pod leaves the OLD pod
    // behind, so the rollout never converges (total 2 vs updated 1) — the
    // exact E2 shape. Deployment generation never changes: same rollout.
    appsApi.readNamespacedDeployment.mockResolvedValue({
      metadata: {
        name: 'linear',
        namespace: 'mcp-server',
        resourceVersion: '1',
        uid: 'linear-deployment',
        generation: 2,
        labels: {
          'clerum.io/managed-by': 'host-context-controller',
          'clerum.io/mcpserver': 'linear',
        },
      },
      spec: { replicas: 1 },
      status: {
        observedGeneration: 2,
        replicas: 2,
        updatedReplicas: 1,
        readyReplicas: 1,
        availableReplicas: 1,
        unavailableReplicas: 0,
      },
    })

    await reconciler.reconcile(makeServer())

    // Every status write fires a MODIFIED watch event on the McpServer, which
    // fires another reconcile MID-POLL. Before the fix each of these calls
    // cancelled the running poll and reset its attempt budget, so the poll
    // never exhausted and RolloutIncomplete was never written — the CRD sat
    // at WaitingForReplicas until the UI's generic timeout. 12 × 10s = 120s,
    // the full 24 × 5s budget, with a reconcile landing every 2 polls.
    for (let i = 0; i < 12; i++) {
      await vi.advanceTimersByTimeAsync(2 * 5000)
      await reconciler.reconcile(makeServer())
    }

    const last = deploymentReadyWrites(customApi).at(-1)
    expect(last?.status).toBe('False')
    // The reconcile that follows the terminal write must not downgrade the
    // verdict back to WaitingForReplicas — the LAST write stays terminal.
    expect(last?.reason).toBe('RolloutIncomplete')
    expect(last?.message).toContain('total 2')
  })

  it('grants a NEW rotation (bumped Deployment generation) a fresh poll budget mid-window', async () => {
    const notConverged = (generation: number) => ({
      metadata: {
        name: 'linear',
        namespace: 'mcp-server',
        resourceVersion: '1',
        uid: 'linear-deployment',
        generation,
        labels: {
          'clerum.io/managed-by': 'host-context-controller',
          'clerum.io/mcpserver': 'linear',
        },
      },
      spec: { replicas: 1 },
      status: {
        observedGeneration: generation,
        replicas: 2,
        updatedReplicas: 1,
        readyReplicas: 1,
        availableReplicas: 1,
        unavailableReplicas: 0,
      },
    })
    appsApi.readNamespacedDeployment.mockResolvedValue(notConverged(2))

    await reconciler.reconcile(makeServer())
    // 12 polls into the first window…
    await vi.advanceTimersByTimeAsync(12 * 5000)
    // …a SECOND rotation lands: new generation, still not converging.
    appsApi.readNamespacedDeployment.mockResolvedValue(notConverged(3))
    await vi.advanceTimersByTimeAsync(13 * 5000)

    // 125s elapsed — past the original 24 × 5s budget — but the new rollout
    // got a fresh window when its generation appeared at t=65s, so a terminal
    // verdict now would blame the new rollout with the old rollout's clock.
    expect(deploymentReadyWrites(customApi).some(w => w.reason === 'RolloutIncomplete')).toBe(false)

    // The fresh window exhausts its own 24 polls after the generation change.
    await vi.advanceTimersByTimeAsync(11 * 5000)
    const last = deploymentReadyWrites(customApi).at(-1)
    expect(last?.reason).toBe('RolloutIncomplete')
    expect(last?.message).toContain('generation 3/3')
  })

  it('a NEW generation arriving while the LAST tick is in flight still gets its own window (in-flight tick race)', async () => {
    // The race the timer-only re-entrancy guard cannot see: pollReadiness()
    // is called by a reconcile WHILE the budget-exhausting tick is suspended
    // inside its Deployment read. The tick's data predates the reconcile's
    // Deployment write (the new generation), and after the tick goes terminal
    // nothing else re-arms the poll — McpServer has no periodic resync and no
    // Deployment watch — so the CRD would keep the OLD generation's
    // RolloutIncomplete forever even though the NEW rollout converges.
    const notConverged = (generation: number) => ({
      metadata: {
        name: 'linear',
        namespace: 'mcp-server',
        resourceVersion: '1',
        uid: 'linear-deployment',
        generation,
        labels: {
          'clerum.io/managed-by': 'host-context-controller',
          'clerum.io/mcpserver': 'linear',
        },
      },
      spec: { replicas: 1 },
      status: {
        observedGeneration: generation,
        replicas: 2,
        updatedReplicas: 1,
        readyReplicas: 1,
        availableReplicas: 1,
        unavailableReplicas: 0,
      },
    })
    const converged = (generation: number) => ({
      metadata: {
        name: 'linear',
        namespace: 'mcp-server',
        resourceVersion: '1',
        uid: 'linear-deployment',
        generation,
        labels: {
          'clerum.io/managed-by': 'host-context-controller',
          'clerum.io/mcpserver': 'linear',
        },
      },
      spec: { replicas: 1 },
      status: {
        observedGeneration: generation,
        replicas: 1,
        updatedReplicas: 1,
        readyReplicas: 1,
        unavailableReplicas: 0,
      },
    })

    // Generation 2 crash-loops and never converges.
    appsApi.readNamespacedDeployment.mockResolvedValue(notConverged(2))
    await reconciler.reconcile(makeServer())

    // Burn 23 of the 24 attempts of the window.
    await vi.advanceTimersByTimeAsync(23 * 5000)

    // Tick 24 — the budget-exhausting one — starts its Deployment read, and
    // we hold that read open: the tick is now suspended mid-await, with the
    // OLD generation's data about to come back.
    let releaseInFlightRead: ((deployment: unknown) => void) | undefined
    appsApi.readNamespacedDeployment.mockImplementationOnce(
      () =>
        new Promise(resolve => {
          releaseInFlightRead = resolve
        })
    )
    await vi.advanceTimersByTimeAsync(5000)
    if (releaseInFlightRead === undefined) {
      throw new Error(
        'race precondition failed: tick 24 never started its Deployment read while suspended'
      )
    }

    // While that tick is in flight, the operator fixes the credential: the
    // reconcile writes the Deployment at generation 3 and, at the end of
    // performReconcile, calls pollReadiness() — landing exactly inside the
    // window the re-entrancy guard used to no-op.
    coreApi.readNamespacedSecret.mockResolvedValue({ data: { 'api-key': stored('fixed-key') } })
    appsApi.readNamespacedDeployment.mockResolvedValue(notConverged(3))
    await reconciler.reconcile(makeServer())

    // The suspended tick now completes with the data it read BEFORE the new
    // rollout existed: stale generation-2 numbers, budget exhausted.
    releaseInFlightRead(notConverged(2))
    await vi.advanceTimersByTimeAsync(0)

    // Generation 3 converges shortly after. Only a poll window that survived
    // the race can observe this.
    appsApi.readNamespacedDeployment.mockResolvedValue(converged(3))
    await vi.advanceTimersByTimeAsync(2 * 5000)

    // The final CRD verdict must belong to generation 3 — DeploymentReady=True.
    // A RolloutIncomplete write here would carry generation 2's stale numbers
    // and, with the window dead, would stick indefinitely.
    const writes = deploymentReadyWrites(customApi)
    expect(writes.at(-1)?.status).toBe('True')
    expect(writes.some(w => w.reason === 'RolloutIncomplete')).toBe(false)
  })

  it('open readiness window does not write RolloutIncomplete after fail-closed retirement', async () => {
    appsApi.readNamespacedDeployment.mockResolvedValue(notConvergedDeployment(2))
    await reconciler.reconcile(makeServer())
    const readsBeforeTick = appsApi.readNamespacedDeployment.mock.calls.length
    await vi.advanceTimersByTimeAsync(5000)
    expect(appsApi.readNamespacedDeployment.mock.calls.length).toBeGreaterThan(readsBeforeTick)

    const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined)
    try {
      await reconcileFailClosed(reconciler, coreApi)
      const lastAfterRetirement = deploymentReadyWrites(customApi).at(-1)
      expect(lastAfterRetirement?.reason).toBe('RuntimeNotDesired')
      expect(lastAfterRetirement?.message).toBe(RUNTIME_NOT_DESIRED_FAIL_CLOSED_MESSAGE)
      const warnsAfterRetirement = warn.mock.calls.length
      const writesAfterRetirement = customApi.patchNamespacedCustomObjectStatus.mock.calls.length

      await vi.advanceTimersByTimeAsync(24 * 5000)

      expect(
        laterConditionWrites(customApi, writesAfterRetirement).some(
          condition =>
            condition.type === 'DeploymentReady' && condition.reason === 'RolloutIncomplete'
        )
      ).toBe(false)
      expect(deploymentReadyWrites(customApi).at(-1)?.reason).toBe('RuntimeNotDesired')
      expect(warn.mock.calls.slice(warnsAfterRetirement).flat().join('\n')).not.toContain(
        'not ready after'
      )
    } finally {
      warn.mockRestore()
    }
  })

  it('in-flight ready-branch tick past map delete cannot write True after retirement', async () => {
    appsApi.readNamespacedDeployment.mockResolvedValue(notConvergedDeployment(2))
    await reconciler.reconcile(makeServer())

    const held = holdNextStatusRead(customApi)
    appsApi.readNamespacedDeployment.mockResolvedValue(convergedDeployment('linear'))
    await vi.advanceTimersByTimeAsync(5000)
    expect(held.isHeld()).toBe(true)
    expect(readinessPollsOf(reconciler).get('linear')).toBeUndefined()

    await reconcileFailClosed(reconciler, coreApi)
    expect(
      deploymentReadyWrites(customApi).some(
        write =>
          write.reason === 'RuntimeNotDesired' &&
          write.message === RUNTIME_NOT_DESIRED_FAIL_CLOSED_MESSAGE
      )
    ).toBe(true)
    const afterRetirement = customApi.patchNamespacedCustomObjectStatus.mock.calls.length

    held.release()
    await vi.advanceTimersByTimeAsync(0)
    expect(held.resolved()).toBe(1)

    const leaked = laterConditionWrites(customApi, afterRetirement)
    expect(
      leaked.some(
        condition =>
          (condition.type === 'DeploymentReady' && condition.status === 'True') ||
          (condition.type === 'Ready' &&
            condition.status === 'True' &&
            condition.reason === 'ReconcileSuccess')
      )
    ).toBe(false)
    expect(deploymentReadyWrites(customApi).at(-1)?.reason).toBe('RuntimeNotDesired')
    expect(reconciler.hasIncompleteReconciliation()).toBe(false)
  })

  it('in-flight ready tick cannot overwrite RuntimeNotDesired while retract write is in flight', async () => {
    appsApi.readNamespacedDeployment.mockResolvedValue(notConvergedDeployment(2))
    await reconciler.reconcile(makeServer())

    const held = holdNextStatusRead(customApi)
    appsApi.readNamespacedDeployment.mockResolvedValue(convergedDeployment('linear'))
    await vi.advanceTimersByTimeAsync(5000)
    expect(held.isHeld()).toBe(true)

    let releaseRetractPatch: (() => void) | undefined
    let retractPatchHeld!: () => void
    const retractPatchGate = new Promise<void>(resolve => {
      retractPatchHeld = resolve
    })
    const livePatch = customApi.patchNamespacedCustomObjectStatus.getMockImplementation()
    customApi.patchNamespacedCustomObjectStatus.mockImplementation(async req => {
      const body = (req as { body?: Array<{ op?: string; path?: string; value?: unknown }> }).body
      const op = body?.find(
        candidate =>
          candidate.op === 'add' &&
          (candidate.path === '/status' || candidate.path === '/status/conditions')
      )
      const conditions = (
        Array.isArray(op?.value)
          ? op.value
          : ((op?.value as { conditions?: Array<{ reason?: string }> }).conditions ?? [])
      ) as Array<{ reason?: string }>
      if (conditions.some(condition => condition.reason === 'RuntimeNotDesired')) {
        await new Promise<void>(resolve => {
          releaseRetractPatch = resolve
          retractPatchHeld()
        })
      }
      if (livePatch) return livePatch(req)
      return {}
    })

    const failClosed = reconcileFailClosed(reconciler, coreApi)
    try {
      // Race the held retract PATCH against reconcile return so a missing
      // RuntimeNotDesired write fails here instead of hanging the suite.
      const retractSeen = await Promise.race([
        retractPatchGate.then(() => 'retract' as const),
        failClosed.then(() => 'finished' as const),
      ])
      expect(retractSeen).toBe('retract')
      expect(releaseRetractPatch).toBeDefined()
      const writesWhileRetractHeld = customApi.patchNamespacedCustomObjectStatus.mock.calls.length

      held.release({ metadata: { resourceVersion: '2' }, status: { conditions: [] } })
      await vi.advanceTimersByTimeAsync(0)

      expect(
        laterConditionWrites(customApi, writesWhileRetractHeld).some(
          condition =>
            (condition.type === 'DeploymentReady' && condition.status === 'True') ||
            (condition.type === 'Ready' &&
              condition.status === 'True' &&
              condition.reason === 'ReconcileSuccess')
        )
      ).toBe(false)

      releaseRetractPatch?.()
      await failClosed
      expect(deploymentReadyWrites(customApi).at(-1)?.reason).toBe('RuntimeNotDesired')
      expect(reconciler.hasIncompleteReconciliation()).toBe(false)
    } finally {
      releaseRetractPatch?.()
      if (livePatch) {
        customApi.patchNamespacedCustomObjectStatus.mockImplementation(livePatch)
      } else {
        customApi.patchNamespacedCustomObjectStatus.mockReset()
      }
    }
  })

  it('in-flight exhaustion tick cannot write RolloutIncomplete after retirement', async () => {
    appsApi.readNamespacedDeployment.mockResolvedValue(notConvergedDeployment(2))
    await reconciler.reconcile(makeServer())
    await vi.advanceTimersByTimeAsync(23 * 5000)

    const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined)
    try {
      const held = holdNextStatusRead(customApi)
      await vi.advanceTimersByTimeAsync(5000)
      expect(held.isHeld()).toBe(true)
      expect(warn.mock.calls.flat().join('\n')).toContain('not ready after 24 polls')

      await reconcileFailClosed(reconciler, coreApi)
      const afterRetirement = customApi.patchNamespacedCustomObjectStatus.mock.calls.length
      held.release()
      await vi.advanceTimersByTimeAsync(0)
      expect(held.resolved()).toBe(1)

      expect(
        laterConditionWrites(customApi, afterRetirement).some(
          condition =>
            condition.type === 'DeploymentReady' && condition.reason === 'RolloutIncomplete'
        )
      ).toBe(false)
      expect(deploymentReadyWrites(customApi).at(-1)?.reason).toBe('RuntimeNotDesired')
    } finally {
      warn.mockRestore()
    }
  })

  it('retirement epoch stops a stale ready-branch tick after a later window was armed', async () => {
    appsApi.readNamespacedDeployment.mockResolvedValue(notConvergedDeployment(2))
    await reconciler.reconcile(makeServer())
    const windowOne = readinessPollsOf(reconciler).get('linear')
    expect(windowOne).toBeDefined()

    const held = holdNextStatusRead(customApi)
    appsApi.readNamespacedDeployment.mockResolvedValue(convergedDeployment('linear'))
    await vi.advanceTimersByTimeAsync(5000)
    expect(held.isHeld()).toBe(true)
    expect(readinessPollsOf(reconciler).get('linear')).toBeUndefined()

    coreApi.readNamespacedSecret.mockResolvedValue({ data: { 'api-key': stored('rotated-key') } })
    appsApi.readNamespacedDeployment.mockResolvedValue(notConvergedDeployment(3))
    await reconciler.reconcile(makeServer())
    const windowTwo = readinessPollsOf(reconciler).get('linear')
    expect(windowTwo).toBeDefined()
    expect(windowTwo).not.toBe(windowOne)

    await reconcileFailClosed(reconciler, coreApi)
    expect(
      deploymentReadyWrites(customApi).some(write => write.reason === 'RuntimeNotDesired')
    ).toBe(true)
    const afterRetirement = customApi.patchNamespacedCustomObjectStatus.mock.calls.length
    expect(readinessPollsOf(reconciler).has('linear')).toBe(false)
    const readsAfterRetract = appsApi.readNamespacedDeployment.mock.calls.length

    held.release()
    await vi.advanceTimersByTimeAsync(24 * 5000)
    expect(held.resolved()).toBe(1)
    expect(appsApi.readNamespacedDeployment.mock.calls.length).toBe(readsAfterRetract)

    const leaked = laterConditionWrites(customApi, afterRetirement)
    expect(
      leaked.some(
        condition =>
          (condition.type === 'DeploymentReady' && condition.status === 'True') ||
          (condition.type === 'Ready' && condition.status === 'True')
      )
    ).toBe(false)
    expect(deploymentReadyWrites(customApi).at(-1)?.reason).toBe('RuntimeNotDesired')
    expect(reconciler.hasIncompleteReconciliation()).toBe(false)
  })

  it('in-flight ready tick records obligation when the revision fence flips without retirement', async () => {
    let current = true
    appsApi.readNamespacedDeployment.mockResolvedValue(notConvergedDeployment(2))
    await reconciler.reconcile(makeServer(), { isCurrent: () => current })

    const held = holdNextStatusRead(customApi)
    appsApi.readNamespacedDeployment.mockResolvedValue(convergedDeployment('linear'))
    await vi.advanceTimersByTimeAsync(5000)
    expect(held.isHeld()).toBe(true)
    const writesWhileHeld = customApi.patchNamespacedCustomObjectStatus.mock.calls.length

    current = false
    held.release()
    await vi.advanceTimersByTimeAsync(0)
    expect(held.resolved()).toBe(1)
    expect(
      laterConditionWrites(customApi, writesWhileHeld).some(
        condition =>
          (condition.type === 'DeploymentReady' && condition.status === 'True') ||
          (condition.type === 'Ready' && condition.status === 'True')
      )
    ).toBe(false)
    expect(reconciler.hasIncompleteReconciliation()).toBe(true)
  })
})

describe('poll-window fence survives a superseding reconcile (F1, watch-recovery race)', () => {
  const appsApi = createMockAppsApi()
  const coreApi = createMockCoreApi()
  const customApi = createMockCustomApi()
  let reconciler: McpServerReconciler
  // The inventory-authority generation IS mcpWatchGeneration in production
  // (k8sClient wiring): a routine McpServer watch recovery bumps it.
  let authorityGeneration: number
  let currentServer: McpServerCRD

  // A rollout mid-flight (total 2 vs updated 1) that never converges on its own.
  const midRollout = {
    metadata: {
      name: 'linear',
      namespace: 'mcp-server',
      resourceVersion: '1',
      uid: 'linear-deployment',
      generation: 2,
      labels: {
        'clerum.io/managed-by': 'host-context-controller',
        'clerum.io/mcpserver': 'linear',
      },
    },
    spec: { replicas: 1 },
    status: {
      observedGeneration: 2,
      replicas: 2,
      updatedReplicas: 1,
      readyReplicas: 1,
      availableReplicas: 1,
      unavailableReplicas: 0,
    },
  }

  beforeEach(() => {
    vi.clearAllMocks()
    // Initial create reads are absent; preserve the queued rollout observations after it.
    appsApi.readNamespacedDeployment.mockRejectedValueOnce({ code: 404 })
    coreApi.readNamespacedService.mockRejectedValueOnce({ code: 404 })
    vi.useFakeTimers()
    authorityGeneration = 1
    currentServer = makeServer()
    coreApi.readNamespacedSecret.mockResolvedValue({ data: { 'api-key': stored('old-key') } })
    reconciler = new McpServerReconciler({} as k8s.KubeConfig, {
      assumeInventoryAuthorityWhenUnconfigured: true,
      appsApi: asAppsApi(appsApi),
      coreApi: asCoreApi(coreApi),
      customApi: asCustomApi(customApi),
    })
    reconciler.setInventoryAuthority(() => ({ known: true, generation: authorityGeneration }))
    reconciler.setResolveCurrentServer(name => (name === 'linear' ? currentServer : undefined))
  })

  afterEach(() => {
    vi.useRealTimers()
  })

  it('rebinds the fence so a converged rollout is recorded after a watch-generation bump (T1)', async () => {
    appsApi.readNamespacedDeployment.mockResolvedValue(midRollout)
    // R1 arms the window under authority generation 1 (timer pending).
    await reconciler.reconcile(makeServer())
    // A routine McpServer watch recovery bumps the inventory-authority generation.
    authorityGeneration = 2
    // R2, current under generation 2, finds R1's pending window and rebinds it.
    await reconciler.reconcile(makeServer())
    // The rollout converges before the (single) pending tick fires.
    appsApi.readNamespacedDeployment.mockResolvedValue(convergedDeployment('linear'))
    await vi.advanceTimersByTimeAsync(5000)

    // Without the rebind, R1's tick evaluates its stale generation-1 fence,
    // self-deletes, and DeploymentReady=True is never written.
    expect(deploymentReadyWrites(customApi).at(-1)?.status).toBe('True')
    expect(
      (reconciler as unknown as { readinessPolls: Map<string, unknown> }).readinessPolls.size
    ).toBe(0)
  })

  it('control: without the generation bump the rollout converges the same way (T2)', async () => {
    appsApi.readNamespacedDeployment.mockResolvedValue(midRollout)
    await reconciler.reconcile(makeServer())
    await reconciler.reconcile(makeServer()) // no bump
    appsApi.readNamespacedDeployment.mockResolvedValue(convergedDeployment('linear'))
    await vi.advanceTimersByTimeAsync(5000)

    expect(deploymentReadyWrites(customApi).at(-1)?.status).toBe('True')
  })

  it('rebind does NOT reset the attempt budget across the bump (E2 guard, T3)', async () => {
    appsApi.readNamespacedDeployment.mockResolvedValue(midRollout) // never converges
    await reconciler.reconcile(makeServer())
    await vi.advanceTimersByTimeAsync(2 * 5000) // 2 attempts consumed
    authorityGeneration = 2
    await reconciler.reconcile(makeServer()) // rebind — must NOT reset attempts
    await vi.advanceTimersByTimeAsync(21 * 5000) // 2 + 21 = 23 attempts, still short of 24

    expect(deploymentReadyWrites(customApi).some(w => w.reason === 'RolloutIncomplete')).toBe(false)
    await vi.advanceTimersByTimeAsync(5000) // the 24th tick
    expect(deploymentReadyWrites(customApi).some(w => w.reason === 'RolloutIncomplete')).toBe(true)
  })

  it('does NOT adopt a same-name recreate — a new uid gets a fresh window (T4)', async () => {
    appsApi.readNamespacedDeployment.mockResolvedValue(midRollout)
    await reconciler.reconcile(makeServer())
    const polls = (reconciler as unknown as { readinessPolls: Map<string, { attempts: number }> })
      .readinessPolls
    const firstWindow = polls.get('linear')
    await vi.advanceTimersByTimeAsync(12 * 5000)

    // Same name, NEW uid: identity retirement (beginStatusTracking), not a
    // rebind, must own this — a rebind adopting the recreate would carry the
    // old window's spent budget.
    const recreated = { ...makeServer(), uid: 'uid-linear-9999' }
    currentServer = recreated
    await reconciler.reconcile(recreated)

    const secondWindow = polls.get('linear')
    expect(secondWindow).not.toBe(firstWindow)
    expect(secondWindow?.attempts).toBe(0)
  })
})