Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript
90.8 KB
Raw Blame
import { describe, expect, it, vi } from 'vitest'
import * as k8s from '@kubernetes/client-node'
import { HostReconciler } from '../src/hostReconciler'
import { HostCRD, HostCrdStatus } from '../src/types'
import {
  type MockAppsApi,
  type MockCustomApi,
  asAppsApi,
  asCoreApi,
  asCustomApi,
  asNetworkingApi,
  asRbacApi,
  createMockAppsApi,
  createMockCoreApi,
  createMockCustomApi,
  createMockNetworkingApi,
  createMockRbacApi,
} from './__fixtures__/testMocks'

vi.mock('../src/config', () => ({
  config: {
    devMode: false,
    port: 8081,
    namespace: 'mcp-server',
    controlPlaneNamespace: 'control-plane',
    hostNamespace: 'mcp-host',
    rpcProxyNamespace: 'rpc-proxy',
    channelsNamespace: 'channels',
    channelReaderImage: 'clerum/channel-reader:test',
    channelReaderImagePullPolicy: 'IfNotPresent',
    hostImage: 'clerum/mcp-host:0.6.0',
    hostImagePullPolicy: 'Always',
    hostImagePullSecretName: 'clerum',
    hostPort: 8080,
    gfsNamespace: 'gfs',
    gfscPort: 8087,
    hostConfigMapName: 'mcp-host-config',
    hostServiceAccountName: 'mcp-host',
    hostWorkspaceStorageClassName: 'do-block-storage-retain',
    hostWorkspaceStorageSize: '10Gi',
    hostWorkspacePath: '/workspace',
    hostResources: {
      requests: { memory: '128Mi', cpu: '100m' },
      limits: { memory: '512Mi', cpu: '500m' },
    },
    desktopImage: 'clerum/mcp-host-desktop:latest',
    desktopPort: 3000,
    desktopResources: {
      requests: { memory: '256Mi', cpu: '250m' },
      limits: { memory: '4Gi', cpu: '1000m' },
    },
    devMcpServers: [],
    devContexts: [],
    devAuthTokens: new Map(),
    controlApiBaseUrl: 'http://control-api.test:8090',
    internalControlJwtHccHmacSecret: 'test-hcc-internal-control-secret',
    hccTargetNamespace: 'mcp-host',
    mcpHostGatewayUrl:
      'http://nginx-workflow-approval-gateway.control-plane.svc.cluster.local:8092',
  },
}))

// Short-circuit the HCC issuer so reconcile tests don't reach the network.
vi.mock('../src/mcpHostRuntimeTokenIssuerClient', () => ({
  issueMcpHostRuntimeTokens: vi.fn().mockResolvedValue({
    accessToken: 'test-mcp-host-runtime-access-token',
    refreshToken: 'test-mcp-host-runtime-refresh-token',
    mcpHostControlToken: 'test-mcp-host-workflow-control-token',
    channelReaderMessageToken: 'test-channel-reader-message-token',
    channelReaderApprovalToken: 'test-channel-reader-approval-token',
    channelReaderWorkflowApprovalDecisionToken: 'test-channel-reader-decision-token',
    channelReaderActivityToken: 'test-channel-reader-activity-token',
    channelReaderCronReadToken: 'test-channel-reader-cron-read-token',
    channelReaderCronAckToken: 'test-channel-reader-cron-ack-token',
    expiresInSeconds: 600,
    refreshExpiresInSeconds: 3600,
    controlExpiresInSeconds: 600,
    channelReaderMessageExpiresInSeconds: 600,
    channelReaderApprovalExpiresInSeconds: 600,
    channelReaderWorkflowApprovalDecisionExpiresInSeconds: 600,
  }),
}))

vi.mock('../src/gfsHostBinding', () => ({
  mintHostGfsToken: vi.fn(async (namespace: string, name: string) => ({
    ['to' + 'ken']: 'gfs-runtime-value',
    expiresInSeconds: 600,
    subject: `host:1st:${namespace}/${name}`,
  })),
}))

function makeStatelessHost(
  overrides: {
    name?: string
    spec?: Partial<HostCRD['spec']>
    status?: HostCrdStatus
    annotations?: Record<string, string>
  } = {}
): HostCRD {
  const name = overrides.name ?? 'stateless-host'
  return {
    name,
    namespace: 'mcp-host',
    ...(overrides.annotations ? { annotations: overrides.annotations } : {}),
    spec: {
      host: name,
      contextRef: 'context-a',
      secretRef: 'host-secret',
      lifecycle: { stateless: true },
      ...overrides.spec,
    },
    ...(overrides.status ? { status: overrides.status } : {}),
  }
}

function createReconciler(options: { isCommunicationChannelCacheSynced?: () => boolean } = {}) {
  const appsApi = createMockAppsApi()
  const coreApi = createMockCoreApi()
  const networkingApi = createMockNetworkingApi()
  const rbacApi = createMockRbacApi()
  const customApi = createMockCustomApi()
  customApi.getNamespacedCustomObject.mockResolvedValue({
    metadata: { name: 'stateless-host', namespace: 'mcp-host' },
    spec: {
      host: 'stateless-host',
      contextRef: 'context-a',
      secretRef: 'host-secret',
      lifecycle: { stateless: true },
    },
    status: { lifecycle: { state: 'suspended', wakeHandledGeneration: 0 } },
  })

  const reconciler = new HostReconciler({} as k8s.KubeConfig, {
    appsApi: asAppsApi(appsApi),
    coreApi: asCoreApi(coreApi),
    networkingApi: asNetworkingApi(networkingApi),
    rbacApi: asRbacApi(rbacApi),
    customApi: asCustomApi(customApi),
    now: () => new Date('2026-07-03T00:00:00.000Z'),
    // Heartbeat cases model the steady state after the watcher completed its
    // CommunicationChannel initial list. Cache-startup fail-closed behavior
    // is covered in hostReconciler.lifecycle.test.ts.
    isCommunicationChannelCacheSynced: options.isCommunicationChannelCacheSynced ?? (() => true),
  })

  return { reconciler, appsApi, coreApi, networkingApi, rbacApi, customApi }
}

function deferred<T = void>(): {
  promise: Promise<T>
  resolve: (value: T) => void
  reject: (reason?: unknown) => void
} {
  let resolve!: (value: T) => void
  let reject!: (reason?: unknown) => void
  const promise = new Promise<T>((res, rej) => {
    resolve = res
    reject = rej
  })
  return { promise, resolve, reject }
}

/** The mcp-host Deployment body sent to the K8s API (excludes channel-reader). */
function hostDeploymentBody(appsApi: MockAppsApi, name: string): k8s.V1Deployment {
  const calls = [
    ...appsApi.createNamespacedDeployment.mock.calls,
    ...appsApi.replaceNamespacedDeployment.mock.calls,
  ]
  const call = calls.find(
    ([arg]) => (arg as { body?: k8s.V1Deployment })?.body?.metadata?.name === name
  )
  if (!call) {
    throw new Error(`No Deployment create/replace call found for "${name}"`)
  }
  return (call[0] as { body: k8s.V1Deployment }).body
}

/**
 * Every /status value written through the CustomObjects API, reconstructed
 * from the JSON Patch ops. The lifecycle writers use three shapes:
 *   - full-status seed: `[{ add /status }]` (heartbeat cores + fresh-Host seed)
 *   - targeted sub-object: `[{ add /status/lifecycle }, { add /status/conditions }]`
 *     (writeLifecycleStatusToCluster, D2 — never clobbers un-computed fields)
 * either optionally preceded by an `add /metadata/resourceVersion` precondition
 * op (D3). This helper strips the precondition op and merges the remaining ops
 * into a single HostCrdStatus so assertions stay shape-agnostic.
 */
function lifecycleStatusWrites(customApi: MockCustomApi): HostCrdStatus[] {
  return customApi.patchNamespacedCustomObjectStatus.mock.calls.map(([arg]) => {
    const body = (arg as { body: Array<{ op: string; path: string; value: unknown }> }).body
    if (!Array.isArray(body)) {
      throw new Error(`Unexpected status patch body: ${JSON.stringify(body)}`)
    }
    const ops = body.filter(op => op.path !== '/metadata/resourceVersion')
    let status: HostCrdStatus = {}
    for (const op of ops) {
      if (op.path === '/status') {
        status = op.value as HostCrdStatus
      } else if (op.path === '/status/lifecycle') {
        status = { ...status, lifecycle: op.value as HostCrdStatus['lifecycle'] }
      } else if (op.path === '/status/conditions') {
        status = { ...status, conditions: op.value as HostCrdStatus['conditions'] }
      } else {
        throw new Error(`Unexpected status patch op path: ${JSON.stringify(op)}`)
      }
    }
    return status
  })
}

/**
 * Raw CustomObjects GET response served to the fresh-read guard inside the
 * heartbeat-path status writers (HostReconciler.readFreshHost). The guard is
 * authoritative over the caller's snapshot, so each test states the
 * server-side truth explicitly.
 */
function freshHostRead(lifecycle?: {
  state: 'active' | 'draining' | 'suspended'
  wakeHandledGeneration: number
  reason?: string
}) {
  return {
    metadata: { name: 'stateless-host', namespace: 'mcp-host' },
    spec: {
      host: 'stateless-host',
      contextRef: 'context-a',
      secretRef: 'host-secret',
      lifecycle: { stateless: true },
    },
    ...(lifecycle ? { status: { lifecycle } } : {}),
  }
}

describe('HostReconciler heartbeat mutation admission', () => {
  const heartbeatMutations = [
    {
      name: 'suspend',
      core: 'suspendHostFromHeartbeatCore',
      coreArgs: ['idle', 0] as const,
      freshLifecycle: { state: 'draining', wakeHandledGeneration: 0 } as const,
      invoke: (reconciler: HostReconciler, host: HostCRD) =>
        reconciler.suspendHostFromHeartbeat(host, 'idle', 0),
    },
    {
      name: 'suspend-blocked reason',
      core: 'publishSuspendBlockedReasonCore',
      coreArgs: ['suspend-blocked: activeTask'] as const,
      freshLifecycle: { state: 'active', wakeHandledGeneration: 0 } as const,
      invoke: (reconciler: HostReconciler, host: HostCRD) =>
        reconciler.publishSuspendBlockedReason(host, 'suspend-blocked: activeTask'),
    },
    {
      name: 'draining',
      core: 'markHostDrainingFromHeartbeatCore',
      coreArgs: [0] as const,
      freshLifecycle: { state: 'active', wakeHandledGeneration: 0 } as const,
      invoke: (reconciler: HostReconciler, host: HostCRD) =>
        reconciler.markHostDrainingFromHeartbeat(host, 0),
    },
    {
      name: 'cancel-drain',
      core: 'markHostActiveFromHeartbeatCore',
      coreArgs: [] as const,
      freshLifecycle: { state: 'draining', wakeHandledGeneration: 0 } as const,
      invoke: (reconciler: HostReconciler, host: HostCRD) =>
        reconciler.markHostActiveFromHeartbeat(host),
    },
  ] as const

  it.each(heartbeatMutations)(
    'rejects queued $name work when Host authority changes before serializer admission',
    async ({ core, invoke }) => {
      const { reconciler } = createReconciler()
      const host = { ...makeStatelessHost(), uid: 'heartbeat-host-uid', generation: 1 }
      const authority = { current: { known: true, generation: 7 } }
      reconciler.setResolveCurrentHost(name => (name === host.name ? host : undefined))
      reconciler.setHostWatchAuthority(() => authority.current)
      const lifecycle = (reconciler as any).lifecycle
      const coreSpy = vi.spyOn(lifecycle, core).mockResolvedValue(undefined)
      const blockerStarted = deferred()
      const releaseBlocker = deferred()
      const occupied = lifecycle.serializeByHost(host.name, async () => {
        blockerStarted.resolve(undefined)
        await releaseBlocker.promise
      }) as Promise<void>
      await blockerStarted.promise

      const pending = invoke(reconciler, host)
      authority.current = { known: false, generation: 8 }
      releaseBlocker.resolve(undefined)
      await occupied

      await expect(pending).rejects.toThrow(/Host inventory authority/)
      expect(coreSpy).not.toHaveBeenCalled()
    }
  )

  it.each(heartbeatMutations)(
    'rejects queued $name work when the Host UID changes before serializer admission',
    async ({ core, invoke }) => {
      const { reconciler } = createReconciler()
      const requested = { ...makeStatelessHost(), uid: 'heartbeat-host-old-uid', generation: 4 }
      let current = requested
      const authority = { current: { known: true, generation: 9 } }
      reconciler.setResolveCurrentHost(name => (name === requested.name ? current : undefined))
      reconciler.setHostWatchAuthority(() => authority.current)
      const lifecycle = (reconciler as any).lifecycle
      const coreSpy = vi.spyOn(lifecycle, core).mockResolvedValue(undefined)
      const blockerStarted = deferred()
      const releaseBlocker = deferred()
      const occupied = lifecycle.serializeByHost(requested.name, async () => {
        blockerStarted.resolve(undefined)
        await releaseBlocker.promise
      }) as Promise<void>
      await blockerStarted.promise

      const pending = invoke(reconciler, requested)
      current = { ...requested, uid: 'heartbeat-host-new-uid', generation: 1 }
      releaseBlocker.resolve(undefined)
      await occupied

      await expect(pending).rejects.toThrow(/Host identity/)
      expect(coreSpy).not.toHaveBeenCalled()
    }
  )

  it.each(heartbeatMutations)(
    'admits the latest same-UID same-spec Host snapshot for $name inside the heartbeat serializer',
    async ({ core, coreArgs, invoke }) => {
      const { reconciler } = createReconciler()
      const requested = { ...makeStatelessHost(), uid: 'heartbeat-host-uid', generation: 4 }
      const current = {
        ...requested,
        resourceVersion: 'cache-rv-latest',
        status: { lifecycle: { state: 'active' as const, wakeHandledGeneration: 2 } },
      }
      reconciler.setResolveCurrentHost(name => (name === requested.name ? current : undefined))
      reconciler.setHostWatchAuthority(() => ({ known: true, generation: 12 }))
      const lifecycle = (reconciler as any).lifecycle
      const coreSpy = vi.spyOn(lifecycle, core).mockResolvedValue(undefined)

      await invoke(reconciler, requested)

      expect(coreSpy).toHaveBeenCalledOnce()
      expect(coreSpy).toHaveBeenCalledWith(current, ...coreArgs, expect.any(Function))
    }
  )

  it.each(heartbeatMutations)(
    'rejects a $name 409 retry that resolves to a recreated same-name Host',
    async ({ freshLifecycle, invoke }) => {
      const { reconciler, customApi } = createReconciler()
      const host = {
        ...makeStatelessHost({ status: { lifecycle: freshLifecycle } }),
        uid: 'heartbeat-host-old-uid',
        generation: 4,
      }
      reconciler.setResolveCurrentHost(name => (name === host.name ? host : undefined))
      reconciler.setHostWatchAuthority(() => ({ known: true, generation: 15 }))
      customApi.getNamespacedCustomObject
        .mockResolvedValueOnce({
          ...freshHostRead(freshLifecycle),
          metadata: {
            name: host.name,
            namespace: host.namespace,
            uid: host.uid,
            generation: 4,
            resourceVersion: 'rv-old',
          },
        })
        .mockResolvedValueOnce({
          ...freshHostRead(freshLifecycle),
          metadata: {
            name: host.name,
            namespace: host.namespace,
            uid: 'heartbeat-host-new-uid',
            generation: 1,
            resourceVersion: 'rv-new',
          },
        })
      customApi.patchNamespacedCustomObjectStatus
        .mockRejectedValueOnce({ code: 409 })
        .mockResolvedValueOnce(undefined)

      await expect(invoke(reconciler, host)).rejects.toThrow(/Host identity/)
      expect(customApi.getNamespacedCustomObject).toHaveBeenCalledTimes(2)
      expect(customApi.patchNamespacedCustomObjectStatus).toHaveBeenCalledTimes(1)
    }
  )

  it.each(heartbeatMutations)(
    'rejects queued $name work when the same Host advances to a new spec generation',
    async ({ core, invoke }) => {
      const { reconciler } = createReconciler()
      const requested = { ...makeStatelessHost(), uid: 'heartbeat-host-uid', generation: 4 }
      let current = requested
      reconciler.setResolveCurrentHost(name => (name === requested.name ? current : undefined))
      reconciler.setHostWatchAuthority(() => ({ known: true, generation: 13 }))
      const lifecycle = (reconciler as any).lifecycle
      const coreSpy = vi.spyOn(lifecycle, core).mockResolvedValue(undefined)
      const blockerStarted = deferred()
      const releaseBlocker = deferred()
      const occupied = lifecycle.serializeByHost(requested.name, async () => {
        blockerStarted.resolve(undefined)
        await releaseBlocker.promise
      }) as Promise<void>
      await blockerStarted.promise

      const pending = invoke(reconciler, requested)
      current = {
        ...requested,
        generation: 5,
        spec: { ...requested.spec, contextRef: 'new-runtime-context' },
      }
      releaseBlocker.resolve(undefined)
      await occupied

      await expect(pending).rejects.toThrow(/Host spec generation/)
      expect(coreSpy).not.toHaveBeenCalled()
    }
  )

  it.each(heartbeatMutations)(
    'does not apply stale $name evidence to a newer same-UID stateless spec',
    async ({ freshLifecycle, invoke }) => {
      const { reconciler, customApi } = createReconciler()
      const host = {
        ...makeStatelessHost({ status: { lifecycle: freshLifecycle } }),
        uid: 'heartbeat-host-uid',
        generation: 4,
      }
      reconciler.setResolveCurrentHost(name => (name === host.name ? host : undefined))
      reconciler.setHostWatchAuthority(() => ({ known: true, generation: 14 }))
      customApi.getNamespacedCustomObject.mockResolvedValue({
        ...freshHostRead(freshLifecycle),
        metadata: {
          name: host.name,
          namespace: host.namespace,
          uid: host.uid,
          generation: 5,
          resourceVersion: 'rv-new-spec',
        },
        spec: { ...host.spec, contextRef: 'new-runtime-context' },
      })
      const reconcileCore = vi
        .spyOn(reconciler as any, 'reconcileCore')
        .mockResolvedValue(undefined)

      await invoke(reconciler, host)

      expect(customApi.patchNamespacedCustomObjectStatus).not.toHaveBeenCalled()
      expect(reconcileCore).not.toHaveBeenCalled()
    }
  )

  it.each(heartbeatMutations)(
    'does not apply stale $name evidence after the same-UID Host becomes stateful',
    async ({ invoke }) => {
      const { reconciler, customApi } = createReconciler()
      const host = { ...makeStatelessHost(), uid: 'heartbeat-host-uid', generation: 4 }
      reconciler.setResolveCurrentHost(name => (name === host.name ? host : undefined))
      reconciler.setHostWatchAuthority(() => ({ known: true, generation: 16 }))
      customApi.getNamespacedCustomObject.mockResolvedValue({
        ...freshHostRead({ state: 'active', wakeHandledGeneration: 0 }),
        metadata: {
          name: host.name,
          namespace: host.namespace,
          uid: host.uid,
          generation: 5,
          resourceVersion: 'rv-stateful',
        },
        spec: {
          ...host.spec,
          lifecycle: { stateless: false },
        },
      })
      const reconcileCore = vi
        .spyOn(reconciler as any, 'reconcileCore')
        .mockResolvedValue(undefined)

      await invoke(reconciler, host)

      expect(customApi.patchNamespacedCustomObjectStatus).not.toHaveBeenCalled()
      expect(reconcileCore).not.toHaveBeenCalled()
    }
  )

  it('does not retry stale heartbeat evidence after a 409 reveals a same-UID stateful spec', async () => {
    const { reconciler, customApi } = createReconciler()
    const host = { ...makeStatelessHost(), uid: 'heartbeat-host-uid', generation: 4 }
    reconciler.setResolveCurrentHost(name => (name === host.name ? host : undefined))
    reconciler.setHostWatchAuthority(() => ({ known: true, generation: 17 }))
    customApi.getNamespacedCustomObject
      .mockResolvedValueOnce({
        ...freshHostRead({ state: 'active', wakeHandledGeneration: 0 }),
        metadata: {
          name: host.name,
          namespace: host.namespace,
          uid: host.uid,
          generation: 4,
          resourceVersion: 'rv-stateless',
        },
      })
      .mockResolvedValueOnce({
        ...freshHostRead({ state: 'active', wakeHandledGeneration: 0 }),
        metadata: {
          name: host.name,
          namespace: host.namespace,
          uid: host.uid,
          generation: 5,
          resourceVersion: 'rv-stateful',
        },
        spec: {
          ...host.spec,
          lifecycle: { stateless: false },
        },
      })
    customApi.patchNamespacedCustomObjectStatus.mockRejectedValueOnce({ code: 409 })

    await reconciler.markHostDrainingFromHeartbeat(host, 0)

    expect(customApi.getNamespacedCustomObject).toHaveBeenCalledTimes(2)
    expect(customApi.patchNamespacedCustomObjectStatus).toHaveBeenCalledTimes(1)
  })

  it('does not retry stale heartbeat evidence after a 409 reveals a newer stateless spec', async () => {
    const { reconciler, customApi } = createReconciler()
    const host = { ...makeStatelessHost(), uid: 'heartbeat-host-uid', generation: 4 }
    reconciler.setResolveCurrentHost(name => (name === host.name ? host : undefined))
    reconciler.setHostWatchAuthority(() => ({ known: true, generation: 17 }))
    customApi.getNamespacedCustomObject
      .mockResolvedValueOnce({
        ...freshHostRead({ state: 'active', wakeHandledGeneration: 0 }),
        metadata: {
          name: host.name,
          namespace: host.namespace,
          uid: host.uid,
          generation: 4,
          resourceVersion: 'rv-old-spec',
        },
      })
      .mockResolvedValueOnce({
        ...freshHostRead({ state: 'active', wakeHandledGeneration: 0 }),
        metadata: {
          name: host.name,
          namespace: host.namespace,
          uid: host.uid,
          generation: 5,
          resourceVersion: 'rv-new-spec',
        },
        spec: { ...host.spec, contextRef: 'new-runtime-context' },
      })
    customApi.patchNamespacedCustomObjectStatus.mockRejectedValueOnce({ code: 409 })

    await reconciler.markHostDrainingFromHeartbeat(host, 0)

    expect(customApi.getNamespacedCustomObject).toHaveBeenCalledTimes(2)
    expect(customApi.patchNamespacedCustomObjectStatus).toHaveBeenCalledTimes(1)
  })

  it('revalidates authority after the fresh read and before the first status patch', async () => {
    const { reconciler, customApi } = createReconciler()
    const host = { ...makeStatelessHost(), uid: 'heartbeat-host-uid', generation: 4 }
    const authority = { current: { known: true, generation: 18 } }
    reconciler.setResolveCurrentHost(name => (name === host.name ? host : undefined))
    reconciler.setHostWatchAuthority(() => authority.current)
    customApi.getNamespacedCustomObject.mockImplementation(async () => {
      authority.current = { known: false, generation: 19 }
      return {
        ...freshHostRead({ state: 'active', wakeHandledGeneration: 0 }),
        metadata: {
          name: host.name,
          namespace: host.namespace,
          uid: host.uid,
          resourceVersion: 'rv-authority-lost',
        },
      }
    })

    await expect(reconciler.markHostDrainingFromHeartbeat(host, 0)).rejects.toThrow(
      /Host inventory authority/
    )
    expect(customApi.patchNamespacedCustomObjectStatus).not.toHaveBeenCalled()
  })

  it('revalidates authority before a 409 retry performs another fresh read', async () => {
    const { reconciler, customApi } = createReconciler()
    const host = { ...makeStatelessHost(), uid: 'heartbeat-host-uid', generation: 4 }
    const authority = { current: { known: true, generation: 21 } }
    reconciler.setResolveCurrentHost(name => (name === host.name ? host : undefined))
    reconciler.setHostWatchAuthority(() => authority.current)
    customApi.getNamespacedCustomObject.mockResolvedValue({
      ...freshHostRead({ state: 'active', wakeHandledGeneration: 0 }),
      metadata: {
        name: host.name,
        namespace: host.namespace,
        uid: host.uid,
        generation: 4,
        resourceVersion: 'rv-before-conflict',
      },
    })
    customApi.patchNamespacedCustomObjectStatus.mockImplementationOnce(async () => {
      authority.current = { known: false, generation: 22 }
      throw { code: 409 }
    })

    await expect(reconciler.markHostDrainingFromHeartbeat(host, 0)).rejects.toThrow(
      /Host inventory authority/
    )
    expect(customApi.getNamespacedCustomObject).toHaveBeenCalledTimes(1)
    expect(customApi.patchNamespacedCustomObjectStatus).toHaveBeenCalledTimes(1)
  })

  it('does not reconcile runtime effects after suspend when the Host UID changes', async () => {
    const { reconciler, customApi } = createReconciler()
    const host = {
      ...makeStatelessHost({
        status: { lifecycle: { state: 'draining', wakeHandledGeneration: 0 } },
      }),
      uid: 'heartbeat-host-old-uid',
      generation: 4,
    }
    let current = host
    reconciler.setResolveCurrentHost(name => (name === host.name ? current : undefined))
    reconciler.setHostWatchAuthority(() => ({ known: true, generation: 24 }))
    const reconcileCore = vi.spyOn(reconciler as any, 'reconcileCore').mockResolvedValue(undefined)
    customApi.getNamespacedCustomObject.mockResolvedValue({
      ...freshHostRead({ state: 'draining', wakeHandledGeneration: 0 }),
      metadata: {
        name: host.name,
        namespace: host.namespace,
        uid: host.uid,
        generation: 4,
        resourceVersion: 'rv-suspend',
      },
    })
    customApi.patchNamespacedCustomObjectStatus.mockImplementationOnce(async () => {
      current = { ...host, uid: 'heartbeat-host-new-uid', generation: 1 }
    })

    await expect(reconciler.suspendHostFromHeartbeat(host, 'idle', 0)).rejects.toThrow(
      /Host identity/
    )
    expect(reconcileCore).not.toHaveBeenCalled()
  })

  it('reconciles suspend follow-on effects from the successful fresh same-UID Host spec', async () => {
    const { reconciler, customApi } = createReconciler()
    const host = {
      ...makeStatelessHost({
        spec: { contextRef: 'latest-context' },
        status: { lifecycle: { state: 'draining', wakeHandledGeneration: 0 } },
      }),
      uid: 'heartbeat-host-uid',
      generation: 5,
    }
    reconciler.setResolveCurrentHost(name => (name === host.name ? host : undefined))
    reconciler.setHostWatchAuthority(() => ({ known: true, generation: 26 }))
    const reconcileCore = vi.spyOn(reconciler as any, 'reconcileCore').mockResolvedValue(undefined)
    customApi.getNamespacedCustomObject
      .mockResolvedValueOnce({
        ...freshHostRead({ state: 'draining', wakeHandledGeneration: 0 }),
        metadata: {
          name: host.name,
          namespace: host.namespace,
          uid: host.uid,
          generation: 5,
          resourceVersion: 'rv-suspend-current',
        },
        spec: { ...host.spec, contextRef: 'latest-context' },
      })
      .mockResolvedValue({
        ...freshHostRead({
          state: 'suspended',
          wakeHandledGeneration: 0,
          reason: 'idle',
        }),
        metadata: {
          name: host.name,
          namespace: host.namespace,
          uid: host.uid,
          generation: 5,
          resourceVersion: 'rv-suspend-committed',
        },
        spec: { ...host.spec, contextRef: 'latest-context' },
      })

    await reconciler.suspendHostFromHeartbeat(host, 'idle', 0)

    expect(reconcileCore).toHaveBeenCalledOnce()
    const reconciledHost = reconcileCore.mock.calls[0][0] as HostCRD
    expect(reconciledHost).toMatchObject({
      uid: host.uid,
      generation: 5,
      spec: expect.objectContaining({ contextRef: 'latest-context' }),
    })
    expect(reconciledHost.status?.lifecycle).toEqual({
      state: 'suspended',
      wakeHandledGeneration: 0,
      reason: 'idle',
    })
  })

  it('rejects a known-identity status write without a fresh resourceVersion', async () => {
    const { reconciler, customApi } = createReconciler()
    const host = { ...makeStatelessHost(), uid: 'heartbeat-host-uid', generation: 4 }
    reconciler.setResolveCurrentHost(name => (name === host.name ? host : undefined))
    reconciler.setHostWatchAuthority(() => ({ known: true, generation: 28 }))
    customApi.getNamespacedCustomObject.mockResolvedValue({
      ...freshHostRead({ state: 'active', wakeHandledGeneration: 0 }),
      metadata: { name: host.name, namespace: host.namespace, uid: host.uid },
    })

    await expect(reconciler.markHostDrainingFromHeartbeat(host, 0)).rejects.toThrow(
      /no fresh resourceVersion/
    )
    expect(customApi.patchNamespacedCustomObjectStatus).not.toHaveBeenCalled()
  })

  it('retries a 409 on the same UID and uses the latest Host for suspend follow-on effects', async () => {
    const { reconciler, customApi } = createReconciler()
    const host = {
      ...makeStatelessHost({
        status: { lifecycle: { state: 'draining', wakeHandledGeneration: 0 } },
      }),
      uid: 'heartbeat-host-uid',
      generation: 4,
    }
    let current = host
    reconciler.setResolveCurrentHost(name => (name === host.name ? current : undefined))
    reconciler.setHostWatchAuthority(() => ({ known: true, generation: 30 }))
    const reconcileCore = vi.spyOn(reconciler as any, 'reconcileCore').mockResolvedValue(undefined)
    customApi.getNamespacedCustomObject
      .mockResolvedValueOnce({
        ...freshHostRead({ state: 'draining', wakeHandledGeneration: 0 }),
        metadata: {
          name: host.name,
          namespace: host.namespace,
          uid: host.uid,
          generation: 4,
          resourceVersion: 'rv-1',
        },
      })
      .mockResolvedValueOnce({
        ...freshHostRead({ state: 'draining', wakeHandledGeneration: 0 }),
        metadata: {
          name: host.name,
          namespace: host.namespace,
          uid: host.uid,
          generation: 4,
          resourceVersion: 'rv-2',
        },
        spec: host.spec,
      })
      .mockResolvedValueOnce({
        ...freshHostRead({
          state: 'suspended',
          wakeHandledGeneration: 0,
          reason: 'idle',
        }),
        metadata: {
          name: host.name,
          namespace: host.namespace,
          uid: host.uid,
          generation: 4,
          resourceVersion: 'rv-3',
        },
        spec: host.spec,
      })
    customApi.patchNamespacedCustomObjectStatus
      .mockImplementationOnce(async () => {
        current = {
          ...host,
          resourceVersion: 'cache-rv-after-conflict',
        }
        throw { code: 409 }
      })
      .mockResolvedValueOnce(undefined)

    await reconciler.suspendHostFromHeartbeat(host, 'idle', 0)

    expect(customApi.getNamespacedCustomObject).toHaveBeenCalledTimes(3)
    expect(customApi.patchNamespacedCustomObjectStatus).toHaveBeenCalledTimes(2)
    expect(reconcileCore.mock.calls[0][0]).toMatchObject({
      uid: host.uid,
      generation: 4,
      spec: expect.objectContaining({ contextRef: 'context-a' }),
    })
  })

  it('does not apply or record suspension when only the follow-on GET observes a newer spec', async () => {
    const { reconciler, customApi } = createReconciler()
    const host = {
      ...makeStatelessHost({
        status: { lifecycle: { state: 'draining', wakeHandledGeneration: 0 } },
      }),
      uid: 'heartbeat-host-uid',
      generation: 4,
    }
    reconciler.setResolveCurrentHost(name => (name === host.name ? host : undefined))
    reconciler.setHostWatchAuthority(() => ({ known: true, generation: 33 }))
    customApi.getNamespacedCustomObject
      .mockResolvedValueOnce({
        ...freshHostRead({ state: 'draining', wakeHandledGeneration: 0 }),
        metadata: {
          name: host.name,
          namespace: host.namespace,
          uid: host.uid,
          generation: 4,
          resourceVersion: 'rv-suspend-write',
        },
      })
      .mockResolvedValueOnce({
        ...freshHostRead({
          state: 'suspended',
          wakeHandledGeneration: 0,
          reason: 'idle',
        }),
        metadata: {
          name: host.name,
          namespace: host.namespace,
          uid: host.uid,
          generation: 5,
          resourceVersion: 'rv-new-spec',
        },
        spec: {
          ...host.spec,
          contextRef: 'replacement-context',
          lifecycle: { stateless: false },
        },
      })
    const reconcileCore = vi.spyOn(reconciler as any, 'reconcileCore').mockResolvedValue(undefined)
    const recordSuspended = vi.spyOn((reconciler as any).lifecycle, 'recordSuspendedApplied')

    await expect(reconciler.suspendHostFromHeartbeat(host, 'idle', 0)).rejects.toThrow(
      /Host spec generation/
    )

    expect(reconcileCore).not.toHaveBeenCalled()
    expect(recordSuspended).not.toHaveBeenCalled()
  })

  it('rejects stale suspend follow-on work when the Host spec changes during Secret validation', async () => {
    const { reconciler, customApi } = createReconciler()
    const host = {
      ...makeStatelessHost({
        status: { lifecycle: { state: 'draining', wakeHandledGeneration: 0 } },
      }),
      uid: 'heartbeat-host-uid',
      generation: 4,
    }
    let current = host
    reconciler.setResolveCurrentHost(name => (name === host.name ? current : undefined))
    reconciler.setHostWatchAuthority(() => ({ known: true, generation: 34 }))
    customApi.getNamespacedCustomObject
      .mockResolvedValueOnce({
        ...freshHostRead({ state: 'draining', wakeHandledGeneration: 0 }),
        metadata: {
          name: host.name,
          namespace: host.namespace,
          uid: host.uid,
          generation: 4,
          resourceVersion: 'rv-suspend-write',
        },
      })
      .mockResolvedValueOnce({
        ...freshHostRead({
          state: 'suspended',
          wakeHandledGeneration: 0,
          reason: 'idle',
        }),
        metadata: {
          name: host.name,
          namespace: host.namespace,
          uid: host.uid,
          generation: 4,
          resourceVersion: 'rv-suspend-follow-on',
        },
      })
    const secretReadStarted = deferred()
    const releaseSecretRead = deferred()
    vi.spyOn(reconciler as any, 'validateHostSecret').mockImplementation(async () => {
      secretReadStarted.resolve(undefined)
      await releaseSecretRead.promise
      return { ok: true }
    })
    const staleMutation = vi
      .spyOn(reconciler as any, 'ensureHostServiceAccount')
      .mockRejectedValue(new Error('stale follow-on mutation was admitted'))

    const pending = reconciler.suspendHostFromHeartbeat(host, 'idle', 0)
    await secretReadStarted.promise
    current = {
      ...host,
      generation: 5,
      spec: {
        ...host.spec,
        contextRef: 'replacement-context',
        lifecycle: { stateless: false },
      },
    }
    releaseSecretRead.resolve(undefined)

    await expect(pending).rejects.toThrow(/Host spec generation/)
    expect(staleMutation).not.toHaveBeenCalled()
  })

  it('does not record a suspended scale-down when follow-on reconcile handles a wake', async () => {
    const { reconciler, customApi } = createReconciler()
    const host = {
      ...makeStatelessHost({
        status: { lifecycle: { state: 'draining', wakeHandledGeneration: 0 } },
      }),
      uid: 'heartbeat-host-uid',
      generation: 4,
    }
    reconciler.setResolveCurrentHost(name => (name === host.name ? host : undefined))
    reconciler.setHostWatchAuthority(() => ({ known: true, generation: 32 }))
    customApi.getNamespacedCustomObject.mockResolvedValue({
      ...freshHostRead({ state: 'draining', wakeHandledGeneration: 0 }),
      metadata: {
        name: host.name,
        namespace: host.namespace,
        uid: host.uid,
        generation: 4,
        resourceVersion: 'rv-wake-follow-on',
      },
    })
    vi.spyOn(reconciler as any, 'reconcileCore').mockImplementation(async (...args: unknown[]) => {
      const reconcileHost = args[0] as HostCRD
      reconcileHost.annotations = { 'clerum.io/wake-requested': '1' }
      reconcileHost.status = {
        lifecycle: { state: 'active', wakeHandledGeneration: 1 },
      }
    })
    const recordSuspended = vi.spyOn((reconciler as any).lifecycle, 'recordSuspendedApplied')

    await expect(reconciler.suspendHostFromHeartbeat(host, 'idle', 0)).resolves.toBe('suspended')

    expect(recordSuspended).not.toHaveBeenCalled()
  })
})

describe('HostReconciler.suspendHostFromHeartbeat', () => {
  it('does not mutate the admitted watch-cache snapshot while applying a suspension', async () => {
    const { reconciler, customApi } = createReconciler()
    const host = makeStatelessHost({
      status: { lifecycle: { state: 'draining', wakeHandledGeneration: 2 } },
    })
    host.uid = 'stateless-host-uid'
    host.generation = 1
    host.resourceVersion = 'cache-rv'
    const snapshot = structuredClone(host)
    Object.freeze(host)
    reconciler.setResolveCurrentHost(name => (name === host.name ? host : undefined))
    reconciler.setHostMutationAuthority(() => ({
      known: true,
      hostRevision: 1,
      contextRevision: 1,
    }))
    const draining = freshHostRead({ state: 'draining', wakeHandledGeneration: 2 })
    Object.assign(draining.metadata, {
      uid: host.uid,
      generation: host.generation,
      resourceVersion: 'draining-rv',
    })
    const suspended = freshHostRead({
      state: 'suspended',
      wakeHandledGeneration: 2,
      reason: 'idle',
    })
    Object.assign(suspended.metadata, {
      uid: host.uid,
      generation: host.generation,
      resourceVersion: 'suspended-rv',
    })
    customApi.getNamespacedCustomObject.mockResolvedValueOnce(draining).mockResolvedValue(suspended)

    await expect(reconciler.suspendHostFromHeartbeat(host, 'idle', 2)).resolves.toBe('suspended')
    expect(host).toEqual(snapshot)
  })

  it("writes state='suspended' + reason 'idle' durably, then reconciles to replicas=0", async () => {
    const { reconciler, appsApi, customApi } = createReconciler()
    const host = makeStatelessHost({
      status: { lifecycle: { state: 'active', wakeHandledGeneration: 2 } },
    })
    // Server timeline: draining when the suspend's fresh GET runs (the
    // poller's durable draining write already landed), suspended for every
    // read after the suspend PATCH — the FIX 1 replicas guard re-reads
    // fresh before deriving replicas.
    customApi.getNamespacedCustomObject
      .mockResolvedValueOnce(freshHostRead({ state: 'draining', wakeHandledGeneration: 2 }))
      .mockResolvedValue(
        freshHostRead({ state: 'suspended', wakeHandledGeneration: 2, reason: 'idle' })
      )

    await reconciler.suspendHostFromHeartbeat(host, 'idle', 2)

    const writes = lifecycleStatusWrites(customApi)
    expect(writes.length).toBeGreaterThanOrEqual(1)
    expect(writes[0].lifecycle).toEqual({
      state: 'suspended',
      wakeHandledGeneration: 2,
      reason: 'idle',
    })
    expect(hostDeploymentBody(appsApi, 'stateless-host').spec?.replicas).toBe(0)
  })

  it('fails loud when the durable status write fails (no scale-down happens)', async () => {
    const { reconciler, appsApi, customApi } = createReconciler()
    customApi.patchNamespacedCustomObjectStatus.mockRejectedValue(new Error('api down'))
    const host = makeStatelessHost({
      status: { lifecycle: { state: 'active', wakeHandledGeneration: 0 } },
    })
    customApi.getNamespacedCustomObject.mockResolvedValue(
      freshHostRead({ state: 'draining', wakeHandledGeneration: 0 })
    )

    await expect(reconciler.suspendHostFromHeartbeat(host, 'idle', 0)).rejects.toThrow('api down')
    expect(appsApi.createNamespacedDeployment).not.toHaveBeenCalled()
    expect(appsApi.replaceNamespacedDeployment).not.toHaveBeenCalled()
  })

  it('a pending wake beats the suspension: the reconcile flips back to active + replicas=1', async () => {
    const { reconciler, appsApi, customApi } = createReconciler()
    // clerum.io/wake-requested generation 7 > wakeHandledGeneration 2: the
    // wake fast-path inside reconcile() must win over the suspend request.
    const host = makeStatelessHost({
      annotations: { 'clerum.io/wake-requested': '7' },
      status: { lifecycle: { state: 'active', wakeHandledGeneration: 2 } },
    })
    // Read 1 (the suspend commit's fresh guard): draining with no pending
    // wake yet — the suspend commits. Later reads (the fast-path's AP-1
    // commit-point read inside the follow-up reconcile): the durable wake
    // annotation is now visible on the suspended CR, so the fast-path
    // re-decides requested 7 > handled 2 from FRESH and wins.
    customApi.getNamespacedCustomObject
      .mockResolvedValueOnce(freshHostRead({ state: 'draining', wakeHandledGeneration: 2 }))
      .mockResolvedValue({
        ...freshHostRead({ state: 'suspended', wakeHandledGeneration: 2 }),
        metadata: {
          name: 'stateless-host',
          namespace: 'mcp-host',
          annotations: { 'clerum.io/wake-requested': '7' },
        },
      })

    await reconciler.suspendHostFromHeartbeat(host, 'idle', 2)

    expect(host.status?.lifecycle?.state).toBe('active')
    expect(host.status?.lifecycle?.wakeHandledGeneration).toBe(7)
    expect(hostDeploymentBody(appsApi, 'stateless-host').spec?.replicas).toBe(1)
  })

  it('no-ops when a wake was handled after the suspend decision: fresh draining with a generation past the entry epoch', async () => {
    const logSpy = vi.spyOn(console, 'log').mockImplementation(() => {})
    try {
      const { reconciler, appsApi, customApi } = createReconciler()
      // The tracker decided this suspend off a snapshot with handled=2 …
      const host = makeStatelessHost({
        status: { lifecycle: { state: 'draining', wakeHandledGeneration: 2 } },
      })
      // … but by the commit point a wake was handled (the fast-path bumped
      // the generation to 3) and an aged drain:true verdict re-persisted
      // `draining`. requested==handled → nothing would ever revive a
      // suspended Host, so the commit MUST no-op.
      customApi.getNamespacedCustomObject.mockResolvedValue(
        freshHostRead({ state: 'draining', wakeHandledGeneration: 3 })
      )

      await reconciler.suspendHostFromHeartbeat(host, 'idle', 2)

      expect(customApi.patchNamespacedCustomObjectStatus).not.toHaveBeenCalled()
      expect(appsApi.createNamespacedDeployment).not.toHaveBeenCalled()
      expect(appsApi.replaceNamespacedDeployment).not.toHaveBeenCalled()
      expect(host.status?.lifecycle?.state).toBe('draining')
      const staleLines = logSpy.mock.calls
        .map(args => JSON.parse(String(args[0])).msg as string)
        .filter(line => line.includes('phase=drained_report_stale'))
      expect(staleLines).toEqual([
        expect.stringMatching(
          /^\[StatelessSuspend\] host=stateless-host phase=drained_report_stale reason=wake_handled_since entryGeneration=2 freshGeneration=3 ts=\d+$/
        ),
      ])
    } finally {
      logSpy.mockRestore()
    }
  })

  it('no-ops when a wake is pending at the commit point (fresh requested > handled)', async () => {
    const logSpy = vi.spyOn(console, 'log').mockImplementation(() => {})
    try {
      const { reconciler, appsApi, customApi } = createReconciler()
      const host = makeStatelessHost({
        status: { lifecycle: { state: 'draining', wakeHandledGeneration: 2 } },
      })
      // Fresh carries a not-yet-handled wake annotation: requested 3 > handled 2.
      customApi.getNamespacedCustomObject.mockResolvedValue({
        metadata: {
          name: 'stateless-host',
          namespace: 'mcp-host',
          annotations: { 'clerum.io/wake-requested': '3' },
        },
        spec: {
          host: 'stateless-host',
          contextRef: 'context-a',
          secretRef: 'host-secret',
          lifecycle: { stateless: true },
        },
        status: { lifecycle: { state: 'draining', wakeHandledGeneration: 2 } },
      })

      await reconciler.suspendHostFromHeartbeat(host, 'idle', 2)

      expect(customApi.patchNamespacedCustomObjectStatus).not.toHaveBeenCalled()
      expect(appsApi.createNamespacedDeployment).not.toHaveBeenCalled()
      expect(appsApi.replaceNamespacedDeployment).not.toHaveBeenCalled()
      const staleLines = logSpy.mock.calls
        .map(args => JSON.parse(String(args[0])).msg as string)
        .filter(line => line.includes('phase=drained_report_stale'))
      expect(staleLines).toEqual([
        expect.stringMatching(
          /^\[StatelessSuspend\] host=stateless-host phase=drained_report_stale reason=wake_pending requestedGeneration=3 handledGeneration=2 ts=\d+$/
        ),
      ])
    } finally {
      logSpy.mockRestore()
    }
  })
})

describe('HostReconciler.publishSuspendBlockedReason', () => {
  it('writes the reason once and is change-only across repeats', async () => {
    const { reconciler, customApi } = createReconciler()
    const host = makeStatelessHost({
      status: { lifecycle: { state: 'active', wakeHandledGeneration: 0 } },
    })
    // AP-1 (9th costume): the writer decides against a FRESH read, not the
    // caller's snapshot. Fresh is active with no reason yet → first write lands.
    customApi.getNamespacedCustomObject.mockResolvedValueOnce(
      freshHostRead({ state: 'active', wakeHandledGeneration: 0 })
    )
    await reconciler.publishSuspendBlockedReason(host, 'SuspendBlocked: activeTask')
    expect(customApi.patchNamespacedCustomObjectStatus).toHaveBeenCalledTimes(1)
    expect(host.status?.lifecycle?.reason).toBe('SuspendBlocked: activeTask')

    // Same reason again → the fresh read now carries it → change-only no-op.
    customApi.getNamespacedCustomObject.mockResolvedValueOnce(
      freshHostRead({
        state: 'active',
        wakeHandledGeneration: 0,
        reason: 'SuspendBlocked: activeTask',
      })
    )
    await reconciler.publishSuspendBlockedReason(host, 'SuspendBlocked: activeTask')
    expect(customApi.patchNamespacedCustomObjectStatus).toHaveBeenCalledTimes(1)

    // Different reason → one more write.
    customApi.getNamespacedCustomObject.mockResolvedValueOnce(
      freshHostRead({
        state: 'active',
        wakeHandledGeneration: 0,
        reason: 'SuspendBlocked: activeTask',
      })
    )
    await reconciler.publishSuspendBlockedReason(host, 'SuspendBlocked: pendingResults')
    expect(customApi.patchNamespacedCustomObjectStatus).toHaveBeenCalledTimes(2)
    expect(host.status?.lifecycle?.reason).toBe('SuspendBlocked: pendingResults')
  })

  it('the published reason survives a subsequent full reconcile (accepted path preserves it)', async () => {
    const { reconciler, customApi } = createReconciler()
    const host = makeStatelessHost({
      status: { lifecycle: { state: 'active', wakeHandledGeneration: 0 } },
    })
    // AP-1 (9th costume): the publish itself decides against a FRESH read — an
    // active Host with no reason yet → the reason write lands.
    customApi.getNamespacedCustomObject.mockResolvedValueOnce(
      freshHostRead({ state: 'active', wakeHandledGeneration: 0 })
    )
    await reconciler.publishSuspendBlockedReason(host, 'SuspendBlocked: activeTask')
    // Server-side truth for the fresh-read the accepted-path status writer
    // performs (AP-1): the durable publishSuspendBlockedReason write already
    // landed, so the fresh Host is active + carries the heartbeat-managed
    // reason. The accepted path must re-source that reason from fresh.
    customApi.getNamespacedCustomObject.mockResolvedValue(
      freshHostRead({
        state: 'active',
        wakeHandledGeneration: 0,
        reason: 'SuspendBlocked: activeTask',
      })
    )
    await reconciler.reconcile(host)

    const writes = lifecycleStatusWrites(customApi)
    const last = writes[writes.length - 1]
    expect(last.lifecycle).toEqual({
      state: 'active',
      wakeHandledGeneration: 0,
      reason: 'SuspendBlocked: activeTask',
    })
  })

  it('a rejection reason from assessLifecycle is NOT preserved as heartbeat-managed', async () => {
    const { reconciler, customApi } = createReconciler()
    // A previously-written rejection message in status.lifecycle.reason must
    // be cleared by the accepted path (only tracker-owned reasons survive).
    const host = makeStatelessHost({
      status: {
        lifecycle: {
          state: 'active',
          wakeHandledGeneration: 0,
          reason:
            '1 CommunicationChannel(s) reference this Host; disassociate them to enable the requested stateless lifecycle',
        },
      },
    })

    // Server-side truth for the fresh-read the accepted-path status writer
    // performs (AP-1): the fresh Host is active and still carries the stale
    // rejection reason. The accepted path re-sources state from fresh but
    // drops the reason because a rejection message is NOT heartbeat-managed.
    customApi.getNamespacedCustomObject.mockResolvedValue(
      freshHostRead({
        state: 'active',
        wakeHandledGeneration: 0,
        reason:
          '1 CommunicationChannel(s) reference this Host; disassociate them to enable the requested stateless lifecycle',
      })
    )

    await reconciler.reconcile(host)

    const writes = lifecycleStatusWrites(customApi)
    expect(writes).toHaveLength(1)
    expect(writes[0].lifecycle).toEqual({ state: 'active', wakeHandledGeneration: 0 })
  })

  it('AP-1 (9th costume): a cached-draining host whose fresh read is suspended does NOT write (no resurrection)', async () => {
    const { reconciler, customApi } = createReconciler()
    // The tracker's !idleEligible branch calls cancelDrainOnEvidence then
    // publishSuspendBlockedReason against the SAME host ref. cancelDrainOnEvidence
    // skipped (fresh already moved past draining) leaving host.status stale
    // `draining`. A suspend landed first on the chain, so the fresh read is
    // `suspended`. The reason writer must NOT re-source the stale draining and
    // write state back — it must no-op so it never resurrects the suspended Host.
    const host = makeStatelessHost({
      status: { lifecycle: { state: 'draining', wakeHandledGeneration: 4 } },
    })
    customApi.getNamespacedCustomObject.mockResolvedValue(
      freshHostRead({ state: 'suspended', wakeHandledGeneration: 4, reason: 'idle' })
    )

    await reconciler.publishSuspendBlockedReason(host, 'SuspendBlocked: activeTask')

    // No write at all: a suspend-blocked reason is meaningless once suspended,
    // and writing it would resurrect the Host (replicas derive from state).
    expect(customApi.patchNamespacedCustomObjectStatus).not.toHaveBeenCalled()
    // The in-memory snapshot is untouched — the suspended Host is preserved.
    expect(host.status?.lifecycle?.state).toBe('draining')
  })

  it('AP-1 (9th costume): writes the reason on the FRESH active state via a targeted /status/lifecycle op', async () => {
    const { reconciler, customApi } = createReconciler()
    // Cached active; fresh read is active carrying a newer wakeHandledGeneration
    // than the snapshot (a wake landed since). The writer must stamp the reason
    // on the FRESH state (state:'active', gen from fresh), never the snapshot.
    const host = makeStatelessHost({
      status: { lifecycle: { state: 'active', wakeHandledGeneration: 1 } },
    })
    customApi.getNamespacedCustomObject.mockResolvedValue(
      freshHostRead({ state: 'active', wakeHandledGeneration: 5 })
    )

    await reconciler.publishSuspendBlockedReason(host, 'SuspendBlocked: activeTask')

    const [patch] = customApi.patchNamespacedCustomObjectStatus.mock.calls
    const body = (patch[0] as { body: Array<{ op: string; path: string; value: unknown }> }).body
    // Targeted op — never a whole-/status spread.
    const lifecycleOp = body.find(op => op.path === '/status/lifecycle')
    if (!lifecycleOp) {
      throw new Error('expected a targeted /status/lifecycle op')
    }
    expect(body.some(op => op.path === '/status')).toBe(false)
    expect(lifecycleOp.value).toEqual({
      state: 'active',
      wakeHandledGeneration: 5,
      reason: 'SuspendBlocked: activeTask',
    })
    expect(host.status?.lifecycle).toEqual({
      state: 'active',
      wakeHandledGeneration: 5,
      reason: 'SuspendBlocked: activeTask',
    })
  })

  it('AP-1 (9th costume): a 409 re-reads fresh and retries under the D3 precondition', async () => {
    const { reconciler, customApi } = createReconciler()
    const host = makeStatelessHost({
      status: { lifecycle: { state: 'active', wakeHandledGeneration: 0 } },
    })
    // First fresh read (rv-1) is active → build proceeds; the patch 409s.
    // Second fresh read (rv-2) is STILL active → build re-runs and succeeds.
    customApi.getNamespacedCustomObject
      .mockResolvedValueOnce(
        freshHostReadWithRv('rv-1', { state: 'active', wakeHandledGeneration: 0 })
      )
      .mockResolvedValueOnce(
        freshHostReadWithRv('rv-2', { state: 'active', wakeHandledGeneration: 0 })
      )
    customApi.patchNamespacedCustomObjectStatus
      .mockRejectedValueOnce(conflict409())
      .mockResolvedValueOnce({})

    await reconciler.publishSuspendBlockedReason(host, 'SuspendBlocked: activeTask')

    expect(customApi.getNamespacedCustomObject.mock.calls.length).toBeGreaterThanOrEqual(2)
    expect(customApi.patchNamespacedCustomObjectStatus).toHaveBeenCalledTimes(2)
    // The retry carried the re-read resourceVersion, proving it re-read fresh.
    const secondBody = (
      customApi.patchNamespacedCustomObjectStatus.mock.calls[1][0] as {
        body: Array<{ op: string; path: string; value: unknown }>
      }
    ).body
    expect(secondBody[0]).toEqual({ op: 'add', path: '/metadata/resourceVersion', value: 'rv-2' })
    expect(host.status?.lifecycle?.reason).toBe('SuspendBlocked: activeTask')
  })
})

describe('HostReconciler.findPodCreationTimestamp', () => {
  it('resolves the creationTimestamp of the pod matching the UID', async () => {
    const { reconciler, coreApi } = createReconciler()
    coreApi.listNamespacedPod.mockResolvedValue({
      items: [
        { metadata: { uid: 'pod-b', creationTimestamp: '2026-07-02T00:00:00Z' } },
        { metadata: { uid: 'pod-a', creationTimestamp: '2026-07-01T00:00:00Z' } },
      ],
    })
    const host = makeStatelessHost()

    const created = await reconciler.findPodCreationTimestamp(host, 'pod-a')
    expect(created).toEqual(new Date('2026-07-01T00:00:00Z'))
    expect(coreApi.listNamespacedPod).toHaveBeenCalledWith({
      namespace: 'mcp-host',
      labelSelector: 'app=stateless-host',
    })
  })

  it('returns null when no pod carries the UID', async () => {
    const { reconciler, coreApi } = createReconciler()
    coreApi.listNamespacedPod.mockResolvedValue({ items: [{ metadata: { uid: 'pod-x' } }] })
    expect(await reconciler.findPodCreationTimestamp(makeStatelessHost(), 'pod-a')).toBeNull()
  })
})

describe('HostReconciler stateless env — CLERUM_POD_UID downward API', () => {
  it('injects CLERUM_POD_UID via fieldRef metadata.uid on stateless hosts', () => {
    const { reconciler } = createReconciler()
    const dep = reconciler.buildDeployment(makeStatelessHost())
    const env = dep.spec?.template?.spec?.containers?.[0]?.env ?? []
    expect(env).toContainEqual({
      name: 'CLERUM_POD_UID',
      valueFrom: { fieldRef: { fieldPath: 'metadata.uid' } },
    })
  })

  it('does not inject CLERUM_POD_UID on non-stateless hosts', () => {
    const { reconciler } = createReconciler()
    const dep = reconciler.buildDeployment(
      makeStatelessHost({ spec: { lifecycle: { stateless: false } } })
    )
    const names = (dep.spec?.template?.spec?.containers?.[0]?.env ?? []).map(e => e.name)
    expect(names).not.toContain('CLERUM_POD_UID')
  })
})

describe('HostReconciler.markHostDrainingFromHeartbeat', () => {
  it("flips an active Host to state='draining' durably (wake generation preserved, stale reason dropped)", async () => {
    const { reconciler, customApi } = createReconciler()
    const host = makeStatelessHost({
      status: {
        lifecycle: {
          state: 'active',
          wakeHandledGeneration: 3,
          reason: 'suspend-blocked: activeTask',
        },
      },
    })
    customApi.getNamespacedCustomObject.mockResolvedValue(
      freshHostRead({
        state: 'active',
        wakeHandledGeneration: 3,
        reason: 'suspend-blocked: activeTask',
      })
    )

    await reconciler.markHostDrainingFromHeartbeat(host, 3)

    const writes = lifecycleStatusWrites(customApi)
    expect(writes).toHaveLength(1)
    expect(writes[0].lifecycle).toEqual({ state: 'draining', wakeHandledGeneration: 3 })
    // In-memory reflection so a same-pass drained report sees the flip.
    expect(host.status?.lifecycle?.state).toBe('draining')
  })

  it('is idempotent: an already-draining Host writes nothing', async () => {
    const { reconciler, customApi } = createReconciler()
    const host = makeStatelessHost({
      status: { lifecycle: { state: 'draining', wakeHandledGeneration: 1 } },
    })
    customApi.getNamespacedCustomObject.mockResolvedValue(
      freshHostRead({ state: 'draining', wakeHandledGeneration: 1 })
    )

    await reconciler.markHostDrainingFromHeartbeat(host, 1)
    await reconciler.markHostDrainingFromHeartbeat(host, 1)

    expect(customApi.patchNamespacedCustomObjectStatus).not.toHaveBeenCalled()
  })

  it('never clobbers a suspended Host back to draining', async () => {
    const { reconciler, customApi } = createReconciler()
    const host = makeStatelessHost({
      status: { lifecycle: { state: 'suspended', wakeHandledGeneration: 2, reason: 'idle' } },
    })
    customApi.getNamespacedCustomObject.mockResolvedValue(
      freshHostRead({ state: 'suspended', wakeHandledGeneration: 2, reason: 'idle' })
    )

    await reconciler.markHostDrainingFromHeartbeat(host, 2)

    expect(customApi.patchNamespacedCustomObjectStatus).not.toHaveBeenCalled()
    expect(host.status?.lifecycle?.state).toBe('suspended')
  })

  it('writes exactly once across repeated drain verdicts for the same episode', async () => {
    const { reconciler, customApi } = createReconciler()
    const host = makeStatelessHost({
      status: { lifecycle: { state: 'active', wakeHandledGeneration: 0 } },
    })
    // Server timeline: active for the first verdict, draining once the
    // durable write landed — the fresh-read guard makes the repeats no-ops.
    customApi.getNamespacedCustomObject
      .mockResolvedValueOnce(freshHostRead({ state: 'active', wakeHandledGeneration: 0 }))
      .mockResolvedValue(freshHostRead({ state: 'draining', wakeHandledGeneration: 0 }))

    await reconciler.markHostDrainingFromHeartbeat(host, 0)
    await reconciler.markHostDrainingFromHeartbeat(host, 0)
    await reconciler.markHostDrainingFromHeartbeat(host, 0)

    expect(customApi.patchNamespacedCustomObjectStatus).toHaveBeenCalledTimes(1)
  })

  it('fails loud when the durable status write fails (no in-memory flip)', async () => {
    const { reconciler, customApi } = createReconciler()
    customApi.patchNamespacedCustomObjectStatus.mockRejectedValue(new Error('api conflict'))
    const host = makeStatelessHost({
      status: { lifecycle: { state: 'active', wakeHandledGeneration: 0 } },
    })
    customApi.getNamespacedCustomObject.mockResolvedValue(
      freshHostRead({ state: 'active', wakeHandledGeneration: 0 })
    )

    await expect(reconciler.markHostDrainingFromHeartbeat(host, 0)).rejects.toThrow('api conflict')
    expect(host.status?.lifecycle?.state).toBe('active')
  })
})

describe('HostReconciler.markHostActiveFromHeartbeat', () => {
  it("reverts a draining Host to state='active' preserving the wake generation", async () => {
    const { reconciler, customApi } = createReconciler()
    const host = makeStatelessHost({
      status: { lifecycle: { state: 'draining', wakeHandledGeneration: 4 } },
    })
    customApi.getNamespacedCustomObject.mockResolvedValue(
      freshHostRead({ state: 'draining', wakeHandledGeneration: 4 })
    )

    await reconciler.markHostActiveFromHeartbeat(host)

    const writes = lifecycleStatusWrites(customApi)
    expect(writes).toHaveLength(1)
    expect(writes[0].lifecycle).toEqual({ state: 'active', wakeHandledGeneration: 4 })
    // In-memory reflection so a same-pass decision sees the revert.
    expect(host.status?.lifecycle?.state).toBe('active')
  })

  it('is a no-op on an already-active Host (no status write)', async () => {
    const { reconciler, customApi } = createReconciler()
    const host = makeStatelessHost({
      status: { lifecycle: { state: 'active', wakeHandledGeneration: 1 } },
    })
    customApi.getNamespacedCustomObject.mockResolvedValue(
      freshHostRead({ state: 'active', wakeHandledGeneration: 1 })
    )

    await reconciler.markHostActiveFromHeartbeat(host)

    expect(customApi.patchNamespacedCustomObjectStatus).not.toHaveBeenCalled()
  })

  it('never revives a suspended Host (that is the wake fast-path job)', async () => {
    const { reconciler, customApi } = createReconciler()
    const host = makeStatelessHost({
      status: { lifecycle: { state: 'suspended', wakeHandledGeneration: 2, reason: 'idle' } },
    })
    customApi.getNamespacedCustomObject.mockResolvedValue(
      freshHostRead({ state: 'suspended', wakeHandledGeneration: 2, reason: 'idle' })
    )

    await reconciler.markHostActiveFromHeartbeat(host)

    expect(customApi.patchNamespacedCustomObjectStatus).not.toHaveBeenCalled()
    expect(host.status?.lifecycle?.state).toBe('suspended')
  })

  it('fails loud when the durable status write fails (no in-memory flip)', async () => {
    const { reconciler, customApi } = createReconciler()
    customApi.patchNamespacedCustomObjectStatus.mockRejectedValue(new Error('api conflict'))
    const host = makeStatelessHost({
      status: { lifecycle: { state: 'draining', wakeHandledGeneration: 0 } },
    })
    customApi.getNamespacedCustomObject.mockResolvedValue(
      freshHostRead({ state: 'draining', wakeHandledGeneration: 0 })
    )

    await expect(reconciler.markHostActiveFromHeartbeat(host)).rejects.toThrow('api conflict')
    expect(host.status?.lifecycle?.state).toBe('draining')
  })

  it('reverts a draining Host while channel authority loss blocks suspension', async () => {
    // A suspension-blocked Host can still be draining from a decision made
    // before the loss. Cancel-drain must stay available, otherwise the emitter
    // remains fenced behind {drain:true} for the whole outage.
    const { reconciler, customApi } = createReconciler({
      isCommunicationChannelCacheSynced: () => false,
    })
    const host = makeStatelessHost({
      status: { lifecycle: { state: 'draining', wakeHandledGeneration: 4 } },
    })
    customApi.getNamespacedCustomObject.mockResolvedValue(
      freshHostRead({ state: 'draining', wakeHandledGeneration: 4 })
    )
    expect(reconciler.getEffectiveLifecycle(host)).toEqual({
      stateless: true,
      state: 'draining',
      suspensionBlocked: true,
    })

    await reconciler.markHostActiveFromHeartbeat(host)

    expect(customApi.getNamespacedCustomObject).toHaveBeenCalled()
    const writes = lifecycleStatusWrites(customApi)
    expect(writes).toHaveLength(1)
    expect(writes[0].lifecycle).toEqual({ state: 'active', wakeHandledGeneration: 4 })
    expect(host.status?.lifecycle?.state).toBe('active')
  })

  it.each([
    { channels: 1, expectedWrites: 0, finalState: 'draining' },
    { channels: 0, expectedWrites: 1, finalState: 'active' },
  ] as const)(
    'reverts a same-revision draining Host only while its fresh read is stateless (channels=$channels)',
    async ({ channels, expectedWrites, finalState }) => {
      // Channel ingress makes a Host effectively not stateless, so the lifecycle
      // it would revert is not the one the stateless executor owns.
      const { reconciler, customApi } = createReconciler()
      reconciler.setCountCommunicationChannels(() => channels)
      const host = {
        ...makeStatelessHost({
          status: { lifecycle: { state: 'draining', wakeHandledGeneration: 3 } },
        }),
        generation: 5,
      }
      const fresh = freshHostRead({ state: 'draining', wakeHandledGeneration: 3 })
      customApi.getNamespacedCustomObject.mockResolvedValue({
        ...fresh,
        metadata: { ...fresh.metadata, generation: 5, resourceVersion: 'rv-draining' },
      })

      await reconciler.markHostActiveFromHeartbeat(host)

      // Liveness witness: the decision was taken on exactly one fresh read.
      expect(customApi.getNamespacedCustomObject).toHaveBeenCalledOnce()
      expect(customApi.patchNamespacedCustomObjectStatus).toHaveBeenCalledTimes(expectedWrites)
      expect(host.status?.lifecycle?.state).toBe(finalState)
    }
  )
})
describe('HostReconciler heartbeat cores — fresh-read guard (cross-instance staleness)', () => {
  // The Host watch callback builds a brand-new HostCRD per ADDED/MODIFIED
  // event, so two independently-triggered callers (drain-grace expiry vs the
  // next polled heartbeat) can hold DIFFERENT object instances for the same
  // host. The in-memory `host.status` reflection only helps same-instance
  // callers — each core must guard against a FRESH GET of the CR.

  it('closing-review scenario: a stale draining snapshot on a second instance cannot resurrect a just-suspended Host', async () => {
    const { reconciler, customApi } = createReconciler()
    // Two DISTINCT HostCRD instances for the same hostRef, as two watch
    // events produce (the watch callback REPLACES the map entry each time).
    const hostA = makeStatelessHost({
      status: { lifecycle: { state: 'draining', wakeHandledGeneration: 2 } },
    })
    const hostB = makeStatelessHost({
      status: { lifecycle: { state: 'draining', wakeHandledGeneration: 2 } },
    })
    // Server timeline: draining when the suspend's fresh GET runs, suspended
    // for every read after the suspend PATCH landed.
    customApi.getNamespacedCustomObject
      .mockResolvedValueOnce(freshHostRead({ state: 'draining', wakeHandledGeneration: 2 }))
      .mockResolvedValue(
        freshHostRead({ state: 'suspended', wakeHandledGeneration: 2, reason: 'idle' })
      )

    await reconciler.suspendHostFromHeartbeat(hostA, 'idle', 2)
    const writesAfterSuspend = lifecycleStatusWrites(customApi)
    expect(writesAfterSuspend[0].lifecycle?.state).toBe('suspended')

    // hostB's stale snapshot still says draining — the revert must observe
    // the FRESH suspended state and write NOTHING.
    await reconciler.markHostActiveFromHeartbeat(hostB)

    const writesAfterRevert = lifecycleStatusWrites(customApi)
    expect(writesAfterRevert).toHaveLength(writesAfterSuspend.length)
    // Final state stays suspended: no write ever asserted 'active'.
    expect(writesAfterRevert.every(w => w.lifecycle?.state !== 'active')).toBe(true)
  })

  it('cancel-drain takes wakeHandledGeneration from the FRESH lifecycle, not the stale snapshot', async () => {
    const { reconciler, customApi } = createReconciler()
    const staleHost = makeStatelessHost({
      status: { lifecycle: { state: 'draining', wakeHandledGeneration: 3 } },
    })
    customApi.getNamespacedCustomObject.mockResolvedValue(
      freshHostRead({ state: 'draining', wakeHandledGeneration: 7 })
    )

    await reconciler.markHostActiveFromHeartbeat(staleHost)

    const writes = lifecycleStatusWrites(customApi)
    expect(writes).toHaveLength(1)
    expect(writes[0].lifecycle).toEqual({ state: 'active', wakeHandledGeneration: 7 })
  })

  it('suspend with a stale draining snapshot over a fresh active CR writes nothing and never scales down', async () => {
    const { reconciler, appsApi, customApi } = createReconciler()
    const staleHost = makeStatelessHost({
      status: { lifecycle: { state: 'draining', wakeHandledGeneration: 1 } },
    })
    // A cancel-drain (or wake) landed in between: the CR is active again.
    customApi.getNamespacedCustomObject.mockResolvedValue(
      freshHostRead({ state: 'active', wakeHandledGeneration: 1 })
    )

    await reconciler.suspendHostFromHeartbeat(staleHost, 'idle', 1)

    expect(customApi.patchNamespacedCustomObjectStatus).not.toHaveBeenCalled()
    expect(appsApi.createNamespacedDeployment).not.toHaveBeenCalled()
    expect(appsApi.replaceNamespacedDeployment).not.toHaveBeenCalled()
  })

  it('drain write with a stale active snapshot over a fresh suspended CR writes nothing', async () => {
    const { reconciler, customApi } = createReconciler()
    const staleHost = makeStatelessHost({
      status: { lifecycle: { state: 'active', wakeHandledGeneration: 0 } },
    })
    customApi.getNamespacedCustomObject.mockResolvedValue(
      freshHostRead({ state: 'suspended', wakeHandledGeneration: 2, reason: 'idle' })
    )

    await reconciler.markHostDrainingFromHeartbeat(staleHost, 0)

    expect(customApi.patchNamespacedCustomObjectStatus).not.toHaveBeenCalled()
    expect(staleHost.status?.lifecycle?.state).toBe('active')
  })

  it('suspendHostFromHeartbeat propagates a fresh GET failure (no patch)', async () => {
    const { reconciler, customApi } = createReconciler()
    customApi.getNamespacedCustomObject.mockRejectedValue(new Error('fresh get failed'))
    const host = makeStatelessHost({
      status: { lifecycle: { state: 'draining', wakeHandledGeneration: 0 } },
    })

    await expect(reconciler.suspendHostFromHeartbeat(host, 'idle', 0)).rejects.toThrow(
      'fresh get failed'
    )
    expect(customApi.patchNamespacedCustomObjectStatus).not.toHaveBeenCalled()
  })

  it('markHostDrainingFromHeartbeat propagates a fresh GET failure (no patch)', async () => {
    const { reconciler, customApi } = createReconciler()
    customApi.getNamespacedCustomObject.mockRejectedValue(new Error('fresh get failed'))
    const host = makeStatelessHost({
      status: { lifecycle: { state: 'active', wakeHandledGeneration: 0 } },
    })

    await expect(reconciler.markHostDrainingFromHeartbeat(host, 0)).rejects.toThrow(
      'fresh get failed'
    )
    expect(customApi.patchNamespacedCustomObjectStatus).not.toHaveBeenCalled()
  })

  it('markHostActiveFromHeartbeat propagates a fresh GET failure (no patch)', async () => {
    const { reconciler, customApi } = createReconciler()
    customApi.getNamespacedCustomObject.mockRejectedValue(new Error('fresh get failed'))
    const host = makeStatelessHost({
      status: { lifecycle: { state: 'draining', wakeHandledGeneration: 0 } },
    })

    await expect(reconciler.markHostActiveFromHeartbeat(host)).rejects.toThrow('fresh get failed')
    expect(customApi.patchNamespacedCustomObjectStatus).not.toHaveBeenCalled()
  })
})

/**
 * Raw CustomObjects GET response carrying a metadata.resourceVersion so the
 * D3 optimistic-concurrency precondition op is emitted (freshHostRead omits
 * it, exercising the no-precondition fallback).
 */
function freshHostReadWithRv(
  resourceVersion: string,
  lifecycle: {
    state: 'active' | 'draining' | 'suspended'
    wakeHandledGeneration: number
    reason?: string
  }
) {
  return {
    metadata: { name: 'stateless-host', namespace: 'mcp-host', resourceVersion },
    spec: {
      host: 'stateless-host',
      contextRef: 'context-a',
      secretRef: 'host-secret',
      lifecycle: { stateless: true },
    },
    status: { lifecycle },
  }
}

/** A K8s 409 Conflict error as getErrorCode reads it (.code). */
function conflict409(): Error {
  return Object.assign(new Error('the object has been modified'), { code: 409 })
}

describe('HostReconciler heartbeat status writes — D3 resourceVersion precondition + 409 retry', () => {
  it('carries metadata.resourceVersion as a precondition op when the fresh read supplies one', async () => {
    const { reconciler, customApi } = createReconciler()
    const host = makeStatelessHost({
      status: { lifecycle: { state: 'active', wakeHandledGeneration: 2 } },
    })
    customApi.getNamespacedCustomObject.mockResolvedValue(
      freshHostReadWithRv('rv-100', { state: 'draining', wakeHandledGeneration: 2 })
    )

    await reconciler.suspendHostFromHeartbeat(host, 'idle', 2)

    const [firstPatch] = customApi.patchNamespacedCustomObjectStatus.mock.calls
    const body = (firstPatch[0] as { body: Array<{ op: string; path: string; value: unknown }> })
      .body
    expect(body[0]).toEqual({ op: 'add', path: '/metadata/resourceVersion', value: 'rv-100' })
    expect(body[1].path).toBe('/status')
  })

  it('on a 409 (stale resourceVersion) re-reads fresh, re-checks the guard, and retries the patch', async () => {
    const { reconciler, appsApi, customApi } = createReconciler()
    const host = makeStatelessHost({
      status: { lifecycle: { state: 'active', wakeHandledGeneration: 2 } },
    })
    // First fresh read (rv-1) is draining → build proceeds. The patch 409s.
    // Second fresh read (rv-2) is STILL draining → build re-runs and succeeds.
    customApi.getNamespacedCustomObject
      .mockResolvedValueOnce(
        freshHostReadWithRv('rv-1', { state: 'draining', wakeHandledGeneration: 2 })
      )
      .mockResolvedValueOnce(
        freshHostReadWithRv('rv-2', { state: 'draining', wakeHandledGeneration: 2 })
      )
    customApi.patchNamespacedCustomObjectStatus
      .mockRejectedValueOnce(conflict409())
      .mockResolvedValueOnce({})

    await reconciler.suspendHostFromHeartbeat(host, 'idle', 2)

    // The suspend write re-read fresh after the 409 (>=2 GETs) and re-patched
    // (>=2 patch attempts); the post-suspend reconcile adds further calls, so
    // assert lower bounds, not exact counts.
    expect(customApi.getNamespacedCustomObject.mock.calls.length).toBeGreaterThanOrEqual(2)
    expect(customApi.patchNamespacedCustomObjectStatus.mock.calls.length).toBeGreaterThanOrEqual(2)
    // The retry carried the re-read resourceVersion, proving it re-read fresh.
    const secondBody = (
      customApi.patchNamespacedCustomObjectStatus.mock.calls[1][0] as {
        body: Array<{ op: string; path: string; value: unknown }>
      }
    ).body
    expect(secondBody[0]).toEqual({ op: 'add', path: '/metadata/resourceVersion', value: 'rv-2' })
    // The suspend committed → scale-down happened.
    expect(hostDeploymentBody(appsApi, 'stateless-host').spec?.replicas).toBe(0)
  })

  it('re-checks the guard on the fresh re-read: a wake that lands during the 409 window aborts the suspend', async () => {
    const { reconciler, appsApi, customApi } = createReconciler()
    const host = makeStatelessHost({
      status: { lifecycle: { state: 'active', wakeHandledGeneration: 2 } },
    })
    // First fresh read is draining (build proceeds), the patch 409s, and the
    // re-read now shows the drain was overturned to active (a cancel-drain or
    // wake landed) → the guard skips: NO suspend, NO further patch.
    customApi.getNamespacedCustomObject
      .mockResolvedValueOnce(
        freshHostReadWithRv('rv-1', { state: 'draining', wakeHandledGeneration: 2 })
      )
      .mockResolvedValueOnce(
        freshHostReadWithRv('rv-2', { state: 'active', wakeHandledGeneration: 2 })
      )
    customApi.patchNamespacedCustomObjectStatus.mockRejectedValueOnce(conflict409())

    await reconciler.suspendHostFromHeartbeat(host, 'idle', 2)

    // Exactly one patch attempt (the 409); the re-checked guard skipped the retry.
    expect(customApi.patchNamespacedCustomObjectStatus).toHaveBeenCalledTimes(1)
    expect(appsApi.createNamespacedDeployment).not.toHaveBeenCalled()
    expect(appsApi.replaceNamespacedDeployment).not.toHaveBeenCalled()
  })

  it('throws loudly when the 409 conflict never clears (retry exhaustion is not swallowed)', async () => {
    const { reconciler, appsApi, customApi } = createReconciler()
    const host = makeStatelessHost({
      status: { lifecycle: { state: 'active', wakeHandledGeneration: 2 } },
    })
    customApi.getNamespacedCustomObject.mockResolvedValue(
      freshHostReadWithRv('rv-x', { state: 'draining', wakeHandledGeneration: 2 })
    )
    customApi.patchNamespacedCustomObjectStatus.mockRejectedValue(conflict409())

    await expect(reconciler.suspendHostFromHeartbeat(host, 'idle', 2)).rejects.toThrow(
      /after 5 attempts \(persistent 409 conflict\)/
    )
    // 5 bounded attempts, then fail loud — no scale-down over unconfirmed state.
    // The suspend never commits, so no post-suspend reconcile adds writes.
    expect(customApi.patchNamespacedCustomObjectStatus).toHaveBeenCalledTimes(5)
    expect(appsApi.createNamespacedDeployment).not.toHaveBeenCalled()
    expect(appsApi.replaceNamespacedDeployment).not.toHaveBeenCalled()
  })
})

describe('HostReconciler.writeLifecycleStatusToCluster — D2 targeted patch does not clobber fresher fields', () => {
  it('writes ONLY /status/lifecycle + /status/conditions, preserving a fresher condition the reconcile did not compute', async () => {
    const { reconciler, customApi } = createReconciler()
    // The reconcile path drives writeLifecycleStatusToCluster; the fresh read
    // it performs carries an unrelated condition (written by another writer)
    // plus a resourceVersion. The targeted patch must keep that condition.
    customApi.getNamespacedCustomObject.mockResolvedValue({
      metadata: { name: 'stateless-host', namespace: 'mcp-host', resourceVersion: 'rv-77' },
      spec: {
        host: 'stateless-host',
        contextRef: 'context-a',
        secretRef: 'host-secret',
        lifecycle: { stateless: true },
      },
      status: {
        lifecycle: { state: 'active', wakeHandledGeneration: 0 },
        conditions: [
          { type: 'UnrelatedThirdPartyCondition', status: 'True', reason: 'X', message: 'keep me' },
        ],
      },
    })
    const host = makeStatelessHost({
      status: { lifecycle: { state: 'active', wakeHandledGeneration: 0 } },
    })

    await reconciler.reconcile(host)

    const call = customApi.patchNamespacedCustomObjectStatus.mock.calls.at(-1)
    expect(call).toBeDefined()
    const body = (call![0] as { body: Array<{ op: string; path: string; value: unknown }> }).body
    // Precondition op from the fresh read, then targeted sub-object ops only —
    // never a whole-/status op that would drop the unrelated condition.
    expect(body.map(op => op.path)).toEqual([
      '/metadata/resourceVersion',
      '/status/lifecycle',
      '/status/conditions',
    ])
    const writtenConditions = body.find(op => op.path === '/status/conditions')!.value as Array<{
      type: string
    }>
    // The fresher unrelated condition survived (D2: no clobber of un-computed fields).
    expect(writtenConditions.map(c => c.type)).toContain('UnrelatedThirdPartyCondition')
  })

  it('seeds the whole /status once when the fresh Host has no status yet (add /status/lifecycle would fail on a missing parent)', async () => {
    const { reconciler, customApi } = createReconciler()
    // Fresh Host with NO status subresource at all.
    customApi.getNamespacedCustomObject.mockResolvedValue({
      metadata: { name: 'stateless-host', namespace: 'mcp-host', resourceVersion: 'rv-1' },
      spec: {
        host: 'stateless-host',
        contextRef: 'context-a',
        secretRef: 'host-secret',
        lifecycle: { stateless: true },
      },
    })
    const host = makeStatelessHost()
    // Force a status write by giving the cached host a spec.lifecycle (opt-in gate).

    await reconciler.reconcile(host)

    const call = customApi.patchNamespacedCustomObjectStatus.mock.calls.at(-1)
    expect(call).toBeDefined()
    const body = (call![0] as { body: Array<{ op: string; path: string; value: unknown }> }).body
    // Whole-/status seed (preceded by the precondition op), NOT targeted ops.
    expect(body.map(op => op.path)).toEqual(['/metadata/resourceVersion', '/status'])
  })
})

describe('HostReconciler.markHostDrainingFromHeartbeat — AP-1 entry-epoch guard (FIX 2)', () => {
  it('no-ops when a wake was handled after the drain verdict: fresh ACTIVE with a generation past the entry epoch', async () => {
    const logSpy = vi.spyOn(console, 'log').mockImplementation(() => {})
    try {
      const { reconciler, customApi } = createReconciler()
      // The tracker decided drain:true off a snapshot with handled=2 …
      const host = makeStatelessHost({
        status: { lifecycle: { state: 'active', wakeHandledGeneration: 2 } },
      })
      // … but by the persist point a wake was handled (fast-path bumped the
      // generation to 3) and the CR is ACTIVE — exactly the state a
      // just-handled wake produces. Re-writing draining would re-fence the
      // woken pod with requested==handled (nothing revives it): MUST no-op.
      customApi.getNamespacedCustomObject.mockResolvedValue(
        freshHostRead({ state: 'active', wakeHandledGeneration: 3 })
      )

      await reconciler.markHostDrainingFromHeartbeat(host, 2)

      expect(customApi.patchNamespacedCustomObjectStatus).not.toHaveBeenCalled()
      expect(host.status?.lifecycle?.state).toBe('active')
      const staleLines = logSpy.mock.calls
        .map(args => JSON.parse(String(args[0])).msg as string)
        .filter(line => line.includes('phase=draining_write_stale'))
      expect(staleLines).toEqual([
        expect.stringMatching(
          /^\[StatelessSuspend\] host=stateless-host phase=draining_write_stale reason=wake_handled_since entryGeneration=2 freshGeneration=3 ts=\d+$/
        ),
      ])
    } finally {
      logSpy.mockRestore()
    }
  })

  it('no-ops when a wake is pending at the persist point (fresh requested > handled)', async () => {
    const logSpy = vi.spyOn(console, 'log').mockImplementation(() => {})
    try {
      const { reconciler, customApi } = createReconciler()
      const host = makeStatelessHost({
        status: { lifecycle: { state: 'active', wakeHandledGeneration: 2 } },
      })
      customApi.getNamespacedCustomObject.mockResolvedValue({
        metadata: {
          name: 'stateless-host',
          namespace: 'mcp-host',
          annotations: { 'clerum.io/wake-requested': '3' },
        },
        spec: {
          host: 'stateless-host',
          contextRef: 'context-a',
          secretRef: 'host-secret',
          lifecycle: { stateless: true },
        },
        status: { lifecycle: { state: 'active', wakeHandledGeneration: 2 } },
      })

      await reconciler.markHostDrainingFromHeartbeat(host, 2)

      expect(customApi.patchNamespacedCustomObjectStatus).not.toHaveBeenCalled()
      const staleLines = logSpy.mock.calls
        .map(args => JSON.parse(String(args[0])).msg as string)
        .filter(line => line.includes('phase=draining_write_stale'))
      expect(staleLines).toEqual([
        expect.stringMatching(
          /^\[StatelessSuspend\] host=stateless-host phase=draining_write_stale reason=wake_pending requestedGeneration=3 handledGeneration=2 ts=\d+$/
        ),
      ])
    } finally {
      logSpy.mockRestore()
    }
  })

  it('a current verdict (epoch equals the fresh generation) still writes draining', async () => {
    const { reconciler, customApi } = createReconciler()
    const host = makeStatelessHost({
      status: { lifecycle: { state: 'active', wakeHandledGeneration: 2 } },
    })
    customApi.getNamespacedCustomObject.mockResolvedValue(
      freshHostRead({ state: 'active', wakeHandledGeneration: 2 })
    )

    await reconciler.markHostDrainingFromHeartbeat(host, 2)

    const writes = lifecycleStatusWrites(customApi)
    expect(writes).toHaveLength(1)
    expect(writes[0].lifecycle).toEqual({ state: 'draining', wakeHandledGeneration: 2 })
  })
})

describe('HostReconciler.suspendHostFromHeartbeat — outcome contract (FIX 2)', () => {
  it("returns 'skipped_stale' when the generation-epoch guard no-ops the commit", async () => {
    const { reconciler, customApi } = createReconciler()
    const host = makeStatelessHost({
      status: { lifecycle: { state: 'draining', wakeHandledGeneration: 2 } },
    })
    customApi.getNamespacedCustomObject.mockResolvedValue(
      freshHostRead({ state: 'draining', wakeHandledGeneration: 3 })
    )
    await expect(reconciler.suspendHostFromHeartbeat(host, 'idle', 2)).resolves.toBe(
      'skipped_stale'
    )
  })

  it("returns 'skipped_stale' when the drain was overturned (fresh active)", async () => {
    const { reconciler, customApi } = createReconciler()
    const host = makeStatelessHost({
      status: { lifecycle: { state: 'draining', wakeHandledGeneration: 1 } },
    })
    customApi.getNamespacedCustomObject.mockResolvedValue(
      freshHostRead({ state: 'active', wakeHandledGeneration: 1 })
    )
    await expect(reconciler.suspendHostFromHeartbeat(host, 'idle', 1)).resolves.toBe(
      'skipped_stale'
    )
  })

  it("returns 'already_suspended' for the idempotent drained-report retry", async () => {
    const { reconciler, customApi } = createReconciler()
    const host = makeStatelessHost({
      status: { lifecycle: { state: 'suspended', wakeHandledGeneration: 2, reason: 'idle' } },
    })
    customApi.getNamespacedCustomObject.mockResolvedValue(
      freshHostRead({ state: 'suspended', wakeHandledGeneration: 2, reason: 'idle' })
    )
    await expect(reconciler.suspendHostFromHeartbeat(host, 'idle', 2)).resolves.toBe(
      'already_suspended'
    )
  })

  it("returns 'suspended' when the commit lands", async () => {
    const { reconciler, customApi } = createReconciler()
    const host = makeStatelessHost({
      status: { lifecycle: { state: 'active', wakeHandledGeneration: 2 } },
    })
    customApi.getNamespacedCustomObject
      .mockResolvedValueOnce(freshHostRead({ state: 'draining', wakeHandledGeneration: 2 }))
      .mockResolvedValue(
        freshHostRead({ state: 'suspended', wakeHandledGeneration: 2, reason: 'idle' })
      )
    await expect(reconciler.suspendHostFromHeartbeat(host, 'idle', 2)).resolves.toBe('suspended')
  })
})

describe('HostReconciler reconcile — stateless replicas derive from FRESH state (FIX 1)', () => {
  it('a stale suspended payload over a fresh ACTIVE CR keeps replicas=1 (never kills the live pod)', async () => {
    const logSpy = vi.spyOn(console, 'log').mockImplementation(() => {})
    try {
      const { reconciler, appsApi, customApi } = createReconciler()
      // The reconcile payload is a STALE cache entry: it still says suspended
      // although the wake was fully handled (fresh: active, requested ==
      // handled == 3) and the pod is serving a turn.
      const host = makeStatelessHost({
        status: { lifecycle: { state: 'suspended', wakeHandledGeneration: 3, reason: 'idle' } },
      })
      customApi.getNamespacedCustomObject.mockResolvedValue(
        freshHostRead({ state: 'active', wakeHandledGeneration: 3 })
      )

      await reconciler.reconcile(host)

      // Replicas derive from FRESH (active), not the stale payload: the live
      // pod is never scaled to 0.
      expect(hostDeploymentBody(appsApi, 'stateless-host').spec?.replicas).toBe(1)
      const guardLines = logSpy.mock.calls
        .map(args => JSON.parse(String(args[0])).msg as string)
        .filter(line => line.includes('disagrees with fresh'))
      expect(guardLines).toEqual([
        '[HostReconciler] Stateless replicas guard for "stateless-host": cached lifecycle state "suspended" disagrees with fresh "active" — deriving replicas from FRESH state',
      ])
    } finally {
      logSpy.mockRestore()
    }
  })

  it('a stale active payload over a fresh SUSPENDED CR keeps replicas=0 (no resurrection without a wake)', async () => {
    const { reconciler, appsApi, customApi } = createReconciler()
    // Mirror direction: the payload predates a heartbeat suspension; fresh
    // says suspended with no pending wake (requested == handled).
    const host = makeStatelessHost({
      status: { lifecycle: { state: 'active', wakeHandledGeneration: 3 } },
    })
    customApi.getNamespacedCustomObject.mockResolvedValue(
      freshHostRead({ state: 'suspended', wakeHandledGeneration: 3, reason: 'idle' })
    )

    await reconciler.reconcile(host)

    expect(hostDeploymentBody(appsApi, 'stateless-host').spec?.replicas).toBe(0)
    // The rebuilt assessment preserves the heartbeat-managed reason.
    expect(host.status?.lifecycle?.state).toBe('suspended')
  })

  it('agreement between cache and fresh keeps the assessment untouched (replicas from the durable state)', async () => {
    const { reconciler, appsApi, customApi } = createReconciler()
    const host = makeStatelessHost({
      status: { lifecycle: { state: 'suspended', wakeHandledGeneration: 2, reason: 'idle' } },
    })
    customApi.getNamespacedCustomObject.mockResolvedValue(
      freshHostRead({ state: 'suspended', wakeHandledGeneration: 2, reason: 'idle' })
    )

    await reconciler.reconcile(host)

    expect(hostDeploymentBody(appsApi, 'stateless-host').spec?.replicas).toBe(0)
  })

  it('a fresh-read failure skips the Deployment scale entirely this pass (loud, conservative)', async () => {
    const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})
    try {
      const { reconciler, appsApi, customApi } = createReconciler()
      const host = makeStatelessHost({
        status: { lifecycle: { state: 'active', wakeHandledGeneration: 0 } },
      })
      customApi.getNamespacedCustomObject.mockRejectedValue(new Error('apiserver unavailable'))

      await reconciler.reconcile(host)

      // No Deployment apply happened: never commit replicas from a
      // possibly-stale cached state without the fresh confirmation.
      expect(appsApi.createNamespacedDeployment).not.toHaveBeenCalled()
      expect(appsApi.replaceNamespacedDeployment).not.toHaveBeenCalled()
      expect(
        errorSpy.mock.calls.filter(args =>
          String(args[0]).includes('skipping the Deployment scale this pass')
        )
      ).toHaveLength(1)
    } finally {
      errorSpy.mockRestore()
    }
  })
})

describe('H2: guarded cache reflection', () => {
  it('reflects a committed lifecycle outcome onto the current cache entry without a watch event', async () => {
    const { reconciler, customApi } = createReconciler()
    const entry = {
      ...makeStatelessHost({
        status: { lifecycle: { state: 'active', wakeHandledGeneration: 0 } },
      }),
      uid: 'uid-1',
      generation: 1,
      resourceVersion: 'rv-1',
    }
    const hosts = new Map<string, HostCRD>([[entry.name, entry]])
    reconciler.setResolveCurrentHost(name => hosts.get(name))
    reconciler.setHostWatchAuthority(() => ({ known: true, generation: 17 }))
    // Exactly the k8sClient.ts wiring shape (uid-guarded apply onto the cache).
    reconciler.setReflectHostOutcome((name, uid, apply) => {
      const cached = hosts.get(name)
      if (!cached || uid === undefined || cached.uid !== uid) return
      apply(cached)
    })
    customApi.getNamespacedCustomObject.mockResolvedValue({
      ...freshHostRead({ state: 'active', wakeHandledGeneration: 0 }),
      metadata: {
        name: entry.name,
        namespace: entry.namespace,
        uid: 'uid-1',
        generation: 1,
        resourceVersion: 'fresh-rv',
      },
    })

    await reconciler.markHostDrainingFromHeartbeat(entry, 0)

    // The status write committed to the API…
    expect(lifecycleStatusWrites(customApi).at(-1)?.lifecycle?.state).toBe('draining')
    // …and — the H2 fix — the committed outcome is visible on the CANONICAL cache
    // entry without waiting for a watch MODIFIED. With the resolver wired, the
    // heartbeat core mutates the admitted clone (Shape A), so this can ONLY be
    // true if the guarded reflector routed the outcome onto the cache entry.
    expect(hosts.get(entry.name)?.status?.lifecycle?.state).toBe('draining')
  })

  it('never reflects a committed outcome onto a same-name recreation with a different uid', async () => {
    const { reconciler, customApi } = createReconciler()
    const entry = {
      ...makeStatelessHost({
        status: { lifecycle: { state: 'active', wakeHandledGeneration: 0 } },
      }),
      uid: 'uid-1',
      generation: 1,
      resourceVersion: 'rv-1',
    }
    const hosts = new Map<string, HostCRD>([[entry.name, entry]])
    const replacement = { ...makeStatelessHost({ name: entry.name }), uid: 'uid-2' }
    reconciler.setResolveCurrentHost(name => hosts.get(name))
    reconciler.setHostWatchAuthority(() => ({ known: true, generation: 17 }))
    reconciler.setReflectHostOutcome((name, uid, apply) => {
      const cached = hosts.get(name)
      if (!cached || uid === undefined || cached.uid !== uid) return
      apply(cached)
    })
    customApi.getNamespacedCustomObject.mockResolvedValue({
      ...freshHostRead({ state: 'active', wakeHandledGeneration: 0 }),
      metadata: {
        name: entry.name,
        namespace: entry.namespace,
        uid: 'uid-1',
        generation: 1,
        resourceVersion: 'fresh-rv',
      },
    })
    // A watch DELETE+ADD recreates the entry (new uid) mid-commit.
    customApi.patchNamespacedCustomObjectStatus.mockImplementation(async () => {
      hosts.set(entry.name, replacement)
      return {}
    })

    await reconciler.markHostDrainingFromHeartbeat(entry, 0)

    // The write committed…
    expect(lifecycleStatusWrites(customApi).at(-1)?.lifecycle?.state).toBe('draining')
    // …but the recreation (uid-2) was NOT corrupted by the stale reflection.
    expect(replacement.status?.lifecycle).toBeUndefined()
  })
})