Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript
14.3 KB
Raw Blame
import { beforeEach, describe, expect, it, vi } from 'vitest'
import type * as k8s from '@kubernetes/client-node'
import {
  asAppsApi,
  asCoreApi,
  asCustomApi,
  asNetworkingApi,
  asRbacApi,
  createMockAppsApi,
  createMockCoreApi,
  createMockCustomApi,
  createMockNetworkingApi,
  createMockRbacApi,
  makeStubKc,
} from '../test/__fixtures__/testMocks'
import { asApiserverService } from './__tests__/asApiserverService'
import { HostReconciler } from './hostReconciler'
import { CREATE_KINDS, createsTotal, writeSkipsTotal } from './metrics'
import type { HostCRD } from './types'

const host: HostCRD = {
  name: 'read-first-host',
  namespace: 'mcp-host',
  uid: 'host-uid',
  spec: { host: 'read-first-host', contextRef: 'context', secretRef: 'host-secret' },
}

function fixture() {
  const core = createMockCoreApi()
  const apps = createMockAppsApi()
  const reconciler = new HostReconciler(makeStubKc(), {
    coreApi: asCoreApi(core),
    appsApi: asAppsApi(apps),
    networkingApi: asNetworkingApi(createMockNetworkingApi()),
    rbacApi: asRbacApi(createMockRbacApi()),
    customApi: asCustomApi(createMockCustomApi()),
  })
  vi.spyOn(reconciler as any, 'computeChannelReaderRevisionForHost').mockResolvedValue('revision')
  return { core, apps, reconciler }
}

async function count(kind: string, outcome: string) {
  const sample = (await createsTotal.get()).values.find(
    v => v.labels.kind === kind && v.labels.outcome === outcome
  )
  expect(sample).toBeDefined()
  return sample!.value
}

const cases = [
  ['Host Service', 'Service', 'ensureService'],
  ['channel-reader Service', 'Service', 'reconcileChannelReaderService'],
  ['Host Deployment', 'Deployment', 'ensureDeployment'],
  ['channel-reader Deployment', 'Deployment', 'reconcileChannelReaderDeployment'],
] as const

function invoke(reconciler: HostReconciler, method: string, revalidate?: () => void) {
  return method === 'ensureDeployment'
    ? (reconciler as any)[method](host, [], 'runtime-revision', undefined, undefined, revalidate)
    : (reconciler as any)[method](host, revalidate)
}

describe('Host read-first Service and Deployment contracts', () => {
  beforeEach(() => {
    createsTotal.reset()
    for (const kind of CREATE_KINDS)
      for (const outcome of ['created', 'conflict', 'error', 'skipped'])
        createsTotal.inc({ kind, outcome }, 0)
  })

  it.each(cases)(
    '%s propagates initial GET403 without a mutation or skip',
    async (_label, kind, method) => {
      const { core, apps, reconciler } = fixture()
      const read = kind === 'Service' ? core.readNamespacedService : apps.readNamespacedDeployment
      const create =
        kind === 'Service' ? core.createNamespacedService : apps.createNamespacedDeployment
      const replace =
        kind === 'Service' ? core.replaceNamespacedService : apps.replaceNamespacedDeployment
      const denied = { code: 403 }
      read.mockRejectedValueOnce(denied)
      await expect(invoke(reconciler, method)).rejects.toBe(denied)
      expect(read).toHaveBeenCalledOnce()
      expect(create).not.toHaveBeenCalled()
      expect(replace).not.toHaveBeenCalled()
      expect(await count(kind, 'skipped')).toBe(0)
      expect(await count(kind, 'error')).toBe(0)
    }
  )

  it.each(cases)(
    '%s revalidates after an absent GET before any POST',
    async (_label, kind, method) => {
      const { core, apps, reconciler } = fixture()
      const read = kind === 'Service' ? core.readNamespacedService : apps.readNamespacedDeployment
      const create =
        kind === 'Service' ? core.createNamespacedService : apps.createNamespacedDeployment
      const replace =
        kind === 'Service' ? core.replaceNamespacedService : apps.replaceNamespacedDeployment
      let current = true
      const superseded = new Error('admission retired during existence GET')
      const revalidate = vi.fn(() => {
        if (!current) throw superseded
      })
      read.mockImplementationOnce(async () => {
        current = false
        throw { code: 404 }
      })
      await expect(invoke(reconciler, method, revalidate)).rejects.toBe(superseded)
      expect(read).toHaveBeenCalledOnce()
      expect(revalidate).toHaveBeenCalledTimes(2)
      expect(create).not.toHaveBeenCalled()
      expect(replace).not.toHaveBeenCalled()
      expect(await count(kind, 'skipped')).toBe(0)
    }
  )

  it('rechecks Host admission after asynchronous Deployment body resolution', async () => {
    const { apps, reconciler } = fixture()
    let current = true
    const superseded = new Error('admission retired during body resolution')
    const revalidate = vi.fn(() => {
      if (!current) throw superseded
    })
    apps.readNamespacedDeployment.mockRejectedValueOnce({ code: 404 })
    const resolveState = vi.fn(async () => {
      expect(apps.readNamespacedDeployment).toHaveBeenCalledOnce()
      current = false
      return {
        runtimeTokenRevision: 'fresh-revision',
        lifecycle: { stateless: false, state: 'active' },
      }
    })
    await expect(
      (reconciler as any).ensureDeployment(
        host,
        [],
        'old-revision',
        undefined,
        resolveState,
        revalidate
      )
    ).rejects.toBe(superseded)
    expect(resolveState).toHaveBeenCalledOnce()
    expect(apps.createNamespacedDeployment).not.toHaveBeenCalled()
    expect(apps.replaceNamespacedDeployment).not.toHaveBeenCalled()
    expect(await count('Deployment', 'skipped')).toBe(0)
  })

  it('uses state resolved after GET404 in the first Deployment POST', async () => {
    const { apps, reconciler } = fixture()
    apps.readNamespacedDeployment.mockRejectedValueOnce({ code: 404 })
    const resolveState = vi.fn(async () => {
      expect(apps.readNamespacedDeployment).toHaveBeenCalledOnce()
      return {
        runtimeTokenRevision: 'fresh-revision',
        lifecycle: { stateless: false, state: 'active' },
      }
    })
    await (reconciler as any).ensureDeployment(host, [], 'old-revision', undefined, resolveState)
    expect(resolveState).toHaveBeenCalledOnce()
    expect(apps.createNamespacedDeployment).toHaveBeenCalledOnce()
    const body = apps.createNamespacedDeployment.mock.calls[0][0].body as k8s.V1Deployment
    expect(body.spec?.template.metadata?.annotations?.['clerum.io/runtime-token-revision']).toBe(
      'fresh-revision'
    )
    expect(await count('Deployment', 'created')).toBe(1)
    expect(await count('Deployment', 'skipped')).toBe(0)
  })

  it.each([0, 1])(
    'builds a present Host Deployment once per attempt with fresh state (%i conflicts)',
    async conflicts => {
      const { apps, reconciler } = fixture()
      const existing = reconciler.buildDeployment(host, [], 'old-revision')
      apps.readNamespacedDeployment.mockImplementation(async () => ({
        ...existing,
        metadata: {
          ...existing.metadata,
          resourceVersion: String(apps.readNamespacedDeployment.mock.calls.length),
        },
      }))
      if (conflicts) apps.replaceNamespacedDeployment.mockRejectedValueOnce({ code: 409 })
      const build = vi.spyOn(reconciler, 'buildDeployment')
      const resolveState = vi.fn(async () => {
        const attempt = apps.readNamespacedDeployment.mock.calls.length
        return {
          runtimeTokenRevision: `fresh-revision-${attempt}`,
          lifecycle: { stateless: true, state: attempt === 1 ? 'active' : 'suspended' },
        }
      })

      await (reconciler as any).ensureDeployment(host, [], 'old-revision', undefined, resolveState)

      const attempts = conflicts + 1
      expect(apps.readNamespacedDeployment).toHaveBeenCalledTimes(attempts)
      expect(resolveState).toHaveBeenCalledTimes(attempts)
      expect(build).toHaveBeenCalledTimes(attempts)
      expect(apps.createNamespacedDeployment).not.toHaveBeenCalled()
      expect(apps.replaceNamespacedDeployment).toHaveBeenCalledTimes(attempts)
      for (let attempt = 1; attempt <= attempts; attempt++) {
        const body = apps.replaceNamespacedDeployment.mock.calls[attempt - 1][0]
          .body as k8s.V1Deployment
        expect(body.metadata?.resourceVersion).toBe(String(attempt))
        expect(
          body.spec?.template.metadata?.annotations?.['clerum.io/runtime-token-revision']
        ).toBe(`fresh-revision-${attempt}`)
        expect(body.spec?.replicas).toBe(attempt === 1 ? 1 : 0)
      }
    }
  )

  it('retains a Host Service POST failure without recording presence suppression', async () => {
    const { core, reconciler } = fixture()
    core.readNamespacedService.mockRejectedValueOnce({ code: 404 })
    core.createNamespacedService.mockRejectedValueOnce({ code: 503 })
    await expect((reconciler as any).ensureService(host)).resolves.toBe(false)
    expect(core.readNamespacedService).toHaveBeenCalledOnce()
    expect(core.createNamespacedService).toHaveBeenCalledOnce()
    expect(core.replaceNamespacedService).not.toHaveBeenCalled()
    expect(await count('Service', 'error')).toBe(1)
    expect(await count('Service', 'skipped')).toBe(0)
  })

  it('retains a Host Service convergence failure without recording a successful skip', async () => {
    const { core, reconciler } = fixture()
    core.replaceNamespacedService.mockRejectedValueOnce({ code: 503 })
    await expect((reconciler as any).ensureService(host)).resolves.toBe(false)
    expect(core.readNamespacedService).toHaveBeenCalledOnce()
    expect(core.createNamespacedService).not.toHaveBeenCalled()
    expect(core.replaceNamespacedService).toHaveBeenCalledOnce()
    expect(await count('Service', 'error')).toBe(0)
    expect(await count('Service', 'skipped')).toBe(0)
  })

  it('counts an equivalent present Host Service once without a POST or PUT', async () => {
    const { core, reconciler } = fixture()
    const desired = (reconciler as any).buildService(host) as k8s.V1Service
    core.readNamespacedService.mockResolvedValue(asApiserverService(desired))
    core.createNamespacedService.mockRejectedValue({ code: 409 })
    await expect((reconciler as any).ensureService(host)).resolves.toBe(true)
    expect(core.readNamespacedService).toHaveBeenCalledOnce()
    expect(core.createNamespacedService).not.toHaveBeenCalled()
    expect(core.replaceNamespacedService).not.toHaveBeenCalled()
    expect(await count('Service', 'skipped')).toBe(1)
  })

  it('rejects an invalid channel-reader retry response before another PUT', async () => {
    const { apps, reconciler } = fixture()
    const stale = (reconciler as any).buildChannelReaderDeployment(
      host,
      'revision'
    ) as k8s.V1Deployment
    stale.metadata = { ...stale.metadata, resourceVersion: '1' }
    stale.spec!.template.spec!.containers[0].image = 'channel-reader:stale'
    apps.readNamespacedDeployment
      .mockResolvedValueOnce(stale)
      .mockResolvedValueOnce({ metadata: { resourceVersion: '2' } })
    apps.replaceNamespacedDeployment.mockRejectedValueOnce({ code: 409 })
    await expect((reconciler as any).reconcileChannelReaderDeployment(host)).rejects.toThrow(
      'metadata.name'
    )
    expect(apps.readNamespacedDeployment).toHaveBeenCalledTimes(2)
    expect(apps.replaceNamespacedDeployment).toHaveBeenCalledOnce()
    expect(apps.createNamespacedDeployment).not.toHaveBeenCalled()
    expect(await count('Deployment', 'skipped')).toBe(0)
  })

  function hostWithoutSecretRef(): HostCRD {
    return {
      name: 'codex-cert',
      namespace: 'mcp-host',
      uid: 'host-uid-nosecret',
      spec: { host: 'codex-cert', contextRef: 'context' },
    }
  }

  function liveHostDeploymentWithoutSecretValue(
    reconciler: HostReconciler,
    hostNoSecret: HostCRD
  ): k8s.V1Deployment {
    const desired = reconciler.buildDeployment(hostNoSecret, [], 'runtime-revision')
    const liveEqual = structuredClone(desired)
    const liveSecret = liveEqual.spec?.template.spec?.containers
      .flatMap(container => container.env ?? [])
      .find(env => env.name === 'CLERUM_LLM_SECRET_REF')
    if (!liveSecret) {
      throw new Error('expected CLERUM_LLM_SECRET_REF on the built Host Deployment')
    }
    delete liveSecret.value
    liveEqual.metadata = { ...liveEqual.metadata, resourceVersion: '11' }
    return liveEqual
  }

  it('T5: Host without secretRef skips replace when live omits empty value', async () => {
    const { apps, reconciler } = fixture()
    const hostNoSecret = hostWithoutSecretRef()
    const desired = reconciler.buildDeployment(hostNoSecret, [], 'runtime-revision')
    const secretEnv = desired.spec?.template.spec?.containers
      .flatMap(container => container.env ?? [])
      .find(env => env.name === 'CLERUM_LLM_SECRET_REF')
    expect(secretEnv).toEqual({ name: 'CLERUM_LLM_SECRET_REF', value: '' })
    const liveEqual = liveHostDeploymentWithoutSecretValue(reconciler, hostNoSecret)
    writeSkipsTotal.reset()
    apps.readNamespacedDeployment.mockResolvedValue(liveEqual)
    await (reconciler as any).ensureDeployment(hostNoSecret, [], 'runtime-revision')
    expect(apps.readNamespacedDeployment).toHaveBeenCalledTimes(1)
    expect(apps.replaceNamespacedDeployment).not.toHaveBeenCalled()
    expect(apps.createNamespacedDeployment).not.toHaveBeenCalled()
  })

  it('T5: Host without secretRef still replaces when the live image differs', async () => {
    const { apps, reconciler } = fixture()
    const hostNoSecret = hostWithoutSecretRef()
    const liveDrift = liveHostDeploymentWithoutSecretValue(reconciler, hostNoSecret)
    liveDrift.spec!.template.spec!.containers[0].image = 'registry.example.com/mcp-host:other'
    liveDrift.metadata = { ...liveDrift.metadata, resourceVersion: '12' }
    apps.readNamespacedDeployment.mockResolvedValue(liveDrift)
    await (reconciler as any).ensureDeployment(hostNoSecret, [], 'runtime-revision')
    expect(apps.readNamespacedDeployment).toHaveBeenCalledTimes(1)
    expect(apps.replaceNamespacedDeployment).toHaveBeenCalledTimes(1)
  })

  it('counts a Host Service create race as conflict and never as skipped', async () => {
    const { core, reconciler } = fixture()
    core.readNamespacedService.mockRejectedValueOnce({ code: 404 })
    core.createNamespacedService.mockRejectedValueOnce({ code: 409 })
    await expect((reconciler as any).ensureService(host)).resolves.toBe(true)
    expect(core.readNamespacedService).toHaveBeenCalledTimes(2)
    expect(core.createNamespacedService).toHaveBeenCalledOnce()
    expect(core.replaceNamespacedService).toHaveBeenCalledOnce()
    expect(core.replaceNamespacedService.mock.calls[0][0].body.metadata.resourceVersion).toBe('1')
    expect(await count('Service', 'conflict')).toBe(1)
    expect(await count('Service', 'skipped')).toBe(0)
  })
})