Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript
15.9 KB
Raw Blame
import { beforeEach, describe, expect, it, vi } from 'vitest'
import {
  asAppsApi,
  asCoreApi,
  asCustomApi,
  asNetworkingApi,
  asRbacApi,
  createMockAppsApi,
  createMockCoreApi,
  createMockCustomApi,
  createMockNetworkingApi,
  createMockRbacApi,
  makeStubKc,
} from '../test/__fixtures__/testMocks'
import { HostReconciler } from './hostReconciler'
import { CREATE_KINDS, createsTotal } from './metrics'
import type { HostCRD } from './types'

const host: HostCRD = {
  name: 'rbac-host',
  namespace: 'mcp-host',
  uid: 'host-uid',
  generation: 1,
  spec: { host: 'rbac-host', contextRef: 'context', secretRef: 'host-secret' },
}

function fixture() {
  const core = createMockCoreApi()
  const rbac = createMockRbacApi()
  const apps = createMockAppsApi()
  const reconciler = new HostReconciler(makeStubKc(), {
    coreApi: asCoreApi(core),
    appsApi: asAppsApi(apps),
    rbacApi: asRbacApi(rbac),
    customApi: asCustomApi(createMockCustomApi()),
    networkingApi: asNetworkingApi(createMockNetworkingApi()),
  })
  // A failing RBAC reconcile must not reach credential issuance. This canary
  // also keeps the negative-control mutation entirely local.
  const provision = vi
    .spyOn(reconciler as any, 'provisionRuntimeTokenRevision')
    .mockImplementation(() => {
      throw new Error('Downstream provisioning must not execute after an RBAC failure')
    })
  return { core, rbac, apps, reconciler, provision }
}

const rbacCases = ['ServiceAccount', 'Role', 'RoleBinding'] as const
type RbacKind = (typeof rbacCases)[number]
function operations(f: ReturnType<typeof fixture>, kind: RbacKind) {
  if (kind === 'ServiceAccount')
    return {
      read: f.core.readNamespacedServiceAccount,
      create: f.core.createNamespacedServiceAccount,
      method: 'ensureHostServiceAccount',
    }
  if (kind === 'Role')
    return {
      read: f.rbac.readNamespacedRole,
      create: f.rbac.createNamespacedRole,
      method: 'ensureHostRole',
    }
  return {
    read: f.rbac.readNamespacedRoleBinding,
    create: f.rbac.createNamespacedRoleBinding,
    method: 'ensureHostRoleBinding',
  }
}

async function count(kind: string, outcome: string) {
  const sample = (await createsTotal.get()).values.find(
    v => v.labels.kind === kind && v.labels.outcome === outcome
  )
  expect(sample).toBeDefined()
  return sample!.value
}

describe('Host read-first RBAC and PVC contracts', () => {
  beforeEach(() => {
    createsTotal.reset()
    for (const kind of CREATE_KINDS)
      for (const outcome of ['created', 'conflict', 'error', 'skipped'])
        createsTotal.inc({ kind, outcome }, 0)
  })

  it.each(rbacCases)('%s propagates initial GET403 without attempting a write', async kind => {
    const f = fixture()
    const { read, create, method } = operations(f, kind)
    const denied = { code: 403 }
    read.mockRejectedValueOnce(denied)
    await expect((f.reconciler as any)[method](host)).rejects.toBe(denied)
    expect(read).toHaveBeenCalledOnce()
    expect(create).not.toHaveBeenCalled()
    expect(f.rbac.replaceNamespacedRole).not.toHaveBeenCalled()
    expect(await count(kind, 'skipped')).toBe(0)
  })

  it.each(rbacCases)(
    '%s propagates an unexpected POST failure after observed absence',
    async kind => {
      const f = fixture()
      const { read, create, method } = operations(f, kind)
      const failure = { code: 503 }
      read.mockRejectedValueOnce({ code: 404 })
      create.mockRejectedValueOnce(failure)
      await expect((f.reconciler as any)[method](host)).rejects.toBe(failure)
      expect(read).toHaveBeenCalledOnce()
      expect(create).toHaveBeenCalledOnce()
      expect(f.rbac.replaceNamespacedRole).not.toHaveBeenCalled()
      expect(await count(kind, 'error')).toBe(1)
      expect(await count(kind, 'skipped')).toBe(0)
    }
  )

  it.each(rbacCases)('%s rereads after POST409 and propagates raced GET403', async kind => {
    const f = fixture()
    const { read, create, method } = operations(f, kind)
    const denied = { code: 403 }
    read.mockRejectedValueOnce({ code: 404 }).mockRejectedValueOnce(denied)
    create.mockRejectedValueOnce({ code: 409 })
    await expect((f.reconciler as any)[method](host)).rejects.toBe(denied)
    expect(read).toHaveBeenCalledTimes(2)
    expect(create).toHaveBeenCalledOnce()
    expect(f.rbac.replaceNamespacedRole).not.toHaveBeenCalled()
    expect(await count(kind, 'conflict')).toBe(1)
    expect(await count(kind, 'skipped')).toBe(0)
  })

  it.each(rbacCases)(
    '%s retains the benign POST409 then GET404 disappearance without a skip',
    async kind => {
      const f = fixture()
      const { read, create, method } = operations(f, kind)
      read.mockRejectedValueOnce({ code: 404 }).mockRejectedValueOnce({ code: 404 })
      create.mockRejectedValueOnce({ code: 409 })
      await expect((f.reconciler as any)[method](host)).resolves.toBeUndefined()
      expect(read).toHaveBeenCalledTimes(2)
      expect(create).toHaveBeenCalledOnce()
      expect(f.rbac.replaceNamespacedRole).not.toHaveBeenCalled()
      expect(await count(kind, 'conflict')).toBe(1)
      expect(await count(kind, 'skipped')).toBe(0)
    }
  )

  it.each(rbacCases)(
    '%s aborts the Host reconcile before downstream resources on failure',
    async kind => {
      const f = fixture()
      const { read, create } = operations(f, kind)
      const denied = { code: 403 }
      read.mockRejectedValueOnce(denied)
      const reportError = vi.spyOn(f.reconciler as any, 'enqueueControllerError')
      await expect(f.reconciler.reconcile(host)).rejects.toBe(denied)
      expect(read).toHaveBeenCalledOnce()
      expect(create).not.toHaveBeenCalled()
      expect(f.core.readNamespacedPersistentVolumeClaim).not.toHaveBeenCalled()
      expect(f.core.createNamespacedPersistentVolumeClaim).not.toHaveBeenCalled()
      expect(f.core.replaceNamespacedPersistentVolumeClaim).not.toHaveBeenCalled()
      expect(f.core.readNamespacedService).not.toHaveBeenCalled()
      expect(f.apps.createNamespacedDeployment).not.toHaveBeenCalled()
      expect(f.apps.replaceNamespacedDeployment).not.toHaveBeenCalled()
      expect(f.provision).not.toHaveBeenCalled()
      expect(reportError).toHaveBeenCalledWith(
        expect.objectContaining({ name: host.name }),
        'reconcile_exception',
        denied
      )
    }
  )

  it.each(['ServiceAccount', 'RoleBinding'] as const)(
    '%s preserves present state with one read and no PUT',
    async kind => {
      const f = fixture()
      const { read, create, method } = operations(f, kind)
      create.mockRejectedValue({ code: 409 })
      await (f.reconciler as any)[method](host)
      expect(read).toHaveBeenCalledOnce()
      expect(create).not.toHaveBeenCalled()
      expect(f.rbac.replaceNamespacedRole).not.toHaveBeenCalled()
      expect(await count(kind, 'skipped')).toBe(1)
    }
  )

  it.each(['ServiceAccount', 'Role', 'RoleBinding', 'PersistentVolumeClaim'] as const)(
    '%s rechecks admission after GET404',
    async kind => {
      const f = fixture()
      const op =
        kind === 'PersistentVolumeClaim'
          ? {
              read: f.core.readNamespacedPersistentVolumeClaim,
              create: f.core.createNamespacedPersistentVolumeClaim,
              method: 'ensurePvc',
            }
          : operations(f, kind)
      let current = true
      const retired = new Error('admission retired during existence read')
      const revalidate = vi.fn(() => {
        if (!current) throw retired
      })
      op.read.mockImplementationOnce(async () => {
        current = false
        throw { code: 404 }
      })
      await expect((f.reconciler as any)[op.method](host, revalidate)).rejects.toBe(retired)
      expect(op.read).toHaveBeenCalledOnce()
      expect(revalidate).toHaveBeenCalledTimes(2)
      expect(op.create).not.toHaveBeenCalled()
      expect(f.rbac.replaceNamespacedRole).not.toHaveBeenCalled()
      expect(f.core.replaceNamespacedPersistentVolumeClaim).not.toHaveBeenCalled()
      expect(await count(kind, 'skipped')).toBe(0)
    }
  )

  it('keeps Role PUT404 disappearance benign without counting a successful skip', async () => {
    const f = fixture()
    f.rbac.replaceNamespacedRole.mockRejectedValueOnce({ code: 404 })
    await expect((f.reconciler as any).ensureHostRole(host)).resolves.toBeUndefined()
    expect(f.rbac.readNamespacedRole).toHaveBeenCalledOnce()
    expect(f.rbac.replaceNamespacedRole).toHaveBeenCalledOnce()
    expect(f.rbac.createNamespacedRole).not.toHaveBeenCalled()
    expect(await count('Role', 'skipped')).toBe(0)
  })

  it.each([403, 500])('propagates Role PUT%d without retrying', async code => {
    const f = fixture()
    const failure = { code }
    f.rbac.replaceNamespacedRole.mockRejectedValueOnce(failure)
    await expect((f.reconciler as any).ensureHostRole(host)).rejects.toBe(failure)
    expect(f.rbac.readNamespacedRole).toHaveBeenCalledOnce()
    expect(f.rbac.replaceNamespacedRole).toHaveBeenCalledOnce()
    expect(f.rbac.createNamespacedRole).not.toHaveBeenCalled()
    expect(await count('Role', 'skipped')).toBe(0)
  })

  it('rereads a conflicted Role and replaces using the fresh resourceVersion', async () => {
    const f = fixture()
    f.rbac.readNamespacedRole
      .mockResolvedValueOnce({ metadata: { name: 'host-rbac-host', resourceVersion: '17' } })
      .mockResolvedValueOnce({ metadata: { name: 'host-rbac-host', resourceVersion: '18' } })
    f.rbac.replaceNamespacedRole.mockRejectedValueOnce({ code: 409 })
    await expect((f.reconciler as any).ensureHostRole(host)).resolves.toBeUndefined()
    expect(f.rbac.readNamespacedRole).toHaveBeenCalledTimes(2)
    expect(f.rbac.replaceNamespacedRole).toHaveBeenCalledTimes(2)
    expect(
      f.rbac.replaceNamespacedRole.mock.calls.map(
        ([request]) => request.body.metadata.resourceVersion
      )
    ).toEqual(['17', '18'])
    expect(f.rbac.createNamespacedRole).not.toHaveBeenCalled()
    expect(await count('Role', 'skipped')).toBe(1)
  })

  it('accepts a Role already converged by the competing writer after a conflict', async () => {
    const f = fixture()
    f.rbac.replaceNamespacedRole.mockImplementationOnce(async ({ body }) => {
      f.rbac.readNamespacedRole.mockResolvedValue({
        ...structuredClone(body),
        metadata: { ...body.metadata, resourceVersion: '18' },
      })
      throw { code: 409 }
    })
    await expect((f.reconciler as any).ensureHostRole(host)).resolves.toBeUndefined()
    expect(f.rbac.readNamespacedRole).toHaveBeenCalledTimes(2)
    expect(f.rbac.replaceNamespacedRole).toHaveBeenCalledOnce()
    expect(f.rbac.createNamespacedRole).not.toHaveBeenCalled()
    expect(await count('Role', 'skipped')).toBe(1)
  })

  it('propagates the final Role conflict after three attempts', async () => {
    const f = fixture()
    const failure = { code: 409 }
    f.rbac.replaceNamespacedRole.mockRejectedValue(failure)
    await expect((f.reconciler as any).ensureHostRole(host)).rejects.toBe(failure)
    expect(f.rbac.readNamespacedRole).toHaveBeenCalledTimes(3)
    expect(f.rbac.replaceNamespacedRole).toHaveBeenCalledTimes(3)
    expect(f.rbac.createNamespacedRole).not.toHaveBeenCalled()
    expect(await count('Role', 'skipped')).toBe(0)
  })

  it('rechecks admission before retrying a conflicted Role write', async () => {
    const f = fixture()
    let current = true
    const retired = new Error('Host spec changed while the Role write was in flight')
    retired.name = 'HostMutationSpecRevisionChangedError'
    const revalidate = () => {
      if (!current) throw retired
    }
    f.rbac.replaceNamespacedRole.mockImplementationOnce(async () => {
      current = false
      throw { code: 409 }
    })
    await expect((f.reconciler as any).ensureHostRole(host, revalidate)).rejects.toBe(retired)
    expect(f.rbac.replaceNamespacedRole).toHaveBeenCalledOnce()
    expect(f.rbac.createNamespacedRole).not.toHaveBeenCalled()
    expect(await count('Role', 'skipped')).toBe(0)
  })

  it('preserves a bound Host PVC without attempting an immutable update', async () => {
    const f = fixture()
    f.core.readNamespacedPersistentVolumeClaim.mockResolvedValueOnce({
      metadata: {
        name: 'rbac-host-workspace',
        namespace: host.namespace,
        uid: 'pvc-uid',
        resourceVersion: '17',
      },
      spec: { volumeName: 'bound-volume' },
    })
    await expect((f.reconciler as any).ensurePvc(host)).resolves.toBe(true)
    expect(f.core.readNamespacedPersistentVolumeClaim).toHaveBeenCalledOnce()
    expect(f.core.createNamespacedPersistentVolumeClaim).not.toHaveBeenCalled()
    expect(f.core.replaceNamespacedPersistentVolumeClaim).not.toHaveBeenCalled()
    expect(await count('PersistentVolumeClaim', 'skipped')).toBe(1)
  })

  it('updates an unbound Host PVC once with its observed resourceVersion', async () => {
    const f = fixture()
    await expect((f.reconciler as any).ensurePvc(host)).resolves.toBe(true)
    expect(f.core.readNamespacedPersistentVolumeClaim).toHaveBeenCalledOnce()
    expect(f.core.createNamespacedPersistentVolumeClaim).not.toHaveBeenCalled()
    expect(f.core.replaceNamespacedPersistentVolumeClaim).toHaveBeenCalledOnce()
    expect(
      f.core.replaceNamespacedPersistentVolumeClaim.mock.calls[0][0].body.metadata.resourceVersion
    ).toBe('1')
    expect(await count('PersistentVolumeClaim', 'skipped')).toBe(1)
  })

  it('counts a Host PVC create race as successful after fresh read convergence', async () => {
    const f = fixture()
    f.core.readNamespacedPersistentVolumeClaim
      .mockRejectedValueOnce({ code: 404 })
      .mockResolvedValueOnce({
        metadata: {
          name: 'rbac-host-workspace',
          namespace: host.namespace,
          uid: 'pvc-uid',
          resourceVersion: '17',
        },
      })
    f.core.createNamespacedPersistentVolumeClaim.mockRejectedValueOnce({ code: 409 })

    await expect((f.reconciler as any).ensurePvc(host)).resolves.toBe(true)

    expect(f.core.readNamespacedPersistentVolumeClaim).toHaveBeenCalledTimes(2)
    expect(f.core.createNamespacedPersistentVolumeClaim).toHaveBeenCalledOnce()
    expect(f.core.replaceNamespacedPersistentVolumeClaim).toHaveBeenCalledOnce()
    expect(await count('PersistentVolumeClaim', 'conflict')).toBe(1)
  })

  it('propagates a Host PVC initial GET failure outside retained write catches', async () => {
    const f = fixture()
    const failure = { code: 403 }
    f.core.readNamespacedPersistentVolumeClaim.mockRejectedValueOnce(failure)
    await expect((f.reconciler as any).ensurePvc(host)).rejects.toBe(failure)
    expect(f.core.readNamespacedPersistentVolumeClaim).toHaveBeenCalledOnce()
    expect(f.core.createNamespacedPersistentVolumeClaim).not.toHaveBeenCalled()
    expect(f.core.replaceNamespacedPersistentVolumeClaim).not.toHaveBeenCalled()
    expect(await count('PersistentVolumeClaim', 'skipped')).toBe(0)
  })

  it('retains the Host PVC POST failure without recording a skip', async () => {
    const f = fixture()
    f.core.readNamespacedPersistentVolumeClaim.mockRejectedValueOnce({ code: 404 })
    f.core.createNamespacedPersistentVolumeClaim.mockRejectedValueOnce({ code: 503 })
    await expect((f.reconciler as any).ensurePvc(host)).resolves.toBe(false)
    expect(f.core.readNamespacedPersistentVolumeClaim).toHaveBeenCalledOnce()
    expect(f.core.createNamespacedPersistentVolumeClaim).toHaveBeenCalledOnce()
    expect(f.core.replaceNamespacedPersistentVolumeClaim).not.toHaveBeenCalled()
    expect(await count('PersistentVolumeClaim', 'error')).toBe(1)
    expect(await count('PersistentVolumeClaim', 'skipped')).toBe(0)
  })

  it.each([409, 503])('retains Host PVC PUT%d without retrying or recording a skip', async code => {
    const f = fixture()
    f.core.replaceNamespacedPersistentVolumeClaim.mockRejectedValueOnce({ code })
    await expect((f.reconciler as any).ensurePvc(host)).resolves.toBe(false)
    expect(f.core.readNamespacedPersistentVolumeClaim).toHaveBeenCalledOnce()
    expect(f.core.replaceNamespacedPersistentVolumeClaim).toHaveBeenCalledOnce()
    expect(f.core.createNamespacedPersistentVolumeClaim).not.toHaveBeenCalled()
    expect(await count('PersistentVolumeClaim', 'skipped')).toBe(0)
  })
})