Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript
16.2 KB
Raw Blame
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
import * as k8s from '@kubernetes/client-node'
import * as dns from 'node:dns/promises'
import { ExternalEgressConvergenceCoordinator } from './externalEgressConvergenceCoordinator'
import { NetworkPolicyReconciler } from './networkPolicyReconciler'
import type { EgressBinding, McpServerCRD, McpServerCrdStatus } from './types'

vi.mock('./config', () => ({
  config: {
    namespace: 'mcp-server',
    controlPlaneNamespace: 'control-plane',
    hostNamespace: 'mcp-host',
    rpcProxyNamespace: 'rpc-proxy',
    runtimeNamespaces: ['mcp-server'],
    externalEgressDnsResolveTimeoutMs: 5000,
    externalEgressOverlapSec: 300,
    externalEgressMaxEntries: 128,
  },
}))
vi.mock('node:dns/promises', () => ({ resolve4: vi.fn() }))
vi.mock('./metrics', () => ({
  createsTotal: { inc: vi.fn() },
  existenceReadsTotal: { inc: vi.fn() },
  writesTotal: { inc: vi.fn() },
  writeSkipsTotal: { inc: vi.fn() },
  externalEgressRetriesAtCap: { set: vi.fn() },
}))

const currentDns: EgressBinding = { dns: 'current.example.com', port: 443 }
const removedDns: EgressBinding = { dns: 'removed.example.com', port: 8443 }
const currentName = 'ext-egress-observation-mcp-current.example.com-443'
const removedName = 'ext-egress-observation-mcp-removed.example.com-8443'
const apiError = (code: number) => Object.assign(new Error(`Kubernetes API ${code}`), { code })

function fixture(bindings: EgressBinding[]) {
  const server: McpServerCRD = {
    name: 'observation-mcp',
    namespace: 'mcp-server',
    uid: 'server-uid',
    generation: 7,
    spec: {
      contextRef: 'dev',
      image: 'observation:1',
      transport: { type: 'streamableHttp', port: 3000 },
      egressBindings: bindings,
    },
  }
  const policies = new Map<string, k8s.V1NetworkPolicy>()
  const unavailableReads = new Set<string>()
  const unavailableDeletes = new Set<string>()
  let status: McpServerCrdStatus = {}
  let version = 0
  const api = {
    listNamespacedNetworkPolicy: vi.fn(async () => ({
      items: [...policies.values()].map(policy => structuredClone(policy)),
    })),
    readNamespacedNetworkPolicy: vi.fn(async ({ name }: { name: string }) => {
      if (unavailableReads.has(name)) throw apiError(503)
      const policy = policies.get(name)
      if (!policy) throw apiError(404)
      return structuredClone(policy)
    }),
    createNamespacedNetworkPolicy: vi.fn(async ({ body }: { body: k8s.V1NetworkPolicy }) => {
      const name = body.metadata!.name!
      if (policies.has(name)) throw apiError(409)
      const policy = structuredClone(body)
      policy.metadata = { ...policy.metadata, uid: `uid-${name}`, resourceVersion: `${++version}` }
      policies.set(name, policy)
      return structuredClone(policy)
    }),
    replaceNamespacedNetworkPolicy: vi.fn(
      async ({ name, body }: { name: string; body: k8s.V1NetworkPolicy }) => {
        const current = policies.get(name)
        if (!current) throw apiError(404)
        if (body.metadata?.resourceVersion !== current.metadata?.resourceVersion)
          throw apiError(409)
        const policy = structuredClone(body)
        policy.metadata = {
          ...policy.metadata,
          uid: current.metadata!.uid,
          resourceVersion: `${++version}`,
        }
        policies.set(name, policy)
        return structuredClone(policy)
      }
    ),
    deleteNamespacedNetworkPolicy: vi.fn(
      async ({ name, body }: { name: string; body: k8s.V1DeleteOptions }) => {
        if (unavailableDeletes.has(name)) throw apiError(503)
        const current = policies.get(name)
        if (!current) throw apiError(404)
        if (
          body.preconditions?.uid !== current.metadata?.uid ||
          body.preconditions?.resourceVersion !== current.metadata?.resourceVersion
        )
          throw apiError(409)
        policies.delete(name)
      }
    ),
  }
  const customApi = {
    getNamespacedCustomObjectStatus: vi.fn(async () => ({
      metadata: { uid: server.uid, generation: server.generation, resourceVersion: 'status-rv' },
      status: structuredClone(status),
    })),
    patchNamespacedCustomObjectStatus: vi.fn(
      async ({ body }: { body: Array<{ op: string; path: string; value: unknown }> }) => {
        for (const operation of body) {
          if (operation.path === '/status') status = operation.value as McpServerCrdStatus
          if (operation.path === '/status/conditions')
            status.conditions = operation.value as McpServerCrdStatus['conditions']
          if (operation.path === '/status/resolvedEgressIPs')
            status.resolvedEgressIPs = operation.value as McpServerCrdStatus['resolvedEgressIPs']
        }
      }
    ),
  }
  const kc = new k8s.KubeConfig()
  kc.loadFromOptions({
    clusters: [{ name: 'unit', server: 'https://unit.invalid' }],
    users: [{ name: 'unit' }],
    contexts: [{ name: 'unit', cluster: 'unit', user: 'unit' }],
    currentContext: 'unit',
  })
  const reconciler = new NetworkPolicyReconciler(kc, new Map())
  Object.assign(reconciler, { networkingApi: api, customApi })
  return {
    server,
    policies,
    api,
    unavailableReads,
    unavailableDeletes,
    reconciler,
    status: () => status,
    condition: () => status.conditions?.find(condition => condition.type === 'ExternalEgressReady'),
    run: () => reconciler.reconcileExternalEgress(server, { isCurrent: () => true }),
    clearCalls: () => Object.values(api).forEach(mock => mock.mockClear()),
  }
}

describe('external egress observation failures do not decide authorization', () => {
  beforeEach(() => {
    vi.mocked(dns.resolve4).mockResolvedValue([{ address: '1.2.3.4', ttl: 300 }] as never)
  })
  afterEach(() => vi.useRealTimers())

  it('keeps the current policy unmodified and revokes a removed sibling when fresh DNS-policy reads are unavailable', async () => {
    const f = fixture([removedDns, currentDns])
    await f.run()
    const before = structuredClone(f.policies.get(currentName))
    const removed = f.policies.get(removedName)!
    f.server.spec.egressBindings = [currentDns]
    f.unavailableReads.add(currentName)
    f.clearCalls()

    await expect(f.run()).rejects.toThrow('Kubernetes API 503')

    expect(f.policies.get(currentName)).toEqual(before)
    expect(f.api.createNamespacedNetworkPolicy).not.toHaveBeenCalled()
    expect(f.api.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()
    expect(f.api.deleteNamespacedNetworkPolicy).toHaveBeenCalledExactlyOnceWith({
      name: removedName,
      namespace: f.server.namespace,
      body: {
        preconditions: {
          uid: removed.metadata!.uid,
          resourceVersion: removed.metadata!.resourceVersion,
        },
      },
    })
    expect(f.condition()).toMatchObject({
      status: 'False',
      reason: 'ExternalEgressReconcileFailed',
    })
    expect(f.condition()?.message).toMatch(/unavailable|could not be verified/)
    expect(f.condition()?.message).not.toMatch(/REVOKED|provenance was absent or invalid/)
    expect(f.status().resolvedEgressIPs).toEqual([])

    f.unavailableReads.clear()
    await f.run()
    expect(f.condition()).toMatchObject({ status: 'True', reason: 'Reconciled' })
    expect(f.policies.get(currentName)).toEqual(before)
    expect(f.policies.has(removedName)).toBe(false)
  })

  it.each([
    {
      binding: { cidr: '8.8.8.8/32', port: 9443 },
      name: 'ext-egress-observation-mcp-8-8-8-8-32-9443',
    },
    {
      binding: { egressClass: 'public-web' as const },
      name: 'ext-egress-observation-mcp-public-web',
    },
  ])(
    'revokes an independent removed binding despite a $name read failure and recovers without a spec change',
    async ({ binding, name }) => {
      const f = fixture([removedDns, binding])
      await f.run()
      const before = structuredClone(f.policies.get(name))
      f.server.spec.egressBindings = [binding]
      f.unavailableReads.add(name)
      f.clearCalls()

      await expect(f.run()).rejects.toThrow('Kubernetes API 503')

      expect(f.policies.has(removedName)).toBe(false)
      expect(f.policies.get(name)).toEqual(before)
      expect(f.api.createNamespacedNetworkPolicy).not.toHaveBeenCalled()
      expect(f.api.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()
      expect(f.condition()).toMatchObject({
        status: 'False',
        reason: 'ExternalEgressReconcileFailed',
      })
      expect(f.status().resolvedEgressIPs).toEqual([])
      f.unavailableReads.clear()
      await f.run()
      expect(f.condition()).toMatchObject({ status: 'True', reason: 'Reconciled' })
      expect(f.policies.get(name)).toEqual(before)
    }
  )

  it('does not resolve or expand until the independent removal barrier succeeds', async () => {
    const f = fixture([removedDns])
    await f.run()
    f.server.spec.egressBindings = [currentDns]
    f.unavailableDeletes.add(removedName)
    f.clearCalls()
    vi.mocked(dns.resolve4).mockClear()

    await expect(f.run()).rejects.toThrow('Kubernetes API 503')

    expect(dns.resolve4).not.toHaveBeenCalled()
    expect(f.api.createNamespacedNetworkPolicy).not.toHaveBeenCalled()
    expect(f.condition()).toMatchObject({ status: 'False', reason: 'CleanupFailed' })
    f.unavailableDeletes.clear()
    await f.run()
    expect(f.policies.has(removedName)).toBe(false)
    expect(f.policies.has(currentName)).toBe(true)
  })

  it('finishes independent removals when one delete fails, without expanding another binding', async () => {
    const f = fixture([removedDns, currentDns])
    await f.run()
    f.server.spec.egressBindings = [{ dns: 'added.example.com', port: 443 }]
    f.unavailableDeletes.add(removedName)
    f.clearCalls()

    await expect(f.run()).rejects.toThrow('Kubernetes API 503')

    expect(f.policies.has(removedName)).toBe(true)
    expect(f.policies.has(currentName)).toBe(false)
    expect(f.api.createNamespacedNetworkPolicy).not.toHaveBeenCalled()
    expect(f.condition()?.reason).toBe('CleanupFailed')
  })

  it('does not add a sibling permission while a current policy observation is pending', async () => {
    const f = fixture([currentDns])
    await f.run()
    f.server.spec.egressBindings = [{ cidr: '8.8.8.8/32', port: 443 }, currentDns]
    f.unavailableReads.add(currentName)
    f.clearCalls()

    await expect(f.run()).rejects.toThrow('Kubernetes API 503')

    expect(f.api.createNamespacedNetworkPolicy).not.toHaveBeenCalled()
    expect(f.api.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()
    expect(f.policies.size).toBe(1)
    expect(f.status().resolvedEgressIPs).toEqual([])
  })

  it('rechecks a deferred no-op after sibling DNS latency and recovers from the changed snapshot', async () => {
    const sibling: EgressBinding = { dns: 'later.example.com', port: 443 }
    const f = fixture([currentDns, sibling])
    await f.run()
    f.clearCalls()
    vi.mocked(dns.resolve4).mockImplementation(async hostname => {
      if (hostname === sibling.dns) {
        const policy = f.policies.get(currentName)!
        policy.metadata!.resourceVersion = 'concurrent-rv'
        policy.spec!.egress!.push({
          to: [{ ipBlock: { cidr: '8.8.8.8/32' } }],
          ports: [{ port: 443, protocol: 'TCP' }],
        })
      }
      return [{ address: '1.2.3.4', ttl: 300 }] as never
    })

    await expect(f.run()).rejects.toThrow(/changed after the external-egress observation/)

    expect(f.condition()?.status).toBe('False')
    expect(f.status().resolvedEgressIPs).toEqual([])
    expect(f.api.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()
    vi.mocked(dns.resolve4).mockResolvedValue([{ address: '1.2.3.4', ttl: 300 }] as never)
    await f.run()
    expect(f.condition()?.status).toBe('True')
    expect(f.policies.get(currentName)?.spec?.egress).toHaveLength(1)
  })

  it('does not write or report status after the generation fence closes during an unavailable read', async () => {
    const f = fixture([currentDns])
    await f.run()
    f.clearCalls()
    const statusBefore = structuredClone(f.status())
    let current = true
    f.api.readNamespacedNetworkPolicy.mockImplementation(async () => {
      current = false
      throw apiError(503)
    })

    await f.reconciler.reconcileExternalEgress(f.server, { isCurrent: () => current })

    expect(f.api.deleteNamespacedNetworkPolicy).not.toHaveBeenCalled()
    expect(f.api.createNamespacedNetworkPolicy).not.toHaveBeenCalled()
    expect(f.api.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()
    expect(f.status()).toEqual(statusBefore)
  })

  it('does not copy a newer resourceVersion onto a policy calculated before a replace conflict', async () => {
    const f = fixture([currentDns])
    await f.run()
    const policy = f.policies.get(currentName)!
    policy.spec!.egress!.push({
      to: [{ ipBlock: { cidr: '8.8.8.8/32' } }],
      ports: [{ port: 443, protocol: 'TCP' }],
    })
    f.clearCalls()
    f.api.replaceNamespacedNetworkPolicy.mockImplementationOnce(async () => {
      // The first PUT races with a newer resourceVersion after the final plan check.
      // A matching owner alone cannot authorize reuse of the older DNS state.
      policy.metadata!.resourceVersion = 'changed-before-replace'
      throw apiError(409)
    })

    await expect(f.run()).rejects.toThrow(/changed after the external-egress observation/)

    expect(f.api.replaceNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)
    expect(f.api.createNamespacedNetworkPolicy).not.toHaveBeenCalled()
    expect(f.condition()?.status).toBe('False')
    await f.run()
    expect(f.condition()?.status).toBe('True')
    expect(f.policies.get(currentName)?.spec?.egress).toHaveLength(1)
  })

  it('does not certify a policy deleted during PUT409 and before the retry read, then recovers', async () => {
    const f = fixture([currentDns])
    await f.run()
    vi.mocked(dns.resolve4).mockResolvedValue([{ address: '8.8.8.8', ttl: 300 }] as never)
    f.api.replaceNamespacedNetworkPolicy.mockImplementationOnce(async () => {
      // A concurrent deletion wins during the first PUT. The retry GET must
      // observe absence and fail admission, rather than report success.
      f.policies.delete(currentName)
      throw apiError(409)
    })

    await expect(f.run()).rejects.toThrow()

    expect(f.policies.has(currentName)).toBe(false)
    expect(f.condition()).toMatchObject({
      status: 'False',
      reason: 'ExternalEgressReconcileFailed',
    })
    expect(f.status().resolvedEgressIPs).toEqual([])
    await f.run()
    expect(f.condition()).toMatchObject({ status: 'True', reason: 'Reconciled' })
    expect(f.policies.get(currentName)?.spec?.egress).toEqual([
      { to: [{ ipBlock: { cidr: '8.8.8.8/32' } }], ports: [{ port: 443, protocol: 'TCP' }] },
    ])
    expect(f.status().resolvedEgressIPs?.[0].ips).toEqual(['8.8.8.8'])
  })

  it('keeps the existing retry coordinator pending until the API recovers and then admits the same generation once', async () => {
    vi.useFakeTimers()
    const f = fixture([currentDns])
    await f.run()
    f.unavailableReads.add(currentName)
    const admitted = vi.fn()
    let coordinator: ExternalEgressConvergenceCoordinator
    coordinator = new ExternalEgressConvergenceCoordinator({
      listServers: () => [f.server],
      getCurrentServer: () => f.server,
      inventoryAuthoritative: () => true,
      sameDesiredRevision: (left, right) =>
        left.uid === right.uid && left.generation === right.generation,
      enqueue: async (_server, work) => work(),
      mutate: async (_type, server, options) =>
        f.reconciler.reconcileExternalEgress(server, options),
      replay: async (type, server, retry) => {
        const completion = await coordinator.reconcile(type, server, { retry })
        admitted(server.generation)
        completion?.complete()
      },
      externalEgressRefreshMinTtlMs: () => f.reconciler.externalEgressRefreshMinTtlMs,
    })
    try {
      const gate = coordinator.prepareStartupGates([f.server])
      expect(await gate.waitFor(f.server)).toBe(false)
      await vi.advanceTimersByTimeAsync(5000)
      expect(admitted).not.toHaveBeenCalled()
      expect(f.condition()?.status).toBe('False')
      f.unavailableReads.clear()
      await vi.advanceTimersByTimeAsync(15000)
      expect(admitted).toHaveBeenCalledExactlyOnceWith(7)
      expect(f.condition()?.status).toBe('True')
      await vi.advanceTimersByTimeAsync(60000)
      expect(admitted).toHaveBeenCalledTimes(1)
    } finally {
      coordinator.stop()
    }
  })
})