Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
import { beforeEach, describe, expect, it, vi } from 'vitest'
import type * as k8s from '@kubernetes/client-node'
import { asApiserverDeployment } from '../__tests__/asApiserverDeployment'
import liveReaderFixture from '../__tests__/fixtures/629/gfsc-reader.deployment.json'
import livePdbFixture from '../__tests__/fixtures/629/gfsc-writer-pdb.json'
import { createsTotal, existenceReadsTotal, writeSkipsTotal } from '../metrics'
import { deploymentMatchesDesired, podDisruptionBudgetMatchesDesired } from '../utils'
import { type GfsFactoryConfig, buildDeployment } from './gfsFactory'
import { K8sGfsApi } from './gfsK8sApi'
const namespace = 'mcp-host'
const metadata = {
name: 'gfs-resource',
namespace,
labels: { 'app.kubernetes.io/managed-by': 'host-context-controller' },
}
const cases = [
{
kind: 'PersistentVolumeClaim',
updates: false,
body: {
kind: 'PersistentVolumeClaim',
metadata,
spec: { accessModes: ['ReadWriteOnce'], resources: { requests: { storage: '1Gi' } } },
},
apply: (api: K8sGfsApi, body: k8s.V1PersistentVolumeClaim) => api.applyPvc(body, namespace),
},
{
kind: 'Service',
updates: false,
body: {
kind: 'Service',
metadata,
spec: { selector: { app: 'gfs' }, ports: [{ port: 8080 }] },
},
apply: (api: K8sGfsApi, body: k8s.V1Service) => api.applyService(body, namespace),
},
{
kind: 'Deployment',
updates: false,
body: {
kind: 'Deployment',
metadata,
spec: {
selector: { matchLabels: { app: 'gfs' } },
template: {
metadata: { labels: { app: 'gfs' } },
spec: { containers: [{ name: 'gfs', image: 'example/gfs:1' }] },
},
},
},
apply: (api: K8sGfsApi, body: k8s.V1Deployment) => api.applyDeployment(body, namespace),
},
{
kind: 'PodDisruptionBudget',
updates: false,
body: {
kind: 'PodDisruptionBudget',
metadata,
spec: { minAvailable: 1, selector: { matchLabels: { app: 'gfs' } } },
},
apply: (api: K8sGfsApi, body: k8s.V1PodDisruptionBudget) =>
api.applyPodDisruptionBudget(body, namespace),
},
] as const
// The SDK mock boundary records real adapter calls, not a fake GfsK8sApi implementation.
function harness(kind: string) {
const events: string[] = []
const read = vi.fn(async (): Promise<unknown> => {
events.push('GET')
throw new Error('Arrange the SDK read outcome for this scenario')
})
const create = vi.fn(async (): Promise<unknown> => {
events.push('POST')
return {}
})
const replace = vi.fn(async ({ body }: { body: { metadata?: { resourceVersion?: string } } }) => {
events.push(`PUT:${body.metadata?.resourceVersion}`)
return body
})
const sdk = {
[`readNamespaced${kind}`]: read,
[`createNamespaced${kind}`]: create,
[`replaceNamespaced${kind}`]: replace,
}
const api = new K8sGfsApi(
sdk as unknown as k8s.CoreV1Api,
sdk as unknown as k8s.AppsV1Api,
sdk as unknown as k8s.NetworkingV1Api,
sdk as unknown as k8s.PolicyV1Api,
{} as k8s.CustomObjectsApi
)
return { api, read, create, replace, events }
}
async function count(metric: typeof createsTotal, kind: string, outcome: string) {
return (
(await metric.get()).values.find(
row => row.labels.kind === kind && row.labels.outcome === outcome
)?.value ?? 0
)
}
beforeEach(() => {
createsTotal.reset()
existenceReadsTotal.reset()
writeSkipsTotal.reset()
})
describe.each(cases)('K8sGfsApi $kind read-first', resource => {
// This union is passed only to its matching case adapter above.
const apply = (api: K8sGfsApi) => resource.apply(api, resource.body as never)
const live = (version = '1') => ({
...resource.body,
metadata: { ...metadata, resourceVersion: version },
})
it('preserves or converges an existing object without POST or a duplicate GET', async () => {
const h = harness(resource.kind)
h.read.mockImplementation(async () => {
h.events.push('GET')
return live()
})
await apply(h.api)
expect(h.events).toEqual(resource.updates ? ['GET', 'PUT:1'] : ['GET'])
expect(h.read).toHaveBeenCalledExactlyOnceWith({ name: metadata.name, namespace })
expect(h.create).not.toHaveBeenCalled()
expect(h.replace).toHaveBeenCalledTimes(resource.updates ? 1 : 0)
if (resource.updates)
expect(h.replace).toHaveBeenCalledWith({ name: metadata.name, namespace, body: live() })
expect(await count(createsTotal, resource.kind, 'skipped')).toBe(1)
expect(await count(existenceReadsTotal, resource.kind, 'found')).toBe(1)
})
it('creates an absent object after its single existence GET', async () => {
const h = harness(resource.kind)
h.read.mockImplementationOnce(async () => {
h.events.push('GET')
throw { code: 404 }
})
await apply(h.api)
expect(h.events).toEqual(['GET', 'POST'])
expect(h.create).toHaveBeenCalledExactlyOnceWith({ namespace, body: resource.body })
expect(h.replace).not.toHaveBeenCalled()
expect(await count(createsTotal, resource.kind, 'created')).toBe(1)
expect(await count(createsTotal, resource.kind, 'skipped')).toBe(0)
expect(await count(existenceReadsTotal, resource.kind, 'absent')).toBe(1)
})
it.each([{ code: 403 }, { response: { statusCode: 500 } }, new Error('transport failure')])(
'propagates initial GET failure without a write: %j',
async error => {
const h = harness(resource.kind)
h.read.mockRejectedValueOnce(error)
await expect(apply(h.api)).rejects.toBe(error)
expect(h.read).toHaveBeenCalledTimes(1)
expect(h.create).not.toHaveBeenCalled()
expect(h.replace).not.toHaveBeenCalled()
expect(await count(existenceReadsTotal, resource.kind, 'error')).toBe(1)
expect(await count(createsTotal, resource.kind, 'skipped')).toBe(0)
}
)
it('rereads after absence races with POST409, without counting a suppressed create', async () => {
const h = harness(resource.kind)
h.read
.mockImplementationOnce(async () => {
h.events.push('GET')
throw { code: 404 }
})
.mockImplementationOnce(async () => {
h.events.push('GET')
return live('2')
})
h.create.mockImplementationOnce(async () => {
h.events.push('POST')
throw { code: 409 }
})
await apply(h.api)
expect(h.events).toEqual(
resource.updates ? ['GET', 'POST', 'GET', 'PUT:2'] : ['GET', 'POST', 'GET']
)
expect(h.create).toHaveBeenCalledTimes(1)
expect(h.read).toHaveBeenCalledTimes(2)
expect(await count(createsTotal, resource.kind, 'conflict')).toBe(1)
expect(await count(createsTotal, resource.kind, 'skipped')).toBe(0)
})
it('propagates the fresh GET403 after POST409', async () => {
const h = harness(resource.kind)
const denied = { code: 403 }
h.read.mockRejectedValueOnce({ code: 404 }).mockRejectedValueOnce(denied)
h.create.mockRejectedValueOnce({ code: 409 })
await expect(apply(h.api)).rejects.toBe(denied)
expect(h.read).toHaveBeenCalledTimes(2)
expect(h.create).toHaveBeenCalledTimes(1)
expect(h.replace).not.toHaveBeenCalled()
expect(await count(createsTotal, resource.kind, 'skipped')).toBe(0)
})
it('preserves benign disappearance after a create conflict', async () => {
const h = harness(resource.kind)
h.read.mockRejectedValueOnce({ code: 404 }).mockRejectedValueOnce({ code: 404 })
h.create.mockRejectedValueOnce({ code: 409 })
await expect(apply(h.api)).resolves.toBe('missing')
expect(h.read).toHaveBeenCalledTimes(2)
expect(h.create).toHaveBeenCalledTimes(1)
expect(h.replace).not.toHaveBeenCalled()
expect(await count(createsTotal, resource.kind, 'skipped')).toBe(0)
})
it('preserves a non-conflict POST failure without convergence', async () => {
const h = harness(resource.kind)
h.read.mockRejectedValueOnce({ code: 404 })
const denied = { code: 403 }
h.create.mockRejectedValueOnce(denied)
await expect(apply(h.api)).rejects.toBe(denied)
expect(h.read).toHaveBeenCalledTimes(1)
expect(h.create).toHaveBeenCalledTimes(1)
expect(h.replace).not.toHaveBeenCalled()
expect(await count(createsTotal, resource.kind, 'error')).toBe(1)
})
})
describe.each(cases.filter(resource => resource.updates))(
'K8sGfsApi $kind PUT conflicts',
resource => {
const apply = (api: K8sGfsApi) => resource.apply(api, resource.body as never)
it('uses the cached first read once, then fresh resourceVersion for the retry', async () => {
const h = harness(resource.kind)
let version = 0
h.read.mockImplementation(async () => {
h.events.push('GET')
return { ...resource.body, metadata: { ...metadata, resourceVersion: String(++version) } }
})
h.replace.mockImplementationOnce(async ({ body }) => {
h.events.push(`PUT:${body.metadata?.resourceVersion}`)
throw { code: 409 }
})
await apply(h.api)
expect(h.events).toEqual(['GET', 'PUT:1', 'GET', 'PUT:2'])
expect(h.create).not.toHaveBeenCalled()
expect(await count(existenceReadsTotal, resource.kind, 'found')).toBe(2)
expect(await count(createsTotal, resource.kind, 'skipped')).toBe(1)
})
it('retains the three-attempt PUT budget and propagates exhaustion', async () => {
const h = harness(resource.kind)
h.read.mockResolvedValue({
...resource.body,
metadata: { ...metadata, resourceVersion: '1' },
})
const conflict = { code: 409 }
h.replace.mockRejectedValue(conflict)
await expect(apply(h.api)).rejects.toBe(conflict)
expect(h.read).toHaveBeenCalledTimes(3)
expect(h.replace).toHaveBeenCalledTimes(3)
expect(h.create).not.toHaveBeenCalled()
expect(await count(createsTotal, resource.kind, 'skipped')).toBe(0)
})
}
)
describe('K8sGfsApi PodDisruptionBudget no-op gate (T4)', () => {
const namespace = 'gfs'
const desired = livePdbFixture as k8s.V1PodDisruptionBudget
async function skipCount() {
return (
(await writeSkipsTotal.get()).values.find(row => row.labels.kind === 'PodDisruptionBudget')
?.value ?? 0
)
}
it('T4: skips replace when the live PDB matches', async () => {
const equal = harness('PodDisruptionBudget')
equal.read.mockImplementation(async () => {
equal.events.push('GET')
return {
...desired,
status: {
currentHealthy: 1,
desiredHealthy: 1,
disruptionsAllowed: 0,
expectedPods: 1,
},
metadata: {
...desired.metadata,
resourceVersion: '1',
uid: 'pdb-uid',
creationTimestamp: new Date('2026-01-01T00:00:00Z'),
managedFields: [{ manager: 'kube-apiserver', operation: 'Update' }],
},
}
})
await equal.api.applyPodDisruptionBudget(desired, namespace)
expect(equal.read).toHaveBeenCalledTimes(1)
expect(equal.replace).toHaveBeenCalledTimes(0)
expect(await skipCount()).toBe(1)
})
it('T4: retries a drifted PDB PUT after 409 using a fresh resourceVersion', async () => {
const h = harness('PodDisruptionBudget')
let version = 0
h.read.mockImplementation(async () => {
h.events.push('GET')
return {
...desired,
metadata: { ...desired.metadata, resourceVersion: String(++version) },
spec: { ...desired.spec, minAvailable: 2 },
}
})
h.replace.mockImplementationOnce(async ({ body }) => {
h.events.push(`PUT:${body.metadata?.resourceVersion}`)
throw { code: 409 }
})
await h.api.applyPodDisruptionBudget(desired, namespace)
expect(h.events).toEqual(['GET', 'PUT:1', 'GET', 'PUT:2'])
expect(h.replace).toHaveBeenCalledTimes(2)
expect(h.create).not.toHaveBeenCalled()
})
it('T4: writes when minAvailable differs', async () => {
const drifted = harness('PodDisruptionBudget')
const live = {
...desired,
metadata: { ...desired.metadata, resourceVersion: '7' },
spec: { ...desired.spec, minAvailable: 2 },
}
drifted.read.mockImplementation(async () => {
drifted.events.push('GET')
return live
})
await drifted.api.applyPodDisruptionBudget(desired, namespace)
expect(drifted.read).toHaveBeenCalledTimes(1)
expect(drifted.replace).toHaveBeenCalledTimes(1)
const replaced = drifted.replace.mock.calls[0][0].body as k8s.V1PodDisruptionBudget
expect(replaced.spec?.minAvailable).toBe(1)
expect(await skipCount()).toBe(0)
})
it('T4: a live unhealthyPodEvictionPolicy is drift', () => {
const live = {
...desired,
spec: { ...desired.spec, unhealthyPodEvictionPolicy: 'AlwaysAllow' as const },
}
expect(podDisruptionBudgetMatchesDesired(desired, live)).toBe(false)
})
})
const REVISION_ANNOTATION = 'deployment.kubernetes.io/revision'
const RESTARTED_AT = 'kubectl.kubernetes.io/restartedAt'
const FOREIGN_TEMPLATE_ANNOTATION = 'container.apparmor.security.beta.kubernetes.io/gfsc'
const readerProducerConfig: GfsFactoryConfig = {
gfsNamespace: 'gfs',
controlPlaneNamespace: 'control-plane',
postgresPodLabels: { app: 'control-postgres' },
postgresPort: 5432,
gfscImage: 'clerum-gfs-controller:test',
gfscImagePullPolicy: 'IfNotPresent',
gfscPort: 8087,
gfscInitImage: 'busybox:1.36',
gfscResources: {
requests: { memory: '128Mi', cpu: '100m' },
limits: { memory: '256Mi', cpu: '500m' },
},
jwtPublicKeyConfigMapName: 'gfs-config',
jwtPublicKeyConfigMapKey: 'jwt-public-key',
pgSecretName: 'gfs-controller-db',
pgSecretKey: 'connection-string',
readerPgSecretName: 'gfs-controller-reader-db',
readerPgSecretKey: 'connection-string',
driveName: 'main',
tokenAudience: 'gfs-controller',
agentReadRlPerMinPerReplica: 300,
agentWriteRlPerMinPerReplica: 120,
}
function producedReader(): k8s.V1Deployment {
return buildDeployment(
{ name: 'gfs', namespace: 'gfs', spec: {} },
readerProducerConfig,
'reader'
)
}
/**
* Synthetic twin of the captured fixture (revision stripped). These cases
* prove annotation merge and fail-open replica drift. They do not put
* `buildDeployment` on the desired side — see the producer+defaulting cases.
*/
function desiredReaderFromFixture(): k8s.V1Deployment {
const desired = structuredClone(liveReaderFixture) as k8s.V1Deployment
if (desired.metadata?.annotations) {
delete desired.metadata.annotations[REVISION_ANNOTATION]
}
return desired
}
function liveReader(overrides: Partial<k8s.V1DeploymentSpec> = {}): k8s.V1Deployment {
const live = structuredClone(liveReaderFixture) as k8s.V1Deployment
const selector = overrides.selector ?? live.spec?.selector
const template = overrides.template ?? live.spec?.template
if (!selector || !template) {
throw new Error('gfsc-reader fixture is missing spec.selector or spec.template')
}
const spec: k8s.V1DeploymentSpec = {
...live.spec,
...overrides,
selector,
template,
}
return {
...live,
metadata: {
...live.metadata,
resourceVersion: '1',
uid: 'reader-uid',
creationTimestamp: new Date('2026-01-01T00:00:00Z'),
managedFields: [{ manager: 'kube-controller-manager', operation: 'Update' }],
},
spec,
status: { observedGeneration: 1, availableReplicas: spec.replicas ?? 2 },
}
}
describe('K8sGfsApi Deployment no-op gate (T3, T8)', () => {
const namespace = 'gfs'
async function deploymentSkipCount() {
return (
(await writeSkipsTotal.get()).values.find(row => row.labels.kind === 'Deployment')?.value ?? 0
)
}
it('T3: the captured fixture is synthetic and does not match buildDeployment', () => {
expect(deploymentMatchesDesired(producedReader(), liveReaderFixture as k8s.V1Deployment)).toBe(
false
)
})
it('T3: skips replace when live is buildDeployment after apiserver defaulting', async () => {
const desired = producedReader()
const live = asApiserverDeployment(desired)
const templateMeta = live.spec?.template?.metadata
if (!templateMeta) throw new Error('expected pod template metadata')
templateMeta.annotations = {
...templateMeta.annotations,
[RESTARTED_AT]: '2026-09-16T20:33:45Z',
}
const equal = harness('Deployment')
equal.read.mockImplementation(async () => {
equal.events.push('GET')
return live
})
await equal.api.applyDeployment(desired, namespace)
expect(equal.read).toHaveBeenCalledTimes(1)
expect(equal.replace).toHaveBeenCalledTimes(0)
expect(await deploymentSkipCount()).toBe(1)
})
it('T3: writes when live has a default-filled field the normalizer does not strip', async () => {
const desired = producedReader()
const live = asApiserverDeployment(desired)
if (!live.spec) throw new Error('expected live spec')
live.spec.minReadySeconds = 7
const drifted = harness('Deployment')
drifted.read.mockImplementation(async () => {
drifted.events.push('GET')
return live
})
await drifted.api.applyDeployment(desired, namespace)
expect(drifted.read).toHaveBeenCalledTimes(1)
expect(drifted.replace).toHaveBeenCalledTimes(1)
expect(await deploymentSkipCount()).toBe(0)
})
it('T3: skips replace when the live reader matches after revision merge', async () => {
const desired = desiredReaderFromFixture()
const equal = harness('Deployment')
equal.read.mockImplementation(async () => {
equal.events.push('GET')
return liveReader()
})
await equal.api.applyDeployment(desired, namespace)
expect(equal.read).toHaveBeenCalledTimes(1)
expect(equal.replace).toHaveBeenCalledTimes(0)
expect(await deploymentSkipCount()).toBe(1)
})
it('T3: writes when live replicas are 0 and restores the desired replica count', async () => {
const desired = desiredReaderFromFixture()
const scaled = harness('Deployment')
scaled.read.mockImplementation(async () => {
scaled.events.push('GET')
return liveReader({ replicas: 0 })
})
await scaled.api.applyDeployment(desired, namespace)
expect(scaled.read).toHaveBeenCalledTimes(1)
expect(scaled.replace).toHaveBeenCalledTimes(1)
const replaced = scaled.replace.mock.calls[0][0].body as k8s.V1Deployment
expect(replaced.spec?.replicas).toBe(desired.spec?.replicas)
expect(await deploymentSkipCount()).toBe(0)
})
it('M1: a foreign pod-template annotation is stripped and forces a replace', async () => {
const desired = producedReader()
const live = asApiserverDeployment(desired)
const templateMeta = live.spec?.template?.metadata
if (!templateMeta) throw new Error('expected pod template metadata')
templateMeta.annotations = {
...templateMeta.annotations,
[RESTARTED_AT]: '2026-09-16T20:33:45Z',
[FOREIGN_TEMPLATE_ANNOTATION]: 'unconfined',
}
const drifted = harness('Deployment')
drifted.read.mockImplementation(async () => {
drifted.events.push('GET')
return live
})
await drifted.api.applyDeployment(desired, namespace)
expect(drifted.replace).toHaveBeenCalledTimes(1)
const replaced = drifted.replace.mock.calls[0][0].body as k8s.V1Deployment
const written = replaced.spec?.template?.metadata?.annotations ?? {}
expect(written[FOREIGN_TEMPLATE_ANNOTATION]).toBeUndefined()
expect(written[RESTARTED_AT]).toBe('2026-09-16T20:33:45Z')
expect(written['clerum.io/gfsc-template-hash']).toBe(
desired.spec?.template?.metadata?.annotations?.['clerum.io/gfsc-template-hash']
)
expect(await deploymentSkipCount()).toBe(0)
})
it('M1: an object-level foreign annotation still skips the replace', async () => {
const desired = producedReader()
const live = asApiserverDeployment(desired)
live.metadata = {
...live.metadata,
annotations: {
...live.metadata?.annotations,
'kubectl.kubernetes.io/last-applied-configuration': '{}',
},
}
const equal = harness('Deployment')
equal.read.mockImplementation(async () => {
equal.events.push('GET')
return live
})
await equal.api.applyDeployment(desired, namespace)
expect(equal.replace).toHaveBeenCalledTimes(0)
expect(await deploymentSkipCount()).toBe(1)
})
it('T8: skips replace when live template has restartedAt and increments writeSkipsTotal', async () => {
const desired = desiredReaderFromFixture()
const restarted = harness('Deployment')
const live = liveReader()
const templateMeta = live.spec?.template?.metadata
if (!templateMeta) throw new Error('expected pod template metadata')
templateMeta.annotations = {
...templateMeta.annotations,
[RESTARTED_AT]: '2026-09-16T20:33:45Z',
}
restarted.read.mockImplementation(async () => {
restarted.events.push('GET')
return live
})
await restarted.api.applyDeployment(desired, namespace)
expect(restarted.read).toHaveBeenCalledTimes(1)
expect(restarted.replace).toHaveBeenCalledTimes(0)
expect(await deploymentSkipCount()).toBe(1)
})
it('T3: retries a drifted reader PUT after 409 using a fresh resourceVersion', async () => {
const desired = desiredReaderFromFixture()
const h = harness('Deployment')
let version = 0
h.read.mockImplementation(async () => {
h.events.push('GET')
const live = liveReader({ replicas: 0 })
live.metadata = { ...live.metadata, resourceVersion: String(++version) }
return live
})
h.replace.mockImplementationOnce(async ({ body }) => {
h.events.push(`PUT:${body.metadata?.resourceVersion}`)
throw { code: 409 }
})
await h.api.applyDeployment(desired, namespace)
expect(h.events).toEqual(['GET', 'PUT:1', 'GET', 'PUT:2'])
expect(h.replace).toHaveBeenCalledTimes(2)
expect(h.create).not.toHaveBeenCalled()
})
})