Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
import { beforeEach, describe, expect, it, vi } from 'vitest'
import type * as k8s from '@kubernetes/client-node'
import { readFileSync, readdirSync } from 'node:fs'
import { join } from 'node:path'
import ts from 'typescript'
import { CREATE_KINDS, createsTotal, existenceReadsTotal, registry } from './metrics'
import { applyNetworkPolicy, observeExistenceRead } from './utils'
async function count(kind: string, outcome: string): Promise<number> {
const sample = (await existenceReadsTotal.get()).values.find(
value => value.labels.kind === kind && value.labels.outcome === outcome
)
expect(sample).toBeDefined()
return sample!.value
}
it('initializes the real read registry before any test seeding', async () => {
expect(registry.getSingleMetric('clerum_hcc_existence_reads_total')).toBe(existenceReadsTotal)
const snapshot = await existenceReadsTotal.get()
expect(snapshot.values).toHaveLength(CREATE_KINDS.length * 3)
for (const kind of CREATE_KINDS) {
for (const outcome of ['found', 'absent', 'error']) expect(await count(kind, outcome)).toBe(0)
}
for (const sample of snapshot.values)
expect(Object.keys(sample.labels).sort()).toEqual(['kind', 'outcome'])
})
// Review each exclusion when its enclosing operation changes. Counts describe
// expressions, not executions; a newly added direct read must be classified.
const readExclusions: Record<string, readonly [number, string]> = {
'hostReconciler.ts::readHostDeploymentOrNull::readNamespacedDeployment': [
1,
'Input to runtime binding and refresh decisions',
],
'hostReconciler.ts::refreshCodexSnapshot::readNamespacedConfigMap': [
1,
'Input allowlist snapshot',
],
'hostReconciler.ts::deleteHostRbac::readNamespacedRoleBinding': [1, 'Ownership before deletion'],
'hostReconciler.ts::deleteHostRbac::readNamespacedRole': [1, 'Ownership before deletion'],
'hostReconciler.ts::deleteHostRbac::readNamespacedServiceAccount': [
1,
'Ownership before deletion',
],
'hostReconciler.ts::deleteMcpHostRuntimeTokenSecret::readNamespacedSecret': [
1,
'Ownership before deletion',
],
'hostReconciler.ts::deleteLegacyChannelReaderRuntimeAuth::readNamespacedSecret': [
1,
'Ownership before deletion',
],
'hostReconciler.ts::validateHostSecret::readNamespacedSecret': [1, 'Input validation'],
'hostReconciler.ts::computeChannelReaderRevisionForHost::readNamespacedSecret': [
1,
'Input to desired revision',
],
'hostReconciler.ts::checkDeploymentReady::readNamespacedDeployment': [1, 'Readiness observation'],
'hostReconciler.ts::checkChannelReaderStatus::readNamespacedDeployment': [
1,
'Status observation',
],
'hostReconciler.ts::deleteRuntimeResources::readNamespacedDeployment': [
1,
'Ownership before deletion',
],
'hostReconciler.ts::deleteRuntimeResources::readNamespacedService': [
1,
'Ownership before deletion',
],
'hostReconciler.ts::deleteRuntimeResources::readNamespacedNetworkPolicy': [
1,
'Ownership before deletion',
],
'hostReconciler.ts::deleteWorkspacePvc::readNamespacedPersistentVolumeClaim': [
1,
'Ownership before deletion',
],
'hostReconciler.ts::deleteChannelReaderDeployment::readNamespacedDeployment': [
1,
'Ownership before deletion',
],
'hostReconciler.ts::deleteChannelReaderService::readNamespacedService': [
1,
'Ownership before deletion',
],
'hostReconciler.ts::deleteMcpHostCodexProxyEgressNetworkPolicy::readNamespacedNetworkPolicy': [
1,
'Ownership before deletion',
],
'hostReconciler.ts::deleteMcpHostGrokProxyEgressNetworkPolicy::readNamespacedNetworkPolicy': [
1,
'Ownership before deletion',
],
'hostReconciler.ts::deleteHostNetworkPolicies::readNamespacedNetworkPolicy': [
1,
'Ownership before deletion',
],
'k8sClient.ts::readSecretMetadata::readNamespacedSecret': [1, 'Authorization input metadata'],
'k8sClient.ts::readSecret::readNamespacedSecret': [1, 'Authorization input data'],
'llmHookReconciler.ts::validateSecret::readNamespacedSecret': [1, 'Input validation'],
'llmHookReconciler.ts::deleteServiceTargetNetworkPolicy::readNamespacedNetworkPolicy': [
1,
'Ownership before deletion',
],
'llmHookReconciler.ts::readServiceSelector::readNamespacedService': [
1,
'Input selector for another resource',
],
'llmHookReconciler.ts::deleteHostEgressNetworkPolicy::readNamespacedNetworkPolicy': [
1,
'Ownership before deletion',
],
'llmHookReconciler.ts::gcPodKey::readNamespacedDeployment': [
1,
'Ownership before garbage collection',
],
'llmHookReconciler.ts::gcPodKey::readNamespacedService': [
1,
'Ownership before garbage collection',
],
'llmHookReconciler.ts::gcPodKey::readNamespacedNetworkPolicy': [
1,
'Ownership before garbage collection',
],
'llmHookReconciler.ts::readDeploymentRollout::readNamespacedDeployment': [
1,
'Readiness observation',
],
'networkPolicyReconciler.ts::judgeLiveExactHostEgress::readNamespacedNetworkPolicy': [
1,
'Retain or revoke verdict after failure',
],
'networkPolicyReconciler.ts::replaceSafetyPolicySnapshot::readNamespacedNetworkPolicy': [
1,
'Identity check for replace-only safety path',
],
'networkPolicyReconciler.ts::deleteLegacyStaticPolicy::readNamespacedNetworkPolicy': [
1,
'Ownership before deletion',
],
'reconciler.ts::readDeploymentRollout::readNamespacedDeployment': [1, 'Readiness observation'],
'reconciler.ts::validateSecret::readNamespacedSecret': [1, 'Input validation'],
'reconciler.ts::deleteDeploymentIfHccOwned::readNamespacedDeployment': [
1,
'Ownership before deletion',
],
'reconciler.ts::deleteConfigMapIfHccOwned::readNamespacedConfigMap': [
1,
'Ownership before deletion',
],
'reconciler.ts::deleteServiceIfHccOwned::readNamespacedService': [1, 'Ownership before deletion'],
'sharedFileSystemReconciler.ts::assessReadiness::readNamespacedPersistentVolumeClaim': [
1,
'Readiness observation',
],
'sharedFileSystemReconciler.ts::assessReadiness::readNamespacedDeployment': [
1,
'Readiness observation',
],
'statelessLifecycleExecutor.ts::handleWakeFastPath::readNamespacedDeployment': [
1,
'Scale-only compatibility path',
],
'k8s/gfsK8sApi.ts::scaleDeployment::readNamespacedDeployment': [
2,
'Scale-only initial and retry reads',
],
'k8s/gfsK8sApi.ts::isDeploymentAvailable::readNamespacedDeployment': [1, 'Readiness observation'],
}
function readProductionSources(): Record<string, string> {
const sources: Record<string, string> = {}
for (const relative of readdirSync(__dirname, { recursive: true }) as string[]) {
if (
!relative.endsWith('.ts') ||
relative.endsWith('.test.ts') ||
relative.includes('__tests__')
)
continue
sources[relative] = readFileSync(join(__dirname, relative), 'utf8')
}
return sources
}
function assertReadInventory(sources: Record<string, string>): void {
const expectedWrapped: Record<string, number> = {
'utils.ts': 1,
'hostReconciler.ts': 9,
'reconciler.ts': 3,
'llmHookReconciler.ts': 3,
'sharedFileSystemReconciler.ts': 3,
'k8s/gfsK8sApi.ts': 5,
'networkPolicyReconciler.ts': 2,
}
const wrapped: Record<string, number> = {}
const excluded: Record<string, number> = {}
for (const [path, text] of Object.entries(sources)) {
const source = ts.createSourceFile(path, text, ts.ScriptTarget.Latest, true)
const visit = (node: ts.Node): void => {
if (
ts.isCallExpression(node) &&
ts.isIdentifier(node.expression) &&
node.expression.text === 'observeExistenceRead'
) {
wrapped[path] = (wrapped[path] ?? 0) + 1
expect(node.arguments).toHaveLength(2)
const [kind, read] = node.arguments
expect(ts.isStringLiteral(kind), path).toBe(true)
expect(ts.isArrowFunction(read), path).toBe(true)
if (ts.isStringLiteral(kind) && ts.isArrowFunction(read)) {
expect(CREATE_KINDS).toContain(kind.text)
expect(read.parameters).toHaveLength(0)
expect(ts.isCallExpression(read.body), path).toBe(true)
if (
ts.isCallExpression(read.body) &&
ts.isPropertyAccessExpression(read.body.expression)
) {
expect(read.body.expression.name.text, path).toBe(`readNamespaced${kind.text}`)
} else throw new Error(`${path}: wrapper must observe a direct API read`)
}
}
if (
ts.isCallExpression(node) &&
ts.isPropertyAccessExpression(node.expression) &&
/^read(?:Namespaced|Cluster)/.test(node.expression.name.text)
) {
const parent = node.parent
const isWrapped =
ts.isArrowFunction(parent) &&
ts.isCallExpression(parent.parent) &&
ts.isIdentifier(parent.parent.expression) &&
parent.parent.expression.text === 'observeExistenceRead'
if (!isWrapped) {
let owner: ts.Node | undefined = node.parent
while (owner && !ts.isMethodDeclaration(owner) && !ts.isFunctionDeclaration(owner))
owner = owner.parent
const name =
owner && (ts.isMethodDeclaration(owner) || ts.isFunctionDeclaration(owner))
? owner.name?.getText(source)
: undefined
const id = `${path}::${name ?? '<module>'}::${node.expression.name.text}`
if (!Object.hasOwn(readExclusions, id))
throw new Error(`Unclassified Kubernetes read: ${id}`)
excluded[id] = (excluded[id] ?? 0) + 1
}
}
ts.forEachChild(node, visit)
}
visit(source)
}
expect(wrapped).toEqual(expectedWrapped)
for (const [id, [count, reason]] of Object.entries(readExclusions)) {
expect(reason.length, id).toBeGreaterThan(0)
expect(excluded[id], `Stale or changed read exclusion: ${id}`).toBe(count)
}
expect(Object.values(wrapped).reduce((sum, count) => sum + count, 0)).toBe(26)
expect(Object.values(excluded).reduce((sum, count) => sum + count, 0)).toBe(44)
}
it('classifies every direct dot-property production SDK read as observed or explicitly excluded', () => {
assertReadInventory(readProductionSources())
})
it.each(['utils.ts', 'new-reader.ts'])('rejects an unclassified read introduced in %s', path => {
const sources = readProductionSources()
// Synthetic source is parsed only: no client method or network request executes.
sources[path] =
(sources[path] ?? '') +
'\nfunction unclassifiedRead(client) { return client.readNamespacedService({}) }'
expect(() => assertReadInventory(sources)).toThrow(
`Unclassified Kubernetes read: ${path}::unclassifiedRead::readNamespacedService`
)
})
it('rejects an exclusion after its read disappears or moves', () => {
const sources = readProductionSources()
delete sources['statelessLifecycleExecutor.ts']
expect(() => assertReadInventory(sources)).toThrow(
'Stale or changed read exclusion: statelessLifecycleExecutor.ts::handleWakeFastPath::readNamespacedDeployment'
)
})
describe('Kubernetes existence-read instrumentation', () => {
beforeEach(() => {
existenceReadsTotal.reset()
createsTotal.reset()
for (const kind of CREATE_KINDS) {
for (const outcome of ['found', 'absent', 'error'])
existenceReadsTotal.inc({ kind, outcome }, 0)
for (const outcome of ['created', 'conflict', 'error', 'skipped'])
createsTotal.inc({ kind, outcome }, 0)
}
})
it.each([{ metadata: { uid: 'existing', resourceVersion: '17' } }, null, undefined])(
'returns the exact resolved value with one read: %j',
async response => {
const read = vi.fn().mockResolvedValue(response)
expect(await observeExistenceRead('Service', read)).toBe(response)
expect(read).toHaveBeenCalledTimes(1)
expect(await count('Service', 'found')).toBe(1)
expect(await count('Service', 'absent')).toBe(0)
expect(await count('Service', 'error')).toBe(0)
expect((await createsTotal.get()).values.every(sample => sample.value === 0)).toBe(true)
}
)
// Minimal fixtures pin classification and rejection identity without constructing
// SDK errors: ApiException.code and getErrorCode's response.statusCode compatibility.
it.each([
[{ code: 404 }, 'absent'],
[{ response: { statusCode: 404 } }, 'absent'],
[{ code: 403 }, 'error'],
[{ response: { statusCode: 500 } }, 'error'],
[{ code: 409 }, 'error'],
[{ code: 'ECONNRESET' }, 'error'],
[null, 'error'],
[undefined, 'error'],
] as const)(
'preserves rejection identity and classifies only 404 as absent: %j',
async (error, outcome) => {
const read = vi.fn().mockRejectedValue(error)
await expect(observeExistenceRead('Service', read)).rejects.toBe(error)
expect(read).toHaveBeenCalledTimes(1)
for (const candidate of ['found', 'absent', 'error'])
expect(await count('Service', candidate)).toBe(candidate === outcome ? 1 : 0)
}
)
it.each([{ code: 404 }, new Error('request construction failed'), null, undefined])(
'preserves synchronous throws with one attempt: %j',
async error => {
const read = vi.fn(() => {
throw error
})
await expect(observeExistenceRead('Deployment', read)).rejects.toBe(error)
expect(read).toHaveBeenCalledTimes(1)
expect(await count('Deployment', error && 'code' in error ? 'absent' : 'error')).toBe(1)
}
)
it('preserves existing observations across module re-import', async () => {
const response = { metadata: { uid: 'observed' } }
const read = vi.fn().mockResolvedValue(response)
expect(await observeExistenceRead('Service', read)).toBe(response)
vi.resetModules()
const reloaded = await import('./metrics')
expect(reloaded.registry).toBe(registry)
expect(reloaded.existenceReadsTotal).toBe(existenceReadsTotal)
expect(read).toHaveBeenCalledTimes(1)
expect(await count('Service', 'found')).toBe(1)
expect(await registry.metrics()).toMatch(
/clerum_hcc_existence_reads_total\{kind="Service",outcome="found"[^}]*\} 1/
)
})
const policy: k8s.V1NetworkPolicy = {
kind: 'NetworkPolicy',
metadata: { name: 'read-metrics-policy' },
spec: { podSelector: {}, policyTypes: ['Ingress'], ingress: [] },
}
it('observes GET404 before POST for an absent policy', async () => {
const api = {
createNamespacedNetworkPolicy: vi.fn().mockResolvedValue(policy),
readNamespacedNetworkPolicy: vi.fn().mockRejectedValue({ code: 404 }),
}
await applyNetworkPolicy(
api as unknown as k8s.NetworkingV1Api,
'read-metrics-policy',
'test',
policy
)
expect(api.createNamespacedNetworkPolicy).toHaveBeenCalledExactlyOnceWith({
namespace: 'test',
body: policy,
})
expect(api.readNamespacedNetworkPolicy).toHaveBeenCalledExactlyOnceWith({
name: 'read-metrics-policy',
namespace: 'test',
})
expect(await count('NetworkPolicy', 'absent')).toBe(1)
expect(await count('NetworkPolicy', 'found')).toBe(0)
})
it('attributes GET conflicts to reads without adding POST conflicts or retrying', async () => {
const error = { response: { statusCode: 409 } }
const api = {
createNamespacedNetworkPolicy: vi.fn().mockRejectedValue({ code: 409 }),
readNamespacedNetworkPolicy: vi.fn().mockRejectedValue(error),
replaceNamespacedNetworkPolicy: vi.fn(),
}
await expect(
applyNetworkPolicy(
api as unknown as k8s.NetworkingV1Api,
'read-metrics-policy',
'test',
policy
)
).rejects.toBe(error)
expect(api.createNamespacedNetworkPolicy).not.toHaveBeenCalled()
expect(api.readNamespacedNetworkPolicy).toHaveBeenCalledExactlyOnceWith({
name: 'read-metrics-policy',
namespace: 'test',
})
expect(api.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()
expect(await count('NetworkPolicy', 'error')).toBe(1)
const samples = (await createsTotal.get()).values.filter(
sample => sample.labels.kind === 'NetworkPolicy'
)
expect(samples.map(sample => [sample.labels.outcome, sample.value])).toEqual([
['created', 0],
['conflict', 0],
['error', 0],
['skipped', 0],
])
})
it('counts physical retry reads once and preserves fresh resource versions', async () => {
const events: string[] = []
let reads = 0
const api = {
createNamespacedNetworkPolicy: vi.fn(async () => {
events.push('POST')
throw { code: 409 }
}),
readNamespacedNetworkPolicy: vi.fn(async () => {
events.push('GET')
return {
...policy,
metadata: { ...policy.metadata, resourceVersion: String(++reads) },
spec: { ...policy.spec, ingress: [{}] },
}
}),
replaceNamespacedNetworkPolicy: vi.fn(async ({ body }: { body: k8s.V1NetworkPolicy }) => {
events.push('PUT:' + body.metadata?.resourceVersion)
if (body.metadata?.resourceVersion === '1') throw { code: 409 }
return body
}),
}
await applyNetworkPolicy(
api as unknown as k8s.NetworkingV1Api,
'read-metrics-policy',
'test',
policy
)
expect(events).toEqual(['GET', 'PUT:1', 'GET', 'PUT:2'])
expect(api.readNamespacedNetworkPolicy).toHaveBeenCalledTimes(2)
expect(api.replaceNamespacedNetworkPolicy).toHaveBeenCalledTimes(2)
expect(await count('NetworkPolicy', 'found')).toBe(2)
expect(await count('NetworkPolicy', 'error')).toBe(0)
const conflicts = (await createsTotal.get()).values.find(
sample => sample.labels.kind === 'NetworkPolicy' && sample.labels.outcome === 'conflict'
)
expect(conflicts?.value).toBe(0)
expect(api.createNamespacedNetworkPolicy).not.toHaveBeenCalled()
const skipped = (await createsTotal.get()).values.find(
sample => sample.labels.kind === 'NetworkPolicy' && sample.labels.outcome === 'skipped'
)
expect(skipped?.value).toBe(1)
})
it('observes a read that expires the existing fence and prevents replace', async () => {
let active = true
const allowed = vi.fn(() => active)
const api = {
createNamespacedNetworkPolicy: vi.fn().mockRejectedValue({ code: 409 }),
readNamespacedNetworkPolicy: vi.fn(async () => {
active = false
return { ...policy, spec: { ...policy.spec, ingress: [{}] } }
}),
replaceNamespacedNetworkPolicy: vi.fn(),
}
await applyNetworkPolicy(
api as unknown as k8s.NetworkingV1Api,
'read-metrics-policy',
'test',
policy,
'[test]',
allowed
)
expect(allowed.mock.results.map(result => result.value)).toEqual([true, false])
expect(api.createNamespacedNetworkPolicy).not.toHaveBeenCalled()
expect(api.readNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)
expect(await count('NetworkPolicy', 'found')).toBe(1)
expect(api.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()
})
})