Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
import { describe, expect, it, vi } from 'vitest'
import {
BoundedInfrastructureTelemetryReporter,
createInfrastructureTelemetryReporter,
hccReconcileOutcomeSourceId,
} from './infrastructureTelemetryReporter'
import {
infrastructureTelemetryFlushesTotal,
infrastructureTelemetryGapsTotal,
infrastructureTelemetryRetriesTotal,
} from './metrics'
/** control-api refuses a reference without one, so every fixture carries it. */
const HOST_UID = '6f1c2f3a-2f4b-4d3a-9b2e-7c0d1a5e8b44'
const projection = {
sourceEventId: 'hcc-health-transition:mcp-host:chatllm:7:active:1',
occurredAt: '2026-07-11T12:00:00.000Z',
hostLookupReference: { name: 'chatllm', namespace: 'mcp-host', generation: 7, uid: HOST_UID },
payload: { transition: 'lifecycle:active', state: 'active' },
} as const
const reconcileProjection = {
occurredAt: '2026-07-11T12:00:01.000Z',
telemetryType: 'reconcile_outcome',
hostLookupReference: { name: 'chatllm', namespace: 'mcp-host', generation: 7, uid: HOST_UID },
payload: {
resource_class: 'Host',
reason_code: 'ready',
status: 'succeeded',
phase: 'deployed',
state: 'ready',
},
} as const
const reconcileSourceEventId = hccReconcileOutcomeSourceId(reconcileProjection)
describe('BoundedInfrastructureTelemetryReporter', () => {
it('does not instantiate when governed tracing is disabled', () => {
const reporter = createInfrastructureTelemetryReporter(false, {
baseUrl: 'http://control-api.test:8090',
})
expect(reporter).toBeUndefined()
})
it('instantiates when governed tracing is enabled', () => {
const reporter = createInfrastructureTelemetryReporter(true, {
baseUrl: 'http://control-api.test:8090',
})
expect(reporter).toBeInstanceOf(BoundedInfrastructureTelemetryReporter)
})
it('keeps enqueue synchronous and submits the health projection off-path', async () => {
const fetchFn = vi.fn().mockResolvedValue({ ok: true }) as unknown as typeof fetch
const reporter = new BoundedInfrastructureTelemetryReporter({
baseUrl: 'http://control-api.test:8090/',
signToken: () => 'signed-request',
fetchFn,
})
reporter.enqueueHealthTransition(projection)
expect(fetchFn).not.toHaveBeenCalled()
await new Promise(resolve => setTimeout(resolve, 0))
expect(fetchFn).toHaveBeenCalledOnce()
const [url, init] = vi.mocked(fetchFn).mock.calls[0]!
expect(url).toBe(
'http://control-api.test:8090/api/v1/internal/tracing/infrastructure-telemetry-events'
)
expect(init?.headers).toMatchObject({ Authorization: 'Bearer signed-request' })
expect(JSON.parse(String(init?.body))).toEqual({
events: [{ ...projection, telemetryType: 'health_transition' }],
})
})
it('submits Host-backed reconcile telemetry with server-observed lookup reference', async () => {
const fetchFn = vi.fn().mockResolvedValue({ ok: true }) as unknown as typeof fetch
const reporter = new BoundedInfrastructureTelemetryReporter({
baseUrl: 'http://control-api.test:8090',
signToken: () => 'signed-request',
fetchFn,
})
reporter.enqueue(reconcileProjection)
await new Promise(resolve => setTimeout(resolve, 0))
expect(JSON.parse(String(vi.mocked(fetchFn).mock.calls[0]![1]?.body))).toEqual({
events: [{ ...reconcileProjection, sourceEventId: reconcileSourceEventId }],
})
})
it('derives reconcile identity only after enqueue returns', async () => {
const fetchFn = vi.fn().mockResolvedValue({ ok: true }) as unknown as typeof fetch
const deriveReconcileSourceEventId = vi.fn(hccReconcileOutcomeSourceId)
const reporter = new BoundedInfrastructureTelemetryReporter({
baseUrl: 'http://control-api.test:8090',
signToken: () => 'signed-request',
fetchFn,
deriveReconcileSourceEventId,
})
reporter.enqueue(reconcileProjection)
expect(deriveReconcileSourceEventId).not.toHaveBeenCalled()
expect(fetchFn).not.toHaveBeenCalled()
await new Promise(resolve => setTimeout(resolve, 0))
expect(deriveReconcileSourceEventId).toHaveBeenCalledOnce()
expect(fetchFn).toHaveBeenCalledOnce()
})
it('derives reconcile identity once and reuses it across transport retries', async () => {
infrastructureTelemetryRetriesTotal.reset()
const fetchFn = vi
.fn()
.mockResolvedValueOnce({ ok: false, status: 503 })
.mockResolvedValueOnce({ ok: true }) as unknown as typeof fetch
const deriveReconcileSourceEventId = vi.fn(() => 'hcc-reconcile-outcome:stable')
const reporter = new BoundedInfrastructureTelemetryReporter({
baseUrl: 'http://control-api.test:8090',
signToken: () => 'signed-request',
fetchFn,
deriveReconcileSourceEventId,
retryLimit: 1,
random: () => 0,
})
reporter.enqueue(reconcileProjection)
await new Promise(resolve => setTimeout(resolve, 60))
expect(fetchFn).toHaveBeenCalledTimes(2)
expect(deriveReconcileSourceEventId).toHaveBeenCalledOnce()
expect((await infrastructureTelemetryRetriesTotal.get()).values[0]?.value).toBe(1)
const sourceIds = vi.mocked(fetchFn).mock.calls.map(call => {
const body = JSON.parse(String(call[1]?.body)) as { events: Array<{ sourceEventId: string }> }
return body.events[0]!.sourceEventId
})
expect(sourceIds).toEqual(['hcc-reconcile-outcome:stable', 'hcc-reconcile-outcome:stable'])
})
it('keeps reconcile identity stable for equivalent state and changes it with the outcome', () => {
expect(hccReconcileOutcomeSourceId(reconcileProjection)).toMatch(
/^hcc-reconcile-outcome-v3:[0-9a-f]{64}$/
)
expect(hccReconcileOutcomeSourceId(reconcileProjection)).toBe(
hccReconcileOutcomeSourceId({ ...reconcileProjection })
)
expect(hccReconcileOutcomeSourceId(reconcileProjection)).not.toBe(
hccReconcileOutcomeSourceId({
...reconcileProjection,
payload: {
...reconcileProjection.payload,
reason_code: 'not_ready',
status: 'failed',
},
})
)
})
it('gives a recreated Host object (same name and generation, new uid) new identities (#691)', () => {
const withUid = (uid: string) => ({
...reconcileProjection,
hostLookupReference: { ...reconcileProjection.hostLookupReference, uid },
})
// Liveness: the identity is deterministic for an identical tuple.
expect(hccReconcileOutcomeSourceId(withUid('uid-new'))).toBe(
hccReconcileOutcomeSourceId(withUid('uid-new'))
)
expect(hccReconcileOutcomeSourceId(withUid('uid-new'))).not.toBe(
hccReconcileOutcomeSourceId(withUid('uid-old'))
)
})
it('settles a 400 unsafe_tracing_input once and keeps retrying a 500 (#326, #328)', async () => {
infrastructureTelemetryFlushesTotal.reset()
infrastructureTelemetryGapsTotal.reset()
const response = (status: number, body: unknown) =>
({ ok: false, status, json: async () => body }) as unknown as Response
const fetchFn = vi.fn(async (_url: unknown, init?: RequestInit) => {
const event = (JSON.parse(String(init?.body)) as { events: Array<{ sourceEventId: string }> })
.events[0]!
return event.sourceEventId === 'rejected-event'
? response(400, { code: 'unsafe_tracing_input', error: 'unsafe' })
: response(500, { error: 'Internal server error' })
}) as unknown as typeof fetch
const reporter = new BoundedInfrastructureTelemetryReporter({
baseUrl: 'http://control-api.test:8090',
signToken: () => 'signed-request',
fetchFn,
random: () => 0,
})
reporter.enqueue({
...projection,
telemetryType: 'lifecycle_transition',
sourceEventId: 'rejected-event',
})
reporter.enqueue({
...projection,
telemetryType: 'lifecycle_transition',
sourceEventId: 'transient-event',
})
await new Promise(resolve => setTimeout(resolve, 150))
const ids = vi
.mocked(fetchFn)
.mock.calls.map(
call =>
(JSON.parse(String(call[1]?.body)) as { events: Array<{ sourceEventId: string }> })
.events[0]!.sourceEventId
)
// Liveness: the 500 on the other event is retried twice.
expect(ids.filter(id => id === 'transient-event')).toHaveLength(3)
expect(ids.filter(id => id === 'rejected-event')).toHaveLength(1)
const flushes = (await infrastructureTelemetryFlushesTotal.get()).values
expect(flushes.find(value => value.labels.result === 'rejected')?.value).toBe(1)
expect(flushes.find(value => value.labels.result === 'exhausted')?.value).toBe(1)
// Both events were lost: one rejected, one after its retries.
const gaps = (await infrastructureTelemetryGapsTotal.get()).values.map(value => value.labels)
expect(gaps).toEqual(
expect.arrayContaining([
{ telemetry_type: 'lifecycle_transition', reason: 'rejected' },
{ telemetry_type: 'lifecycle_transition', reason: 'retry_exhausted' },
])
)
expect(gaps).toHaveLength(2)
})
it('counts a 409 idempotency conflict as an evidence gap (#696)', async () => {
infrastructureTelemetryFlushesTotal.reset()
infrastructureTelemetryGapsTotal.reset()
const fetchFn = vi.fn(
async () =>
({
ok: false,
status: 409,
json: async () => ({ code: 'tracing_idempotency_conflict' }),
}) as unknown as Response
) as unknown as typeof fetch
const reporter = new BoundedInfrastructureTelemetryReporter({
baseUrl: 'http://control-api.test:8090',
signToken: () => 'signed-request',
fetchFn,
random: () => 0,
})
reporter.enqueue({
...projection,
telemetryType: 'lifecycle_transition',
sourceEventId: 'conflicting-event',
})
await new Promise(resolve => setTimeout(resolve, 150))
// Liveness: the conflict was submitted once and settled.
expect(fetchFn).toHaveBeenCalledOnce()
const flushes = (await infrastructureTelemetryFlushesTotal.get()).values
expect(flushes.find(value => value.labels.result === 'conflict')?.value).toBe(1)
// The stored row holds a different payload hash, so this observation was
// never stored: a gap, told apart from a rejection by `reason`.
expect((await infrastructureTelemetryGapsTotal.get()).values).toEqual([
{
labels: { telemetry_type: 'lifecycle_transition', reason: 'conflict' },
value: 1,
},
])
})
it('isolates a blackholed submission and continues flushing later telemetry', async () => {
infrastructureTelemetryGapsTotal.reset()
const fetchFn = vi
.fn()
.mockResolvedValueOnce({ ok: false, status: 503 })
.mockResolvedValue({ ok: true }) as unknown as typeof fetch
const reporter = new BoundedInfrastructureTelemetryReporter({
baseUrl: 'http://control-api.test:8090',
signToken: () => 'signed-request',
fetchFn,
retryLimit: 0,
})
reporter.enqueue({
...reconcileProjection,
payload: { ...reconcileProjection.payload, status: 'failed', state: 'not_ready' },
})
reporter.enqueue(reconcileProjection)
await new Promise(resolve => setTimeout(resolve, 0))
await new Promise(resolve => setTimeout(resolve, 0))
expect(fetchFn).toHaveBeenCalledTimes(2)
expect((await infrastructureTelemetryGapsTotal.get()).values[0]).toMatchObject({
labels: { telemetry_type: 'reconcile_outcome', reason: 'retry_exhausted' },
value: 1,
})
expect(JSON.parse(String(vi.mocked(fetchFn).mock.calls[1]![1]?.body))).toEqual({
events: [{ ...reconcileProjection, sourceEventId: reconcileSourceEventId }],
})
})
it('flushes queued telemetry during stop without waiting for the scheduled microtask', async () => {
const fetchFn = vi.fn().mockResolvedValue({ ok: true }) as unknown as typeof fetch
const reporter = new BoundedInfrastructureTelemetryReporter({
baseUrl: 'http://control-api.test:8090',
signToken: () => 'signed-request',
fetchFn,
})
reporter.enqueue(reconcileProjection)
await reporter.stop()
expect(fetchFn).toHaveBeenCalledOnce()
expect(JSON.parse(String(vi.mocked(fetchFn).mock.calls[0]![1]?.body))).toEqual({
events: [{ ...reconcileProjection, sourceEventId: reconcileSourceEventId }],
})
})
it('bounds stop when a telemetry submission never settles', async () => {
const fetchFn = vi.fn(() => new Promise<Response>(() => undefined)) as unknown as typeof fetch
const reporter = new BoundedInfrastructureTelemetryReporter({
baseUrl: 'http://control-api.test:8090',
signToken: () => 'signed-request',
fetchFn,
timeoutMs: 1,
})
reporter.enqueueHealthTransition(projection)
const start = Date.now()
await reporter.stop(10)
expect(Date.now() - start).toBeLessThan(250)
expect(fetchFn).toHaveBeenCalledOnce()
})
})