Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript
12.6 KB
Raw Blame
import { describe, expect, it, vi } from 'vitest'
import {
  BoundedInfrastructureTelemetryReporter,
  createInfrastructureTelemetryReporter,
  hccReconcileOutcomeSourceId,
} from './infrastructureTelemetryReporter'
import {
  infrastructureTelemetryFlushesTotal,
  infrastructureTelemetryGapsTotal,
  infrastructureTelemetryRetriesTotal,
} from './metrics'

/** control-api refuses a reference without one, so every fixture carries it. */
const HOST_UID = '6f1c2f3a-2f4b-4d3a-9b2e-7c0d1a5e8b44'

const projection = {
  sourceEventId: 'hcc-health-transition:mcp-host:chatllm:7:active:1',
  occurredAt: '2026-07-11T12:00:00.000Z',
  hostLookupReference: { name: 'chatllm', namespace: 'mcp-host', generation: 7, uid: HOST_UID },
  payload: { transition: 'lifecycle:active', state: 'active' },
} as const

const reconcileProjection = {
  occurredAt: '2026-07-11T12:00:01.000Z',
  telemetryType: 'reconcile_outcome',
  hostLookupReference: { name: 'chatllm', namespace: 'mcp-host', generation: 7, uid: HOST_UID },
  payload: {
    resource_class: 'Host',
    reason_code: 'ready',
    status: 'succeeded',
    phase: 'deployed',
    state: 'ready',
  },
} as const

const reconcileSourceEventId = hccReconcileOutcomeSourceId(reconcileProjection)

describe('BoundedInfrastructureTelemetryReporter', () => {
  it('does not instantiate when governed tracing is disabled', () => {
    const reporter = createInfrastructureTelemetryReporter(false, {
      baseUrl: 'http://control-api.test:8090',
    })

    expect(reporter).toBeUndefined()
  })

  it('instantiates when governed tracing is enabled', () => {
    const reporter = createInfrastructureTelemetryReporter(true, {
      baseUrl: 'http://control-api.test:8090',
    })

    expect(reporter).toBeInstanceOf(BoundedInfrastructureTelemetryReporter)
  })

  it('keeps enqueue synchronous and submits the health projection off-path', async () => {
    const fetchFn = vi.fn().mockResolvedValue({ ok: true }) as unknown as typeof fetch
    const reporter = new BoundedInfrastructureTelemetryReporter({
      baseUrl: 'http://control-api.test:8090/',
      signToken: () => 'signed-request',
      fetchFn,
    })

    reporter.enqueueHealthTransition(projection)

    expect(fetchFn).not.toHaveBeenCalled()
    await new Promise(resolve => setTimeout(resolve, 0))
    expect(fetchFn).toHaveBeenCalledOnce()
    const [url, init] = vi.mocked(fetchFn).mock.calls[0]!
    expect(url).toBe(
      'http://control-api.test:8090/api/v1/internal/tracing/infrastructure-telemetry-events'
    )
    expect(init?.headers).toMatchObject({ Authorization: 'Bearer signed-request' })
    expect(JSON.parse(String(init?.body))).toEqual({
      events: [{ ...projection, telemetryType: 'health_transition' }],
    })
  })

  it('submits Host-backed reconcile telemetry with server-observed lookup reference', async () => {
    const fetchFn = vi.fn().mockResolvedValue({ ok: true }) as unknown as typeof fetch
    const reporter = new BoundedInfrastructureTelemetryReporter({
      baseUrl: 'http://control-api.test:8090',
      signToken: () => 'signed-request',
      fetchFn,
    })

    reporter.enqueue(reconcileProjection)

    await new Promise(resolve => setTimeout(resolve, 0))
    expect(JSON.parse(String(vi.mocked(fetchFn).mock.calls[0]![1]?.body))).toEqual({
      events: [{ ...reconcileProjection, sourceEventId: reconcileSourceEventId }],
    })
  })

  it('derives reconcile identity only after enqueue returns', async () => {
    const fetchFn = vi.fn().mockResolvedValue({ ok: true }) as unknown as typeof fetch
    const deriveReconcileSourceEventId = vi.fn(hccReconcileOutcomeSourceId)
    const reporter = new BoundedInfrastructureTelemetryReporter({
      baseUrl: 'http://control-api.test:8090',
      signToken: () => 'signed-request',
      fetchFn,
      deriveReconcileSourceEventId,
    })

    reporter.enqueue(reconcileProjection)

    expect(deriveReconcileSourceEventId).not.toHaveBeenCalled()
    expect(fetchFn).not.toHaveBeenCalled()
    await new Promise(resolve => setTimeout(resolve, 0))
    expect(deriveReconcileSourceEventId).toHaveBeenCalledOnce()
    expect(fetchFn).toHaveBeenCalledOnce()
  })

  it('derives reconcile identity once and reuses it across transport retries', async () => {
    infrastructureTelemetryRetriesTotal.reset()
    const fetchFn = vi
      .fn()
      .mockResolvedValueOnce({ ok: false, status: 503 })
      .mockResolvedValueOnce({ ok: true }) as unknown as typeof fetch
    const deriveReconcileSourceEventId = vi.fn(() => 'hcc-reconcile-outcome:stable')
    const reporter = new BoundedInfrastructureTelemetryReporter({
      baseUrl: 'http://control-api.test:8090',
      signToken: () => 'signed-request',
      fetchFn,
      deriveReconcileSourceEventId,
      retryLimit: 1,
      random: () => 0,
    })

    reporter.enqueue(reconcileProjection)
    await new Promise(resolve => setTimeout(resolve, 60))

    expect(fetchFn).toHaveBeenCalledTimes(2)
    expect(deriveReconcileSourceEventId).toHaveBeenCalledOnce()
    expect((await infrastructureTelemetryRetriesTotal.get()).values[0]?.value).toBe(1)
    const sourceIds = vi.mocked(fetchFn).mock.calls.map(call => {
      const body = JSON.parse(String(call[1]?.body)) as { events: Array<{ sourceEventId: string }> }
      return body.events[0]!.sourceEventId
    })
    expect(sourceIds).toEqual(['hcc-reconcile-outcome:stable', 'hcc-reconcile-outcome:stable'])
  })

  it('keeps reconcile identity stable for equivalent state and changes it with the outcome', () => {
    expect(hccReconcileOutcomeSourceId(reconcileProjection)).toMatch(
      /^hcc-reconcile-outcome-v3:[0-9a-f]{64}$/
    )
    expect(hccReconcileOutcomeSourceId(reconcileProjection)).toBe(
      hccReconcileOutcomeSourceId({ ...reconcileProjection })
    )
    expect(hccReconcileOutcomeSourceId(reconcileProjection)).not.toBe(
      hccReconcileOutcomeSourceId({
        ...reconcileProjection,
        payload: {
          ...reconcileProjection.payload,
          reason_code: 'not_ready',
          status: 'failed',
        },
      })
    )
  })

  it('gives a recreated Host object (same name and generation, new uid) new identities (#691)', () => {
    const withUid = (uid: string) => ({
      ...reconcileProjection,
      hostLookupReference: { ...reconcileProjection.hostLookupReference, uid },
    })

    // Liveness: the identity is deterministic for an identical tuple.
    expect(hccReconcileOutcomeSourceId(withUid('uid-new'))).toBe(
      hccReconcileOutcomeSourceId(withUid('uid-new'))
    )
    expect(hccReconcileOutcomeSourceId(withUid('uid-new'))).not.toBe(
      hccReconcileOutcomeSourceId(withUid('uid-old'))
    )
  })

  it('settles a 400 unsafe_tracing_input once and keeps retrying a 500 (#326, #328)', async () => {
    infrastructureTelemetryFlushesTotal.reset()
    infrastructureTelemetryGapsTotal.reset()
    const response = (status: number, body: unknown) =>
      ({ ok: false, status, json: async () => body }) as unknown as Response
    const fetchFn = vi.fn(async (_url: unknown, init?: RequestInit) => {
      const event = (JSON.parse(String(init?.body)) as { events: Array<{ sourceEventId: string }> })
        .events[0]!
      return event.sourceEventId === 'rejected-event'
        ? response(400, { code: 'unsafe_tracing_input', error: 'unsafe' })
        : response(500, { error: 'Internal server error' })
    }) as unknown as typeof fetch
    const reporter = new BoundedInfrastructureTelemetryReporter({
      baseUrl: 'http://control-api.test:8090',
      signToken: () => 'signed-request',
      fetchFn,
      random: () => 0,
    })

    reporter.enqueue({
      ...projection,
      telemetryType: 'lifecycle_transition',
      sourceEventId: 'rejected-event',
    })
    reporter.enqueue({
      ...projection,
      telemetryType: 'lifecycle_transition',
      sourceEventId: 'transient-event',
    })
    await new Promise(resolve => setTimeout(resolve, 150))

    const ids = vi
      .mocked(fetchFn)
      .mock.calls.map(
        call =>
          (JSON.parse(String(call[1]?.body)) as { events: Array<{ sourceEventId: string }> })
            .events[0]!.sourceEventId
      )
    // Liveness: the 500 on the other event is retried twice.
    expect(ids.filter(id => id === 'transient-event')).toHaveLength(3)
    expect(ids.filter(id => id === 'rejected-event')).toHaveLength(1)
    const flushes = (await infrastructureTelemetryFlushesTotal.get()).values
    expect(flushes.find(value => value.labels.result === 'rejected')?.value).toBe(1)
    expect(flushes.find(value => value.labels.result === 'exhausted')?.value).toBe(1)
    // Both events were lost: one rejected, one after its retries.
    const gaps = (await infrastructureTelemetryGapsTotal.get()).values.map(value => value.labels)
    expect(gaps).toEqual(
      expect.arrayContaining([
        { telemetry_type: 'lifecycle_transition', reason: 'rejected' },
        { telemetry_type: 'lifecycle_transition', reason: 'retry_exhausted' },
      ])
    )
    expect(gaps).toHaveLength(2)
  })

  it('counts a 409 idempotency conflict as an evidence gap (#696)', async () => {
    infrastructureTelemetryFlushesTotal.reset()
    infrastructureTelemetryGapsTotal.reset()
    const fetchFn = vi.fn(
      async () =>
        ({
          ok: false,
          status: 409,
          json: async () => ({ code: 'tracing_idempotency_conflict' }),
        }) as unknown as Response
    ) as unknown as typeof fetch
    const reporter = new BoundedInfrastructureTelemetryReporter({
      baseUrl: 'http://control-api.test:8090',
      signToken: () => 'signed-request',
      fetchFn,
      random: () => 0,
    })

    reporter.enqueue({
      ...projection,
      telemetryType: 'lifecycle_transition',
      sourceEventId: 'conflicting-event',
    })
    await new Promise(resolve => setTimeout(resolve, 150))

    // Liveness: the conflict was submitted once and settled.
    expect(fetchFn).toHaveBeenCalledOnce()
    const flushes = (await infrastructureTelemetryFlushesTotal.get()).values
    expect(flushes.find(value => value.labels.result === 'conflict')?.value).toBe(1)
    // The stored row holds a different payload hash, so this observation was
    // never stored: a gap, told apart from a rejection by `reason`.
    expect((await infrastructureTelemetryGapsTotal.get()).values).toEqual([
      {
        labels: { telemetry_type: 'lifecycle_transition', reason: 'conflict' },
        value: 1,
      },
    ])
  })

  it('isolates a blackholed submission and continues flushing later telemetry', async () => {
    infrastructureTelemetryGapsTotal.reset()
    const fetchFn = vi
      .fn()
      .mockResolvedValueOnce({ ok: false, status: 503 })
      .mockResolvedValue({ ok: true }) as unknown as typeof fetch
    const reporter = new BoundedInfrastructureTelemetryReporter({
      baseUrl: 'http://control-api.test:8090',
      signToken: () => 'signed-request',
      fetchFn,
      retryLimit: 0,
    })

    reporter.enqueue({
      ...reconcileProjection,
      payload: { ...reconcileProjection.payload, status: 'failed', state: 'not_ready' },
    })
    reporter.enqueue(reconcileProjection)

    await new Promise(resolve => setTimeout(resolve, 0))
    await new Promise(resolve => setTimeout(resolve, 0))

    expect(fetchFn).toHaveBeenCalledTimes(2)
    expect((await infrastructureTelemetryGapsTotal.get()).values[0]).toMatchObject({
      labels: { telemetry_type: 'reconcile_outcome', reason: 'retry_exhausted' },
      value: 1,
    })
    expect(JSON.parse(String(vi.mocked(fetchFn).mock.calls[1]![1]?.body))).toEqual({
      events: [{ ...reconcileProjection, sourceEventId: reconcileSourceEventId }],
    })
  })

  it('flushes queued telemetry during stop without waiting for the scheduled microtask', async () => {
    const fetchFn = vi.fn().mockResolvedValue({ ok: true }) as unknown as typeof fetch
    const reporter = new BoundedInfrastructureTelemetryReporter({
      baseUrl: 'http://control-api.test:8090',
      signToken: () => 'signed-request',
      fetchFn,
    })

    reporter.enqueue(reconcileProjection)
    await reporter.stop()

    expect(fetchFn).toHaveBeenCalledOnce()
    expect(JSON.parse(String(vi.mocked(fetchFn).mock.calls[0]![1]?.body))).toEqual({
      events: [{ ...reconcileProjection, sourceEventId: reconcileSourceEventId }],
    })
  })

  it('bounds stop when a telemetry submission never settles', async () => {
    const fetchFn = vi.fn(() => new Promise<Response>(() => undefined)) as unknown as typeof fetch
    const reporter = new BoundedInfrastructureTelemetryReporter({
      baseUrl: 'http://control-api.test:8090',
      signToken: () => 'signed-request',
      fetchFn,
      timeoutMs: 1,
    })

    reporter.enqueueHealthTransition(projection)
    const start = Date.now()
    await reporter.stop(10)

    expect(Date.now() - start).toBeLessThan(250)
    expect(fetchFn).toHaveBeenCalledOnce()
  })
})