Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript
15.7 KB
Raw Blame
import { beforeEach, describe, expect, it, vi } from 'vitest'
import {
  type AdministrativeHostOutcomeProjection,
  BoundedAdministrativeOutcomeReporter,
  createAdministrativeOutcomeReporter,
} from './administrativeOutcomeReporter'
import { administrativeOutcomeReporterTotal } from './metrics'

/**
 * Cross-service format contract (#694). `EXPECTED_STATUS_REF` is the exact
 * string control-api's `STATUS_REF` must parse
 * (control-api/src/services/tracing/adminOperationBindingResolver.ts), which
 * pins this same literal in
 * control-api/test/services.adminOperationBindingResolver.test.ts.
 *
 * The duplication is a convention, not an enforced contract. The two packages
 * cannot import each other, so nothing mechanically ties the two literals
 * together: add a segment to the template below and this test fails while
 * control-api's keeps parsing the old string and stays green. Drift surfaces
 * in production instead, as a 403 with no `code` from the `$`-anchored
 * STATUS_REF, which reporterHttpFailure classifies as retryable. Change one
 * side and you must change the other literal by hand.
 */
const HOST_UID = '6f1c2f3a-2f4b-4d3a-9b2e-7c0d1a5e8b44'
const HOST_REF = { name: 'chatllm', namespace: 'mcp-host', generation: 7, uid: HOST_UID }
const EXPECTED_STATUS_REF = `host:mcp-host/chatllm:generation=7:uid=${HOST_UID}`

describe('createAdministrativeOutcomeReporter', () => {
  it('does not construct a reporter when governed tracing is disabled', () => {
    expect(
      createAdministrativeOutcomeReporter(false, {
        baseUrl: 'http://control-api.test:8090',
      })
    ).toBeUndefined()
  })

  it('constructs a reporter when governed tracing is enabled', () => {
    expect(
      createAdministrativeOutcomeReporter(true, {
        baseUrl: 'http://control-api.test:8090',
      })
    ).toBeInstanceOf(BoundedAdministrativeOutcomeReporter)
  })
})

describe('BoundedAdministrativeOutcomeReporter', () => {
  it('submits a typed linked outcome off-path', async () => {
    const fetchFn = vi.fn().mockResolvedValue({ ok: true }) as unknown as typeof fetch
    const reporter = new BoundedAdministrativeOutcomeReporter({
      baseUrl: 'http://control-api.test:8090',
      signToken: () => 'signed',
      fetchFn,
    })
    reporter.enqueueHostOutcome({
      sourceEventId: 'hcc-admin-outcome-v2:op-1:7:host-uid-1:succeeded',
      occurredAt: '2026-07-11T10:00:00.000Z',
      hostRef: HOST_REF,
      outcome: 'succeeded',
      reasonCode: 'reconciled',
    })
    expect(fetchFn).not.toHaveBeenCalled()
    await new Promise(resolve => setTimeout(resolve, 0))
    const body = JSON.parse(String(vi.mocked(fetchFn).mock.calls[0]![1]?.body))
    expect(body.events[0]).toEqual(
      expect.objectContaining({
        kind: 'linked_outcome',
        sourceStatusRef: EXPECTED_STATUS_REF,
        payload: { resource_class: 'Host', status: 'succeeded' },
      })
    )
  })

  it('isolates a failed submission from enqueue and bounds retries', async () => {
    const fetchFn = vi.fn().mockRejectedValue(new Error('unreachable')) as unknown as typeof fetch
    const reporter = new BoundedAdministrativeOutcomeReporter({
      baseUrl: 'http://control-api.test:8090',
      signToken: () => 'signed',
      fetchFn,
      retryLimit: 0,
    })
    expect(() =>
      reporter.enqueueHostOutcome({
        sourceEventId: 'outcome-1',
        occurredAt: '2026-07-11T10:00:00.000Z',
        hostRef: HOST_REF,
        outcome: 'failed',
        reasonCode: 'reconcile_failed',
      })
    ).not.toThrow()
    await new Promise(resolve => setTimeout(resolve, 0))
    expect(fetchFn).toHaveBeenCalledOnce()
  })

  it('flushes a queued linked outcome during stop without waiting for the scheduled microtask', async () => {
    const fetchFn = vi.fn().mockResolvedValue({ ok: true }) as unknown as typeof fetch
    const reporter = new BoundedAdministrativeOutcomeReporter({
      baseUrl: 'http://control-api.test:8090',
      signToken: () => 'signed',
      fetchFn,
    })

    reporter.enqueueHostOutcome({
      sourceEventId: 'outcome-queued',
      occurredAt: '2026-07-11T10:00:00.000Z',
      hostRef: HOST_REF,
      outcome: 'succeeded',
      reasonCode: 'reconciled',
    })
    await reporter.stop()

    expect(fetchFn).toHaveBeenCalledOnce()
    const body = JSON.parse(String(vi.mocked(fetchFn).mock.calls[0]![1]?.body))
    expect(body.events[0]).toEqual(
      expect.objectContaining({
        sourceEventId: 'outcome-queued',
        kind: 'linked_outcome',
        sourceStatusRef: EXPECTED_STATUS_REF,
      })
    )
  })

  it('contains shutdown submission failures without scheduling retries', async () => {
    const fetchFn = vi.fn().mockRejectedValue(new Error('unreachable')) as unknown as typeof fetch
    const reporter = new BoundedAdministrativeOutcomeReporter({
      baseUrl: 'http://control-api.test:8090',
      signToken: () => 'signed',
      fetchFn,
      retryLimit: 5,
      random: () => 0,
    })

    reporter.enqueueHostOutcome({
      sourceEventId: 'outcome-failed',
      occurredAt: '2026-07-11T10:00:00.000Z',
      hostRef: HOST_REF,
      outcome: 'failed',
      reasonCode: 'reconcile_failed',
    })
    await reporter.stop()
    await new Promise(resolve => setTimeout(resolve, 60))

    expect(fetchFn).toHaveBeenCalledOnce()
  })
})

function outcome(sourceEventId: string): AdministrativeHostOutcomeProjection {
  return {
    sourceEventId,
    occurredAt: '2026-09-18T10:00:00.000Z',
    hostRef: HOST_REF,
    outcome: 'succeeded',
    reasonCode: 'reconciled',
  }
}

function failedResponse(status: number, body?: unknown): Response {
  return {
    ok: false,
    status,
    json: async () => {
      if (body === undefined) throw new SyntaxError('Unexpected end of JSON input')
      return body
    },
  } as unknown as Response
}

async function counted(result: string): Promise<number> {
  const metric = await administrativeOutcomeReporterTotal.get()
  return metric.values.find(value => value.labels.result === result)?.value ?? 0
}

function submittedIds(fetchFn: typeof fetch): string[] {
  return vi.mocked(fetchFn).mock.calls.map(call => {
    const body = JSON.parse(String(call[1]?.body)) as { events: Array<{ sourceEventId: string }> }
    return body.events[0]!.sourceEventId
  })
}

// Default retry delays with random() = 0 are 25 ms then 50 ms.
const settle = () => new Promise(resolve => setTimeout(resolve, 150))

describe('BoundedAdministrativeOutcomeReporter — once per process (#327, #326)', () => {
  beforeEach(() => administrativeOutcomeReporterTotal.reset())

  function reporterWith(
    fetchFn: typeof fetch,
    bounds: { capacity?: number; dedupeCapacity?: number } = {}
  ) {
    return new BoundedAdministrativeOutcomeReporter({
      baseUrl: 'http://control-api.test:8090',
      signToken: () => 'signed',
      fetchFn,
      random: () => 0,
      ...bounds,
    })
  }

  it('deduplicates a key while its first copy is queued or waiting for a retry', async () => {
    const fetchFn = vi.fn().mockResolvedValue(failedResponse(500)) as unknown as typeof fetch
    const reporter = reporterWith(fetchFn)

    reporter.enqueueHostOutcome(outcome('in-flight'))
    reporter.enqueueHostOutcome(outcome('in-flight'))
    // The first attempt has failed; the retry fires at 25 ms.
    await new Promise(resolve => setTimeout(resolve, 10))
    expect(fetchFn).toHaveBeenCalledOnce()
    reporter.enqueueHostOutcome(outcome('in-flight'))
    await settle()

    // Liveness: the one queued copy used its whole retry budget.
    expect(fetchFn).toHaveBeenCalledTimes(3)
    expect(await counted('retry_exhausted')).toBe(1)
    expect(await counted('deduplicated')).toBe(2)
  })

  it('sends an outcome again after it was dropped because the buffer was full', async () => {
    const fetchFn = vi.fn().mockResolvedValue({ ok: true }) as unknown as typeof fetch
    const reporter = reporterWith(fetchFn, { capacity: 1, dedupeCapacity: 2 })

    reporter.enqueueHostOutcome(outcome('buffered'))
    reporter.enqueueHostOutcome(outcome('dropped'))
    await settle()
    expect(await counted('buffer_full')).toBe(1)
    reporter.enqueueHostOutcome(outcome('dropped'))
    await settle()

    expect(submittedIds(fetchFn)).toEqual(['buffered', 'dropped'])
    expect(await counted('deduplicated')).toBe(0)
  })

  it('keeps a key that is seen on every pass ahead of keys seen once', async () => {
    const fetchFn = vi.fn().mockResolvedValue({ ok: true }) as unknown as typeof fetch
    const reporter = reporterWith(fetchFn, { capacity: 1, dedupeCapacity: 2 })

    for (const id of ['repeated', 'once', 'repeated', 'newer', 'repeated']) {
      reporter.enqueueHostOutcome(outcome(id))
      await settle()
    }

    // 'newer' evicted 'once', not 'repeated', because 'repeated' was seen again.
    expect(submittedIds(fetchFn)).toEqual(['repeated', 'once', 'newer'])
    expect(await counted('deduplicated')).toBe(2)
  })

  it('requires the dedupe bound to exceed the queue capacity', () => {
    const fetchFn = vi.fn() as unknown as typeof fetch

    expect(() => reporterWith(fetchFn, { capacity: 4, dedupeCapacity: 4 })).toThrow(
      'dedupeCapacity must be an integer greater than capacity'
    )
    expect(reporterWith(fetchFn, { capacity: 4, dedupeCapacity: 5 })).toBeInstanceOf(
      BoundedAdministrativeOutcomeReporter
    )
  })

  it('sends an accepted outcome once however often the reconciler re-observes it', async () => {
    const fetchFn = vi.fn().mockResolvedValue({ ok: true }) as unknown as typeof fetch
    const reporter = reporterWith(fetchFn)

    reporter.enqueueHostOutcome(outcome('hcc-admin-outcome-v2:op-1:7:host-uid-1:succeeded'))
    await settle()
    reporter.enqueueHostOutcome(outcome('hcc-admin-outcome-v2:op-1:7:host-uid-1:succeeded'))
    reporter.enqueueHostOutcome(outcome('hcc-admin-outcome-v2:op-1:7:host-uid-1:succeeded'))
    await settle()

    expect(fetchFn).toHaveBeenCalledOnce()
    expect(await counted('accepted')).toBe(1)
    expect(await counted('deduplicated')).toBe(2)
  })

  it('settles a 409 idempotency conflict without retry and deduplicates the key afterwards', async () => {
    const fetchFn = vi.fn(async (_url: unknown, init?: RequestInit) => {
      const id = (JSON.parse(String(init?.body)) as { events: Array<{ sourceEventId: string }> })
        .events[0]!.sourceEventId
      return id === 'conflicting'
        ? failedResponse(409, { code: 'tracing_idempotency_conflict', error: 'conflict' })
        : failedResponse(500, { error: 'Internal server error' })
    }) as unknown as typeof fetch
    const reporter = reporterWith(fetchFn)

    reporter.enqueueHostOutcome(outcome('conflicting'))
    reporter.enqueueHostOutcome(outcome('transient'))
    await settle()
    reporter.enqueueHostOutcome(outcome('conflicting'))
    await settle()

    const ids = submittedIds(fetchFn)
    // Liveness: the transient failure on another key is retried twice.
    expect(ids.filter(id => id === 'transient')).toHaveLength(3)
    expect(ids.filter(id => id === 'conflicting')).toHaveLength(1)
    expect(await counted('conflict')).toBe(1)
    expect(await counted('deduplicated')).toBe(1)
  })

  it('sends an outcome again after its retries were exhausted', async () => {
    const fetchFn = vi
      .fn()
      .mockResolvedValueOnce(failedResponse(500))
      .mockResolvedValueOnce(failedResponse(500))
      .mockResolvedValueOnce(failedResponse(500))
      .mockResolvedValue({ ok: true }) as unknown as typeof fetch
    const reporter = reporterWith(fetchFn)

    reporter.enqueueHostOutcome(outcome('hcc-admin-outcome-v2:op-1:7:host-uid-1:failed'))
    await settle()
    expect(await counted('retry_exhausted')).toBe(1)
    reporter.enqueueHostOutcome(outcome('hcc-admin-outcome-v2:op-1:7:host-uid-1:failed'))
    await settle()

    expect(fetchFn).toHaveBeenCalledTimes(4)
    expect(await counted('accepted')).toBe(1)
    expect(await counted('deduplicated')).toBe(0)
  })

  it.each([
    ['a 403 binding not yet visible', failedResponse(403, { error: 'Forbidden' })],
    ['a 409 without a body', failedResponse(409)],
    ['a 409 without a code', failedResponse(409, { error: 'Conflict' })],
    ['a 400 without a code', failedResponse(400, { error: 'Bad Request' })],
  ])('keeps retrying %s', async (_label, response) => {
    const fetchFn = vi.fn().mockResolvedValue(response) as unknown as typeof fetch
    const reporter = reporterWith(fetchFn)

    reporter.enqueueHostOutcome(outcome('retryable'))
    await settle()

    expect(fetchFn).toHaveBeenCalledTimes(3)
    expect(await counted('retry_exhausted')).toBe(1)
    expect(await counted('conflict')).toBe(0)
    expect(await counted('rejected')).toBe(0)
  })

  it('forgets the oldest settled outcome once the dedupe bound is reached', async () => {
    const fetchFn = vi.fn().mockResolvedValue({ ok: true }) as unknown as typeof fetch
    const reporter = reporterWith(fetchFn, { capacity: 1, dedupeCapacity: 2 })

    for (const id of ['first', 'second', 'third']) {
      reporter.enqueueHostOutcome(outcome(id))
      await settle()
    }
    reporter.enqueueHostOutcome(outcome('third'))
    reporter.enqueueHostOutcome(outcome('first'))
    await settle()

    // 'third' is still remembered (witness); 'first' was evicted and is sent again.
    expect(submittedIds(fetchFn)).toEqual(['first', 'second', 'third', 'first'])
    expect(await counted('deduplicated')).toBe(1)
  })

  /**
   * #329 — the loop this cuts. A Host whose intent annotation names a
   * generation it has already passed answers 409
   * `administrative_intent_generation_drift` forever, and before this change
   * that answer was a plain retryable failure: 3 submits per enqueue, and
   * `onDrop` deleted the dedupe key so the next reconciler pass enqueued it
   * again. Measured in dev at 109 requests/hour for a single drifted Host (327
   * refusals over 3 hours); the arithmetic for 3 enqueues × 3 submits per
   * 300-second reconciler cycle predicts 108, which is how the measurement was
   * tied to this loop rather than to some other source of 403s.
   *
   * The terminal classification routes it to `onTerminal`, which PRESERVES the
   * key, so the re-enqueue is deduplicated instead of resubmitted. The bound
   * is honest: 1 submit, then 0 while the key stays resident — not zero, and
   * not permanent, since `seen` is memory and does not survive a restart.
   */
  it('settles a 409 generation-drift refusal once and stops re-sending it (#329)', async () => {
    const fetchFn = vi.fn(async (_url: unknown, init?: RequestInit) => {
      const id = (JSON.parse(String(init?.body)) as { events: Array<{ sourceEventId: string }> })
        .events[0]!.sourceEventId
      return id === 'drifted'
        ? failedResponse(409, {
            code: 'administrative_intent_generation_drift',
            error: 'ignored',
          })
        : failedResponse(500, { error: 'Internal server error' })
    }) as unknown as typeof fetch
    const reporter = reporterWith(fetchFn)

    reporter.enqueueHostOutcome(outcome('drifted'))
    reporter.enqueueHostOutcome(outcome('transient'))
    await settle()
    // The reconciler observes the same drifted Host on its next pass.
    reporter.enqueueHostOutcome(outcome('drifted'))
    await settle()

    const ids = submittedIds(fetchFn)
    // Liveness witness: a retryable failure on another key in the same run
    // used its full budget of 3, so the single 'drifted' submit is the
    // terminal classification and not a reporter that stopped submitting.
    expect(ids.filter(id => id === 'transient')).toHaveLength(3)
    expect(ids.filter(id => id === 'drifted')).toHaveLength(1)
    expect(await counted('rejected')).toBe(1)
    expect(await counted('deduplicated')).toBe(1)
    // Not a conflict: that label belongs to `tracing_idempotency_conflict`,
    // and conflating them would hide the drift inside an existing metric.
    expect(await counted('conflict')).toBe(0)
  })
})