Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
import { afterEach, describe, expect, it, vi } from 'vitest'
import type { DbClient } from '../src/db.js'
import type {
AdministrativeEventSubmitterPrincipalV1,
AgentRunEventSubmitterPrincipalV1,
} from '../src/middleware/tracingSubmitterAuth.js'
import {
governedTraceAcceptedTotal,
governedTraceConflictingTotal,
governedTraceLastErrorTimestampSeconds,
governedTraceOperationalErrorsTotal,
governedTraceRejectedTotal,
governedTraceReplayedTotal,
} from '../src/observability/metrics.js'
import type { AdministrativeEventSubmissionService } from '../src/routes/internal/administrativeEvents.js'
import type { AgentRunEventSubmissionService } from '../src/routes/internal/agentRunEvents.js'
import type { InfrastructureTelemetryEventSubmissionService } from '../src/routes/internal/infrastructureTelemetryEvents.js'
import {
TracingIdempotencyConflictError,
UnsafeTracingInputError,
} from '../src/services/tracing/append.js'
import type {
GovernedAppendResult,
TracingTransactionRunner,
} from '../src/services/tracing/contracts.js'
import {
AdministrativeIntentGenerationDriftError,
InvalidTracingInputError,
RouteTracingSubmissionService,
TracingBindingUnavailableError,
} from '../src/services/tracing/routeSubmissionService.js'
import {
NOW,
adminBinding,
agentBinding,
agentInput,
infraBinding,
infraPrincipal,
} from './services.tracingFixtures.js'
const workflowPrincipal: AgentRunEventSubmitterPrincipalV1 = {
kind: 'wrc_internal_control',
sourceService: 'workflow-recipes',
serviceSub: 'wrc-provisioner',
credentialId: 'test-principal',
allowedEventTypes: ['run_start', 'run_end'],
}
const workflowInput = {
...agentInput,
runId: agentBinding.runId,
}
const administrativePrincipal = {
kind: 'hcc_internal_control',
sourceService: 'host-context-controller',
serviceSub: 'hcc-provisioner',
credentialId: 'test-principal',
allowedKinds: ['linked_outcome'],
} as const satisfies AdministrativeEventSubmitterPrincipalV1
const administrativeInput = {
sourceEventId: 'administrative-1',
occurredAt: NOW,
kind: 'linked_outcome',
}
function appendResult(kind: 'accepted' | 'replayed'): GovernedAppendResult {
return {
kind,
accepted: kind === 'accepted' ? 1 : 0,
replayed: kind === 'replayed' ? 1 : 0,
family: 'agent_run',
eventId: '11111111-1111-4111-8111-111111111111',
streamSequence: '41',
payloadSha256: 'a'.repeat(64),
ingestedAt: NOW,
}
}
function transactionHarness() {
const db = { query: vi.fn() } as DbClient
const transactionSpy = vi.fn(async (work: (client: DbClient) => Promise<unknown>) => work(db))
return {
db,
transactionSpy,
transaction: transactionSpy as TracingTransactionRunner,
}
}
describe('route tracing submission facade', () => {
afterEach(() => {
vi.restoreAllMocks()
})
it('satisfies all three route services and appends a batch in one transaction', async () => {
const h = transactionHarness()
const resolve = vi.fn().mockResolvedValue(agentBinding)
const appendManyInTransaction = vi
.fn()
.mockResolvedValue([appendResult('accepted'), appendResult('accepted')])
const service = new RouteTracingSubmissionService({
transaction: h.transaction,
agentRunBindingResolver: { resolve },
agentRunEventAppender: { appendManyInTransaction },
})
const agentRoute: AgentRunEventSubmissionService = service
const administrativeRoute: AdministrativeEventSubmissionService = service
const infrastructureRoute: InfrastructureTelemetryEventSubmissionService = service
expect([agentRoute, administrativeRoute, infrastructureRoute]).toEqual([
service,
service,
service,
])
await expect(
agentRoute.submit({
principal: workflowPrincipal,
events: [{ ...workflowInput }, { ...workflowInput, sourceEventId: 'activity-2' }],
})
).resolves.toEqual({ accepted: 2, replayed: 0 })
expect(resolve).toHaveBeenCalledTimes(2)
expect(h.transactionSpy).toHaveBeenCalledTimes(1)
expect(appendManyInTransaction).toHaveBeenCalledOnce()
expect(appendManyInTransaction).toHaveBeenCalledWith(
expect.objectContaining({ query: expect.any(Function) }),
workflowPrincipal,
expect.any(Array)
)
expect(
appendManyInTransaction.mock.calls[0]![2].map(entry => entry.input.sourceEventId)
).toEqual(['activity-1', 'activity-2'])
expect(appendManyInTransaction.mock.calls[0]![2].map(entry => entry.input.runId)).toEqual([
undefined,
undefined,
])
})
it('returns accepted and replayed totals from family append results', async () => {
const h = transactionHarness()
const appendManyInTransaction = vi
.fn()
.mockResolvedValue([appendResult('accepted'), appendResult('replayed')])
const service = new RouteTracingSubmissionService({
transaction: h.transaction,
agentRunBindingResolver: { resolve: vi.fn().mockResolvedValue(agentBinding) },
agentRunEventAppender: { appendManyInTransaction },
})
await expect(
service.submit({
principal: workflowPrincipal,
events: [{ ...workflowInput }, { ...workflowInput, sourceEventId: 'activity-2' }],
})
).resolves.toEqual({ accepted: 1, replayed: 1 })
})
it('uses the resolver batch path once for a multi-event agent submission', async () => {
const h = transactionHarness()
const resolve = vi.fn()
const resolveMany = vi.fn().mockResolvedValue([agentBinding, agentBinding])
const appendManyInTransaction = vi
.fn()
.mockResolvedValue([appendResult('accepted'), appendResult('accepted')])
const service = new RouteTracingSubmissionService({
transaction: h.transaction,
agentRunBindingResolver: { resolve, resolveMany },
agentRunEventAppender: { appendManyInTransaction },
})
await expect(
service.submit({
principal: workflowPrincipal,
events: [{ ...workflowInput }, { ...workflowInput, sourceEventId: 'activity-2' }],
})
).resolves.toEqual({ accepted: 2, replayed: 0 })
expect(resolveMany).toHaveBeenCalledOnce()
expect(resolve).not.toHaveBeenCalled()
})
it('fails closed when an administrative batch resolver returns the wrong size', async () => {
const h = transactionHarness()
const appendManyInTransaction = vi.fn()
const service = new RouteTracingSubmissionService({
transaction: h.transaction,
administrativeOperationBindingResolver: {
resolve: vi.fn(),
resolveMany: vi.fn().mockResolvedValue([]),
},
administrativeEventAppender: { appendManyInTransaction },
})
await expect(
service.submit({ principal: administrativePrincipal, events: [administrativeInput] })
).rejects.toThrow('trusted operation binding resolver returned an unexpected batch size')
expect(h.transactionSpy).not.toHaveBeenCalled()
expect(appendManyInTransaction).not.toHaveBeenCalled()
})
/**
* #329 — the classification trap. `rejectionReason` allowlists error classes
* plus statuses [400, 403]; an unlisted 409 falls through to
* `submission_failed`, which `operationalStatus.ts` treats as CRITICAL. A
* drift refusal is an operator-attributable rejection, not a control-api
* malfunction, so mislabelling it would trade a 403 log loop for a critical
* alert per drifted Host.
*/
it('classifies a generation-drift refusal as event_rejected, not submission_failed', async () => {
const lastError = vi.spyOn(governedTraceLastErrorTimestampSeconds, 'set')
const operationalError = vi.spyOn(governedTraceOperationalErrorsTotal, 'inc')
const rejected = vi.spyOn(governedTraceRejectedTotal, 'inc')
const h = transactionHarness()
const appendManyInTransaction = vi.fn()
const service = new RouteTracingSubmissionService({
transaction: h.transaction,
administrativeOperationBindingResolver: {
resolve: vi.fn(),
resolveMany: vi
.fn()
.mockResolvedValue([{ refusal: 'administrative_intent_generation_drift' }]),
},
administrativeEventAppender: { appendManyInTransaction },
})
const failure = service.submit({
principal: administrativePrincipal,
events: [administrativeInput],
})
await expect(failure).rejects.toBeInstanceOf(AdministrativeIntentGenerationDriftError)
await expect(failure).rejects.toMatchObject({
code: 'administrative_intent_generation_drift',
status: 409,
statusCode: 409,
eventIndex: 0,
})
// Liveness: the rejection path ran to completion — the event was counted
// as rejected and the error clock was stamped.
// Labels, not just the count: a regression that counted the refusal under
// the wrong label would leave a bare call-count assertion green while the
// dashboard built on it moved the event to another series.
expect(rejected).toHaveBeenCalledOnce()
expect(rejected).toHaveBeenCalledWith({
family: 'administrative',
source: 'host-context-controller',
type: 'linked_outcome',
})
expect(lastError).toHaveBeenCalledWith(
{ scope: 'administrative', reason: 'event_rejected' },
expect.any(Number)
)
expect(lastError).not.toHaveBeenCalledWith(
{ scope: 'administrative', reason: 'submission_failed' },
expect.any(Number)
)
expect(operationalError).not.toHaveBeenCalledWith({
scope: 'administrative',
reason: 'submission_failed',
})
expect(h.transactionSpy).not.toHaveBeenCalled()
expect(appendManyInTransaction).not.toHaveBeenCalled()
})
it('records accepted and replayed events with only bounded trace labels after commit', async () => {
const accepted = vi.spyOn(governedTraceAcceptedTotal, 'inc')
const replayed = vi.spyOn(governedTraceReplayedTotal, 'inc')
const h = transactionHarness()
const appendManyInTransaction = vi
.fn()
.mockResolvedValue([appendResult('accepted'), appendResult('replayed')])
const service = new RouteTracingSubmissionService({
transaction: h.transaction,
agentRunBindingResolver: { resolve: vi.fn().mockResolvedValue(agentBinding) },
agentRunEventAppender: { appendManyInTransaction },
})
await service.submit({
principal: workflowPrincipal,
events: [
{ ...workflowInput },
{ ...workflowInput, eventType: 'run_end', sourceEventId: 'activity-2' },
],
})
expect(accepted).toHaveBeenCalledWith(
{ family: 'agent_run', source: 'workflow-recipes', type: 'run_start' },
1
)
expect(replayed).toHaveBeenCalledWith(
{ family: 'agent_run', source: 'workflow-recipes', type: 'run_end' },
1
)
})
it('records rejected conflict events without placing trace identifiers in labels', async () => {
const rejected = vi.spyOn(governedTraceRejectedTotal, 'inc')
const conflicting = vi.spyOn(governedTraceConflictingTotal, 'inc')
const lastError = vi.spyOn(governedTraceLastErrorTimestampSeconds, 'set')
const h = transactionHarness()
const service = new RouteTracingSubmissionService({
transaction: h.transaction,
agentRunBindingResolver: { resolve: vi.fn().mockResolvedValue(agentBinding) },
agentRunEventAppender: {
appendManyInTransaction: vi
.fn()
.mockRejectedValue(
new TracingIdempotencyConflictError(
'agent_run',
'wrc_internal_control',
workflowInput.sourceEventId
)
),
},
})
await expect(
service.submit({ principal: workflowPrincipal, events: [{ ...workflowInput }] })
).rejects.toBeInstanceOf(TracingIdempotencyConflictError)
expect(rejected).toHaveBeenCalledWith({
family: 'agent_run',
source: 'workflow-recipes',
type: 'run_start',
})
expect(conflicting).toHaveBeenCalledWith({
family: 'agent_run',
source: 'workflow-recipes',
type: 'run_start',
})
expect(lastError).toHaveBeenCalledWith(
{ scope: 'agent_run', reason: 'idempotency_conflict' },
expect.any(Number)
)
expect(lastError).not.toHaveBeenCalledWith(
{ scope: 'agent_run', reason: 'event_rejected' },
expect.any(Number)
)
expect(JSON.stringify(conflicting.mock.calls)).not.toContain(workflowInput.runId)
})
it('does not start a transaction or append when a binding resolver rejects', async () => {
const rejected = vi.spyOn(governedTraceRejectedTotal, 'inc')
const lastError = vi.spyOn(governedTraceLastErrorTimestampSeconds, 'set')
const operationalError = vi.spyOn(governedTraceOperationalErrorsTotal, 'inc')
const h = transactionHarness()
const resolverError = new Error('trusted lookup failed')
const appendManyInTransaction = vi.fn()
const service = new RouteTracingSubmissionService({
transaction: h.transaction,
agentRunBindingResolver: { resolve: vi.fn().mockRejectedValue(resolverError) },
agentRunEventAppender: { appendManyInTransaction },
})
await expect(
service.submit({ principal: workflowPrincipal, events: [{ ...workflowInput }] })
).rejects.toBe(resolverError)
expect(h.transactionSpy).not.toHaveBeenCalled()
expect(appendManyInTransaction).not.toHaveBeenCalled()
expect(rejected).toHaveBeenCalledWith({
family: 'agent_run',
source: 'workflow-recipes',
type: 'run_start',
})
expect(operationalError).toHaveBeenCalledWith({
scope: 'agent_run',
reason: 'submission_failed',
})
expect(lastError).toHaveBeenCalledWith(
{ scope: 'agent_run', reason: 'submission_failed' },
expect.any(Number)
)
})
it('fails closed with a typed 403 error when no trusted resolver is configured', async () => {
const rejected = vi.spyOn(governedTraceRejectedTotal, 'inc')
const lastError = vi.spyOn(governedTraceLastErrorTimestampSeconds, 'set')
const h = transactionHarness()
const appendManyInTransaction = vi.fn()
const service = new RouteTracingSubmissionService({
transaction: h.transaction,
agentRunEventAppender: { appendManyInTransaction },
})
const failure = service.submit({ principal: workflowPrincipal, events: [{ ...workflowInput }] })
await expect(failure).rejects.toMatchObject({
name: 'TracingBindingUnavailableError',
code: 'tracing_binding_unavailable',
status: 403,
statusCode: 403,
bindingKind: 'run',
eventIndex: 0,
})
await expect(failure).rejects.toBeInstanceOf(TracingBindingUnavailableError)
expect(h.transactionSpy).not.toHaveBeenCalled()
expect(appendManyInTransaction).not.toHaveBeenCalled()
expect(rejected).toHaveBeenCalledWith({
family: 'agent_run',
source: 'workflow-recipes',
type: 'run_start',
})
expect(lastError).toHaveBeenCalledWith(
{ scope: 'agent_run', reason: 'event_rejected' },
expect.any(Number)
)
})
it.each([
['identity', { agentSub: 'body-controlled-agent' }],
['binding', { operationId: adminBinding.operationId }],
['money', { cost: 1.25 }],
])('rejects server-owned %s authority from the body before resolution', async (_kind, extra) => {
const rejected = vi.spyOn(governedTraceRejectedTotal, 'inc')
const lastError = vi.spyOn(governedTraceLastErrorTimestampSeconds, 'set')
const h = transactionHarness()
const resolve = vi.fn().mockResolvedValue(agentBinding)
const appendManyInTransaction = vi.fn()
const service = new RouteTracingSubmissionService({
transaction: h.transaction,
agentRunBindingResolver: { resolve },
agentRunEventAppender: { appendManyInTransaction },
})
await expect(
service.submit({
principal: workflowPrincipal,
events: [{ ...workflowInput, ...extra }],
})
).rejects.toBeInstanceOf(UnsafeTracingInputError)
expect(resolve).not.toHaveBeenCalled()
expect(h.transactionSpy).not.toHaveBeenCalled()
expect(appendManyInTransaction).not.toHaveBeenCalled()
expect(rejected).toHaveBeenCalledWith({
family: 'agent_run',
source: 'workflow-recipes',
type: 'run_start',
})
expect(lastError).toHaveBeenCalledWith(
{ scope: 'agent_run', reason: 'event_rejected' },
expect.any(Number)
)
})
it.each([
['accepts', 'host-uid-1', 1],
['rejects', 42, 0],
['rejects', '', 0],
] as const)(
'%s a Host lookup reference uid of %o (#691)',
async (_verdict, uid, expectedResolves) => {
const rejected = vi.spyOn(governedTraceRejectedTotal, 'inc')
const h = transactionHarness()
const resolve = vi.fn().mockResolvedValue(infraBinding)
const appendManyInTransaction = vi.fn().mockResolvedValue([appendResult('accepted')])
const service = new RouteTracingSubmissionService({
transaction: h.transaction,
infrastructureWorkloadBindingResolver: { resolve },
infrastructureTelemetryAppender: { appendManyInTransaction },
})
const submission = service.submit({
principal: infraPrincipal,
events: [
{
sourceEventId: 'reconcile-1',
occurredAt: NOW,
telemetryType: 'reconcile_outcome',
hostLookupReference: { name: 'chatllm', namespace: 'mcp-host', generation: 1, uid },
payload: { status: 'succeeded', reason_code: 'ready' },
},
],
})
if (expectedResolves === 1) {
await expect(submission).resolves.toEqual({ accepted: 1, replayed: 0 })
expect(resolve.mock.calls[0]![1].hostLookupReference).toEqual({
name: 'chatllm',
namespace: 'mcp-host',
generation: 1,
uid,
})
} else {
await expect(submission).rejects.toBeInstanceOf(InvalidTracingInputError)
// Liveness: the rejection was counted, so validation ran on this event.
expect(rejected).toHaveBeenCalled()
}
expect(resolve).toHaveBeenCalledTimes(expectedResolves)
}
)
it('names an unknown event field as not permitted, not as server-owned (#328)', async () => {
const rejected = vi.spyOn(governedTraceRejectedTotal, 'inc')
const h = transactionHarness()
const resolve = vi.fn().mockResolvedValue(infraBinding)
const service = new RouteTracingSubmissionService({
transaction: h.transaction,
infrastructureWorkloadBindingResolver: { resolve },
infrastructureTelemetryAppender: { appendManyInTransaction: vi.fn() },
})
await expect(
service.submit({
principal: infraPrincipal,
events: [
{
sourceEventId: 'reconcile-1',
occurredAt: NOW,
telemetryType: 'reconcile_outcome',
hostLookupReference: { name: 'chatllm', namespace: 'mcp-host', generation: 1 },
payload: { status: 'succeeded', reason_code: 'ready' },
gfsEvidence: 'rotated',
},
],
})
).rejects.toMatchObject({
code: 'unsafe_tracing_input',
reason: 'not_permitted',
message: 'tracing input field is not permitted: events[0].gfsEvidence',
})
// Liveness: the rejection was counted, so validation ran on this event.
expect(rejected).toHaveBeenCalled()
expect(resolve).not.toHaveBeenCalled()
})
})