Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript
10.4 KB
Raw Blame
import { afterEach, describe, expect, it, vi } from 'vitest'
import { readFileSync } from 'node:fs'
import { pool } from '../src/db.js'
import { getTracingOperationsRecentErrorSeconds } from '../src/services/tracing/operationalLimits.js'
import type { TracingOperationsLimits } from '../src/services/tracing/operations/contracts.js'
import type {
  MetricSample,
  TracingOperationsMetricState,
} from '../src/services/tracing/operations/metricState.js'
import { TracingOperationsSnapshotService } from '../src/services/tracing/operations/tracingOperationsSnapshotService.js'

const NOW = new Date('2026-07-13T12:00:00.000Z')
const LIMITS: TracingOperationsLimits = {
  bodyBytes: 524_288,
  eventsPerRequest: 100,
  maxInFlight: 32,
  ingestPoolMax: 4,
  readPoolMax: 3,
  poolConnectionTimeoutMs: 2_000,
  ingestStatementTimeoutMs: 5_000,
  readStatementTimeoutMs: 2_000,
  recentErrorSeconds: 300,
}

function sample(value: number, labels: MetricSample['labels'] = {}): MetricSample {
  return { value, labels }
}

function state(
  overrides: Partial<TracingOperationsMetricState> = {}
): TracingOperationsMetricState {
  return {
    accepted: [],
    replayed: [],
    rejected: [],
    conflicting: [],
    admission: [],
    operationalErrors: [],
    inFlight: [],
    poolConnections: [],
    poolRejections: [],
    poolStatementTimeouts: [],
    lastErrors: [],
    ...overrides,
  }
}

function service(metrics: TracingOperationsMetricState, limits: TracingOperationsLimits = LIMITS) {
  return new TracingOperationsSnapshotService({
    now: () => NOW,
    instanceStartedAt: '2026-07-13T11:00:00.000Z',
    readMetrics: async () => metrics,
    readLimits: () => limits,
  })
}

afterEach(() => {
  vi.restoreAllMocks()
  delete process.env.TRACING_OPERATIONS_RECENT_ERROR_SECONDS
})

describe('TracingOperationsSnapshotService', () => {
  it('keeps the snapshot module graph free of database, Kubernetes, filesystem, and network clients', () => {
    const sourceFiles = [
      '../src/services/tracing/operationalLimits.ts',
      '../src/services/tracing/operations/contracts.ts',
      '../src/services/tracing/operations/metricState.ts',
      '../src/services/tracing/operations/operationalStatus.ts',
      '../src/services/tracing/operations/tracingOperationsSnapshotService.ts',
    ]
    const forbiddenImport =
      /(?:from\s+|import\s*\()\s*['"](?:pg|node:(?:fs|http|https|net|tls)|[^'"]*(?:db|k8s|pools)\.js)['"]/

    for (const relativePath of sourceFiles) {
      const source = readFileSync(new URL(relativePath, import.meta.url), 'utf8')
      expect(source, relativePath).not.toMatch(forbiddenImport)
    }
  })

  it('returns an honest healthy since-restart snapshot from current metric objects', async () => {
    const snapshot = await service(
      state({
        accepted: [sample(4), sample(6)],
        replayed: [sample(2)],
        admission: [sample(9, { result: 'accepted', reason: 'none' })],
        poolConnections: [
          sample(2, { pool: 'ingest', state: 'active' }),
          sample(1, { pool: 'ingest', state: 'idle' }),
          sample(1, { pool: 'read', state: 'idle' }),
        ],
      })
    ).read()

    expect(snapshot).toMatchObject({
      generatedAt: NOW.toISOString(),
      instanceStartedAt: '2026-07-13T11:00:00.000Z',
      scope: 'control-api-instance',
      health: 'healthy',
      limits: LIMITS,
      ingestion: {
        acceptedEvents: 10,
        replayedEvents: 2,
        rejectedEvents: 0,
        conflictingEvents: 0,
        admissionRequests: 9,
        admissionRejected: 0,
        inFlight: 0,
      },
    })
    expect(snapshot.pools).toEqual([
      expect.objectContaining({ name: 'ingest', active: 2, idle: 1, waiting: 0 }),
      expect.objectContaining({ name: 'read', active: 0, idle: 1, waiting: 0 }),
    ])
    expect(snapshot.errors).toEqual([])
  })

  it('shows oversized requests with the canonical message, hard ceiling, and warning health', async () => {
    const occurredAtSeconds = NOW.getTime() / 1_000 - 30
    const snapshot = await service(
      state({
        admission: [
          sample(3, { family: 'agent_run', result: 'rejected', reason: 'body_too_large' }),
        ],
        operationalErrors: [sample(3, { scope: 'agent_run', reason: 'body_too_large' })],
        lastErrors: [sample(occurredAtSeconds, { scope: 'agent_run', reason: 'body_too_large' })],
      })
    ).read()

    expect(snapshot.health).toBe('warning')
    expect(snapshot.errors).toEqual([
      {
        reason: 'body_too_large',
        message: 'Tracing request exceeded 512 KiB and was rejected.',
        severity: 'warning',
        countSinceRestart: 3,
        lastOccurredAt: '2026-07-13T11:59:30.000Z',
        relatedSetting: '512 KiB hard ceiling (not ENV-configurable)',
        effectiveValue: 524_288,
        operatorAction:
          'Reduce payload fields or split the request; do not raise the hard ceiling.',
      },
    ])
  })

  it('keeps informational and old errors visible without falsely escalating current health', async () => {
    const snapshot = await service(
      state({
        admission: [
          sample(1, { result: 'rejected', reason: 'invalid_json' }),
          sample(2, { result: 'rejected', reason: 'batch_too_large' }),
        ],
        operationalErrors: [
          sample(1, { scope: 'agent_run', reason: 'invalid_json' }),
          sample(2, { scope: 'agent_run', reason: 'batch_too_large' }),
        ],
        lastErrors: [
          sample(NOW.getTime() / 1_000 - 10, { scope: 'agent_run', reason: 'invalid_json' }),
          sample(NOW.getTime() / 1_000 - 600, {
            scope: 'agent_run',
            reason: 'batch_too_large',
          }),
        ],
      })
    ).read()

    expect(snapshot.health).toBe('healthy')
    expect(snapshot.errors.map(error => error.reason)).toEqual(['batch_too_large', 'invalid_json'])
  })

  it('classifies recent pool failure and a saturated request budget as critical', async () => {
    const snapshot = await service(
      state({
        inFlight: [sample(32)],
        poolConnections: [sample(1, { pool: 'read', state: 'waiting' })],
        poolRejections: [sample(2, { pool: 'read' })],
        poolStatementTimeouts: [sample(1, { pool: 'ingest' })],
        operationalErrors: [
          sample(2, { scope: 'pool', reason: 'pool_rejected' }),
          sample(1, { scope: 'pool', reason: 'statement_timeout' }),
        ],
        lastErrors: [
          sample(NOW.getTime() / 1_000 - 5, { scope: 'pool', reason: 'pool_rejected' }),
          sample(NOW.getTime() / 1_000 - 4, { scope: 'pool', reason: 'statement_timeout' }),
        ],
      })
    ).read()

    expect(snapshot.health).toBe('critical')
    expect(snapshot.pools[1]).toMatchObject({ name: 'read', waiting: 1, rejectedSinceRestart: 2 })
    expect(snapshot.errors.map(error => [error.reason, error.severity])).toEqual([
      ['pool_rejected', 'critical'],
      ['statement_timeout', 'critical'],
    ])
  })

  it('keeps operational occurrence counts independent from atomic rejected-event totals', async () => {
    const snapshot = await service(
      state({
        rejected: [sample(100)],
        conflicting: [sample(1)],
        operationalErrors: [
          sample(2, { scope: 'agent_run', reason: 'event_rejected' }),
          sample(1, { scope: 'agent_run', reason: 'idempotency_conflict' }),
        ],
        lastErrors: [
          sample(NOW.getTime() / 1_000, { scope: 'agent_run', reason: 'event_rejected' }),
          sample(NOW.getTime() / 1_000, {
            scope: 'agent_run',
            reason: 'idempotency_conflict',
          }),
        ],
      })
    ).read()

    expect(
      snapshot.errors.find(error => error.reason === 'event_rejected')?.countSinceRestart
    ).toBe(2)
    expect(
      snapshot.errors.find(error => error.reason === 'idempotency_conflict')?.countSinceRestart
    ).toBe(1)
    expect(snapshot.ingestion).toMatchObject({ rejectedEvents: 100, conflictingEvents: 1 })
  })

  it('exposes closed binding and prompt capture reasons without sensitive context', async () => {
    const reasons = [
      'attribution_binding_unavailable',
      'attribution_binding_conflict',
      'prompt_history_disabled',
      'prompt_history_key_unavailable',
      'prompt_history_rejected',
    ] as const
    const snapshot = await service(
      state({
        operationalErrors: reasons.map(reason => sample(1, { scope: 'agent_run', reason })),
        lastErrors: reasons.map(reason =>
          sample(NOW.getTime() / 1_000, {
            scope: 'agent_run',
            reason,
          })
        ),
      })
    ).read()

    expect(snapshot.health).toBe('warning')
    expect(snapshot.errors.map(error => error.reason).sort()).toEqual([...reasons].sort())
    expect(JSON.stringify(snapshot)).not.toMatch(
      /runId|sessionId|hostRef|userId|approvalRequestId|promptText/
    )
  })

  it('treats a recent unexpected submission failure as critical without exposing raw errors', async () => {
    const snapshot = await service(
      state({
        operationalErrors: [sample(1, { scope: 'agent_run', reason: 'submission_failed' })],
        lastErrors: [
          sample(NOW.getTime() / 1_000 - 2, {
            scope: 'agent_run',
            reason: 'submission_failed',
          }),
        ],
      })
    ).read()

    expect(snapshot.health).toBe('critical')
    expect(snapshot.errors).toEqual([
      expect.objectContaining({
        reason: 'submission_failed',
        message: 'A tracing submission failed after request admission.',
        countSinceRestart: 1,
      }),
    ])
    expect(JSON.stringify(snapshot)).not.toContain('stack')
  })

  it('reads only in-process metrics within a bounded response and latency budget', async () => {
    const fetchSpy = vi.spyOn(globalThis, 'fetch')
    const querySpy = vi.spyOn(pool, 'query')
    const startedAt = performance.now()

    const snapshot = await new TracingOperationsSnapshotService().read()

    expect(performance.now() - startedAt).toBeLessThan(100)
    expect(Buffer.byteLength(JSON.stringify(snapshot))).toBeLessThan(16 * 1024)
    expect(snapshot.errors.length).toBeLessThanOrEqual(10)
    expect(fetchSpy).not.toHaveBeenCalled()
    expect(querySpy).not.toHaveBeenCalled()
  })

  it('uses one bounded health-recency ENV and falls back for invalid values', () => {
    process.env.TRACING_OPERATIONS_RECENT_ERROR_SECONDS = '900'
    expect(getTracingOperationsRecentErrorSeconds()).toBe(900)
    process.env.TRACING_OPERATIONS_RECENT_ERROR_SECONDS = '999999'
    expect(getTracingOperationsRecentErrorSeconds()).toBe(300)
  })
})