Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
import { type Mock, beforeEach, describe, expect, it, vi } from 'vitest'
import { LlmPortAdapter } from '../../../core/adapters/llmPortAdapter'
import { LlmError, LlmErrorCode } from '../../../core/errors'
import { CodexProxyError } from '../../codexLlmProxyClient'
import { CodexSubscriptionProvider } from '../../codexSubscription'
import { GrokSubscriptionProvider } from '../../grokSubscription'
import { type ClassifiedLike, FailoverEngine } from '../engine'
import type { FailoverSwitchEvent, LlmPolicy, ModelPair } from '../types'
const PRIMARY: ModelPair = { provider: 'claude', model: 'claude-sonnet-4-6' }
function policy(overrides: Partial<LlmPolicy> = {}): LlmPolicy {
return {
cooldownSeconds: 300,
triggerOn: ['insufficient_quota', 'auth', 'provider_unavailable', 'rate_limited'],
fallbacks: [{ provider: 'openai', model: 'gpt-5.4' }],
...overrides,
}
}
/** A classifier that reads `LlmError` fields (as the agent glue does). */
function classify(err: unknown): { code: LlmErrorCode; retryable: boolean } | null {
if (err instanceof LlmError) return { code: err.code as LlmErrorCode, retryable: err.retryable }
return null
}
function llmError(code: LlmErrorCode, retryable: boolean): LlmError {
return new LlmError('boom', 'claude', code, retryable)
}
describe('FailoverEngine', () => {
let metricInc: Mock<(labels: { from: string; to: string; reason: string }) => void>
let onSwitch: Mock<(event: FailoverSwitchEvent) => void>
let now: number
beforeEach(() => {
metricInc = vi.fn()
onSwitch = vi.fn()
now = 1_000_000
})
function engine(p: LlmPolicy = policy()): FailoverEngine {
return new FailoverEngine(p, {
now: () => now,
onSwitch: e => onSwitch(e),
metricInc: l => metricInc(l),
})
}
it('serves the primary and does not switch when the call succeeds', async () => {
const e = engine()
const build = vi.fn(t => () => Promise.resolve(`ok:${t.kind}`))
const res = await e.run(PRIMARY, build, classify)
expect(res).toBe('ok:primary')
expect(metricInc).not.toHaveBeenCalled()
expect(e.servedBy()).toEqual({ ...PRIMARY, fallback: false })
})
it.each([
[LlmErrorCode.InsufficientQuota, false, 'insufficient_quota'],
[LlmErrorCode.AuthenticationFailed, false, 'auth'],
[LlmErrorCode.RateLimited, true, 'rate_limited'],
[LlmErrorCode.ModelOverloaded, true, 'provider_unavailable'],
[LlmErrorCode.ApiCallFailed, true, 'provider_unavailable'],
])(
'each eligible class (%s) triggers a switch to the fallback',
async (code, retryable, reason) => {
const e = engine()
const res = await e.run(
PRIMARY,
t =>
t.kind === 'primary'
? () => Promise.reject(llmError(code, retryable))
: () => Promise.resolve('fallback-served'),
classify
)
expect(res).toBe('fallback-served')
expect(metricInc).toHaveBeenCalledWith({
from: 'claude/claude-sonnet-4-6',
to: 'openai/gpt-5.4',
reason,
})
expect(onSwitch).toHaveBeenCalledWith({
from: PRIMARY,
to: { provider: 'openai', model: 'gpt-5.4' },
reason,
})
expect(e.servedBy()).toEqual({ provider: 'openai', model: 'gpt-5.4', fallback: true })
}
)
it('does NOT trigger on a non-eligible error (400/validation) — propagates', async () => {
const e = engine()
const err = llmError(LlmErrorCode.ApiCallFailed, false) // 400 → not retryable
await expect(
e.run(
PRIMARY,
t => (t.kind === 'primary' ? () => Promise.reject(err) : () => Promise.resolve('nope')),
classify
)
).rejects.toBe(err)
expect(metricInc).not.toHaveBeenCalled()
})
it('keeps the primary eligible after a local admission failure', async () => {
const e = engine()
const local = llmError(LlmErrorCode.ApiCallFailed, false)
const firstBuild = vi.fn(() => () => Promise.reject(local))
await expect(e.run(PRIMARY, firstBuild, classify)).rejects.toBe(local)
expect(firstBuild).toHaveBeenCalledTimes(1)
expect(onSwitch).not.toHaveBeenCalled()
expect(metricInc).not.toHaveBeenCalled()
const nextBuild = vi.fn(target => () => Promise.resolve(target.kind))
await expect(e.run(PRIMARY, nextBuild, classify)).resolves.toBe('primary')
expect(nextBuild).toHaveBeenCalledTimes(1)
expect(e.servedBy()).toEqual({ ...PRIMARY, fallback: false })
})
it('respects a restricted triggerOn: an out-of-set class propagates', async () => {
const e = engine(policy({ triggerOn: ['auth'] }))
const err = llmError(LlmErrorCode.RateLimited, true) // rate_limited not in triggerOn
await expect(
e.run(
PRIMARY,
t => (t.kind === 'primary' ? () => Promise.reject(err) : () => Promise.resolve('nope')),
classify
)
).rejects.toBe(err)
expect(metricInc).not.toHaveBeenCalled()
})
it('advances through an ordered list when the first fallback also fails', async () => {
const e = engine(
policy({
fallbacks: [
{ provider: 'openai', model: 'gpt-5.4' },
{ provider: 'zai', model: 'glm-5.1' },
],
})
)
const res = await e.run(
PRIMARY,
t => {
if (t.kind === 'primary')
return () => Promise.reject(llmError(LlmErrorCode.RateLimited, true))
if (t.kind === 'fallback' && t.index === 0)
return () => Promise.reject(llmError(LlmErrorCode.ModelOverloaded, true))
return () => Promise.resolve('second-fallback')
},
classify
)
expect(res).toBe('second-fallback')
expect(metricInc).toHaveBeenCalledTimes(2)
expect(metricInc).toHaveBeenNthCalledWith(1, {
from: 'claude/claude-sonnet-4-6',
to: 'openai/gpt-5.4',
reason: 'rate_limited',
})
expect(metricInc).toHaveBeenNthCalledWith(2, {
from: 'openai/gpt-5.4',
to: 'zai/glm-5.1',
reason: 'provider_unavailable',
})
expect(e.servedBy()).toEqual({ provider: 'zai', model: 'glm-5.1', fallback: true })
})
it('skips an unconstructible fallback (null builder) without a phantom switch', async () => {
const e = engine(
policy({
fallbacks: [
{ provider: 'openai', model: 'gpt-5.4' }, // unconstructible
{ provider: 'zai', model: 'glm-5.1' },
],
})
)
const res = await e.run(
PRIMARY,
t => {
if (t.kind === 'primary')
return () => Promise.reject(llmError(LlmErrorCode.RateLimited, true))
if (t.kind === 'fallback' && t.index === 0) return null // skip
return () => Promise.resolve('third')
},
classify
)
expect(res).toBe('third')
// Exactly one switch: primary → the SECOND fallback (the skipped one emits none).
expect(metricInc).toHaveBeenCalledTimes(1)
expect(metricInc).toHaveBeenCalledWith({
from: 'claude/claude-sonnet-4-6',
to: 'zai/glm-5.1',
reason: 'rate_limited',
})
})
describe('Codex request limits', () => {
// The primary runs the real Codex provider behind the port adapter, so the
// error the engine sees is produced by CodexSubscriptionProvider.classifyError.
function codexPrimary(stream: ReturnType<typeof vi.fn>) {
const adapter = new LlmPortAdapter(
new CodexSubscriptionProvider('gpt-5.3-codex', {
authorizer: {
authorize: vi.fn().mockResolvedValue({
providerAttemptId: 'attempt-1',
requestHash: 'a'.repeat(64),
executionTicket: 'ticket-123456',
expiresAt: '2026-08-20T10:00:00.000Z',
}),
},
proxy: { stream },
attemptContext: () => ({
policyRevision: 1,
policyHash: 'b'.repeat(64),
hostRef: 'chatllm',
}),
} as never),
'gpt-5.3-codex',
'codex-subscription'
)
const thrown: unknown[] = []
const call = vi.fn(
(): Promise<unknown> =>
adapter
.completeWithTools({
messages: [{ role: 'user', content: 'hi' }],
tools: [{ name: 'echo', description: 'echo', parameters: {} }],
})
.catch((err: unknown) => {
thrown.push(err)
throw err
})
)
return { call, thrown }
}
it('propagates a tool-call limit without switching or setting a cooldown', async () => {
const stream = vi.fn().mockResolvedValue({
text: '',
outcome: 'success',
toolCalls: Array.from({ length: 257 }, (_, index) => ({
id: `call-${index}`,
name: 'echo',
arguments: {},
})),
})
const primary = codexPrimary(stream)
const fallbackBuild = vi.fn(() => () => Promise.resolve('fallback-served'))
const e = engine()
const failure = await e
.run(PRIMARY, t => (t.kind === 'primary' ? primary.call : fallbackBuild()), classify)
.catch((err: unknown) => err)
// Liveness witness: the primary ran once and really reached the proxy.
expect(primary.call).toHaveBeenCalledTimes(1)
expect(stream).toHaveBeenCalledTimes(1)
expect(primary.thrown).toHaveLength(1)
expect(failure).toBe(primary.thrown[0])
expect(failure).toBeInstanceOf(LlmError)
expect(failure).toMatchObject({
code: LlmErrorCode.ToolCallLimitExceeded,
retryable: false,
})
expect(fallbackBuild).not.toHaveBeenCalled()
expect(metricInc).not.toHaveBeenCalled()
// No cooldown was set: the next call starts at the primary again.
const nextPrimary = vi.fn(() => Promise.resolve('primary-again'))
const next = await e.run(
PRIMARY,
t => (t.kind === 'primary' ? nextPrimary : () => Promise.resolve('fallback')),
classify
)
expect(next).toBe('primary-again')
expect(nextPrimary).toHaveBeenCalledTimes(1)
})
it('still fails over a Codex provider outage to the fallback', async () => {
const stream = vi
.fn()
.mockRejectedValue(new CodexProxyError('provider_unavailable', 'upstream 5xx'))
const primary = codexPrimary(stream)
const e = engine()
const res = await e.run(
PRIMARY,
t => (t.kind === 'primary' ? primary.call : () => Promise.resolve('fallback-served')),
classify
)
expect(primary.call).toHaveBeenCalledTimes(1)
expect(primary.thrown[0]).toMatchObject({
code: LlmErrorCode.ModelOverloaded,
retryable: true,
})
expect(res).toBe('fallback-served')
expect(metricInc).toHaveBeenCalledWith({
from: 'claude/claude-sonnet-4-6',
to: 'openai/gpt-5.4',
reason: 'provider_unavailable',
})
})
// A stream cut at its total cap spent the attempt's whole budget; the
// fallback would spend it again on the same request.
it('propagates a stream duration cap without switching or setting a cooldown', async () => {
const stream = vi
.fn()
.mockRejectedValue(
new CodexProxyError(
'stream_duration_exceeded',
'proxy stream failed with 504 (stream_duration_exceeded)'
)
)
const primary = codexPrimary(stream)
const fallbackBuild = vi.fn(() => () => Promise.resolve('fallback-served'))
const e = engine()
const failure = await e
.run(PRIMARY, t => (t.kind === 'primary' ? primary.call : fallbackBuild()), classify)
.catch((err: unknown) => err)
// Liveness witness: the primary ran once and really reached the proxy.
expect(primary.call).toHaveBeenCalledTimes(1)
expect(stream).toHaveBeenCalledTimes(1)
expect(failure).toBe(primary.thrown[0])
expect(failure).toMatchObject({
code: LlmErrorCode.StreamDurationExceeded,
retryable: false,
})
expect(fallbackBuild).not.toHaveBeenCalled()
expect(metricInc).not.toHaveBeenCalled()
// No cooldown was set: the next call starts at the primary again.
const nextPrimary = vi.fn(() => Promise.resolve('primary-again'))
const next = await e.run(
PRIMARY,
t => (t.kind === 'primary' ? nextPrimary : () => Promise.resolve('fallback')),
classify
)
expect(next).toBe('primary-again')
expect(nextPrimary).toHaveBeenCalledTimes(1)
})
// The same claim for Grok. This is issue #680 in its most direct form:
// before the taxonomy, the limit classified as ModelOverloaded/retryable,
// failover fired, and the fallback answered from partial tool results.
it('propagates a Grok tool-call limit without switching or setting a cooldown', async () => {
const stream = vi.fn().mockResolvedValue({
text: '',
outcome: 'success',
toolCalls: Array.from({ length: 257 }, (_, index) => ({
id: `call-${index}`,
name: 'echo',
arguments: {},
})),
})
const adapter = new LlmPortAdapter(
new GrokSubscriptionProvider('grok-4.6', {
authorizer: {
authorize: vi.fn().mockResolvedValue({
providerAttemptId: 'attempt-1',
requestHash: 'a'.repeat(64),
executionTicket: 'ticket-123456',
expiresAt: '2026-08-20T10:00:00.000Z',
}),
},
proxy: { stream },
attemptContext: () => ({
policyRevision: 1,
policyHash: 'b'.repeat(64),
hostRef: 'chatllm',
}),
} as never),
'grok-4.6',
'grok-subscription'
)
const thrown: unknown[] = []
const call = vi.fn(
(): Promise<unknown> =>
adapter
.completeWithTools({
messages: [{ role: 'user', content: 'hi' }],
tools: [{ name: 'echo', description: 'echo', parameters: {} }],
})
.catch((err: unknown) => {
thrown.push(err)
throw err
})
)
const fallbackBuild = vi.fn(() => () => Promise.resolve('fallback-served'))
const e = engine()
const failure = await e
.run(PRIMARY, t => (t.kind === 'primary' ? call : fallbackBuild()), classify)
.catch((err: unknown) => err)
// Liveness witness: the primary ran once and really reached the proxy.
expect(call).toHaveBeenCalledTimes(1)
expect(stream).toHaveBeenCalledTimes(1)
expect(thrown).toHaveLength(1)
expect(failure).toBe(thrown[0])
expect(failure).toBeInstanceOf(LlmError)
expect(failure).toMatchObject({
code: LlmErrorCode.ToolCallLimitExceeded,
retryable: false,
})
expect(fallbackBuild).not.toHaveBeenCalled()
expect(metricInc).not.toHaveBeenCalled()
})
})
it('is sticky: while cooling, the next call starts at the fallback (no primary retry)', async () => {
const e = engine()
// Call 1: primary fails eligibly → cooldown set, fallback serves.
await e.run(
PRIMARY,
t =>
t.kind === 'primary'
? () => Promise.reject(llmError(LlmErrorCode.RateLimited, true))
: () => Promise.resolve('fb1'),
classify
)
// Call 2 within the cooldown window: primary must NOT be attempted.
const primaryBuild = vi.fn(() => () => Promise.resolve('primary-should-not-run'))
const res = await e.run(
PRIMARY,
t => (t.kind === 'primary' ? primaryBuild() : () => Promise.resolve('fb2')),
classify
)
expect(res).toBe('fb2')
expect(primaryBuild).not.toHaveBeenCalled()
// No new switch recorded on call 2 (already on the fallback).
expect(metricInc).toHaveBeenCalledTimes(1)
})
it('lazy recovery: after the cooldown expires the primary is retried', async () => {
const e = engine(policy({ cooldownSeconds: 300 }))
await e.run(
PRIMARY,
t =>
t.kind === 'primary'
? () => Promise.reject(llmError(LlmErrorCode.RateLimited, true))
: () => Promise.resolve('fb'),
classify
)
expect(e.servedBy()?.fallback).toBe(true)
// Advance past the cooldown.
now += 300_000 + 1
const res = await e.run(PRIMARY, () => () => Promise.resolve('primary-recovered'), classify)
expect(res).toBe('primary-recovered')
expect(e.servedBy()).toEqual({ ...PRIMARY, fallback: false })
})
it('exhausted list propagates the last error (canned fallback is the last resort)', async () => {
const e = engine()
const lastErr = llmError(LlmErrorCode.ModelOverloaded, true)
await expect(
e.run(
PRIMARY,
t =>
t.kind === 'primary'
? () => Promise.reject(llmError(LlmErrorCode.RateLimited, true))
: () => Promise.reject(lastErr),
classify
)
).rejects.toBe(lastErr)
})
it('clearCooldown lets the next call retry the primary immediately (key recovery)', async () => {
const e = engine()
await e.run(
PRIMARY,
t =>
t.kind === 'primary'
? () => Promise.reject(llmError(LlmErrorCode.AuthenticationFailed, false))
: () => Promise.resolve('fb'),
classify
)
// Without clearing, the primary would be skipped (cooling) — clear it.
e.clearCooldown()
const res = await e.run(PRIMARY, () => () => Promise.resolve('primary-immediately'), classify)
expect(res).toBe('primary-immediately')
expect(e.servedBy()).toEqual({ ...PRIMARY, fallback: false })
})
it('passes the correct FULL-list index to the builder as it advances (index alignment)', async () => {
const e = engine(
policy({
fallbacks: [
{ provider: 'openai', model: 'gpt-5.4' }, // index 0 — unconstructible
{ provider: 'zai', model: 'glm-5.1' }, // index 1 — serves
],
})
)
const seenIndexes: number[] = []
const res = await e.run(
PRIMARY,
t => {
if (t.kind === 'primary')
return () => Promise.reject(llmError(LlmErrorCode.RateLimited, true))
seenIndexes.push(t.index)
if (t.index === 0) return null // unconstructible → skip
return () => Promise.resolve(`served-${t.index}`)
},
classify
)
expect(res).toBe('served-1')
// The builder saw both full-list indices in order (0 skipped, 1 served).
expect(seenIndexes).toEqual([0, 1])
})
it('honours an Attempt.servedModel override for servedBy + the metric to-label', async () => {
// FIX-2: a same-provider fallback serves the SESSION model, not entry.model.
// The builder returns { run, servedModel } and the engine must use that model
// for servedBy and the `to` metric label instead of the target's entry model.
const e = engine(policy({ fallbacks: [{ provider: 'claude', model: 'claude-haiku-4-5' }] }))
const res = await e.run(
PRIMARY,
t =>
t.kind === 'primary'
? () => Promise.reject(llmError(LlmErrorCode.RateLimited, true))
: { run: () => Promise.resolve('served'), servedModel: 'claude-sonnet-4-6' },
classify
)
expect(res).toBe('served')
expect(e.servedBy()).toEqual({ provider: 'claude', model: 'claude-sonnet-4-6', fallback: true })
expect(metricInc).toHaveBeenCalledWith({
from: 'claude/claude-sonnet-4-6',
to: 'claude/claude-sonnet-4-6',
reason: 'rate_limited',
})
})
it('setPolicy resets the sticky cooldown', async () => {
const e = engine()
await e.run(
PRIMARY,
t =>
t.kind === 'primary'
? () => Promise.reject(llmError(LlmErrorCode.RateLimited, true))
: () => Promise.resolve('fb'),
classify
)
e.setPolicy(policy())
expect(e.servedBy()).toBeNull()
// After reset the primary is attempted again immediately.
const res = await e.run(PRIMARY, () => () => Promise.resolve('primary'), classify)
expect(res).toBe('primary')
})
it('assigns a distinct providerAttemptIndex to each physical attempt', async () => {
const e = engine(
policy({
fallbacks: [
{ provider: 'openai', model: 'gpt-5.4' },
{ provider: 'zai', model: 'glm-5.1' },
],
})
)
const indexes: number[] = []
await e.run(
PRIMARY,
(t, physical) => {
indexes.push(physical.providerAttemptIndex)
if (t.kind === 'primary')
return () => Promise.reject(llmError(LlmErrorCode.RateLimited, true))
if (t.kind === 'fallback' && t.index === 0)
return () => Promise.reject(llmError(LlmErrorCode.ModelOverloaded, true))
return () => Promise.resolve('ok')
},
classify
)
expect(indexes).toEqual([1, 2, 3])
})
it('treats budget_denied as terminal unless the policy opts in', async () => {
const err = new LlmError(
'budget denied',
'codex-subscription',
LlmErrorCode.InsufficientQuota,
false,
undefined,
undefined,
'budget_denied'
)
const classifyWithCode = (caught: unknown): ClassifiedLike | null => {
if (caught instanceof LlmError) {
return {
code: caught.code as LlmErrorCode,
retryable: caught.retryable,
providerCode: caught.providerCode,
}
}
return classify(caught)
}
const denied = engine()
await expect(
denied.run(
PRIMARY,
t => (t.kind === 'primary' ? () => Promise.reject(err) : () => Promise.resolve('nope')),
classifyWithCode
)
).rejects.toBe(err)
const optedIn = engine(policy({ budgetDeniedFailover: true }))
await expect(
optedIn.run(
PRIMARY,
t => (t.kind === 'primary' ? () => Promise.reject(err) : () => Promise.resolve('fallback')),
classifyWithCode
)
).resolves.toBe('fallback')
})
})