Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript
21.5 KB
Raw Blame
import { type Mock, beforeEach, describe, expect, it, vi } from 'vitest'
import { LlmPortAdapter } from '../../../core/adapters/llmPortAdapter'
import { LlmError, LlmErrorCode } from '../../../core/errors'
import { CodexProxyError } from '../../codexLlmProxyClient'
import { CodexSubscriptionProvider } from '../../codexSubscription'
import { GrokSubscriptionProvider } from '../../grokSubscription'
import { type ClassifiedLike, FailoverEngine } from '../engine'
import type { FailoverSwitchEvent, LlmPolicy, ModelPair } from '../types'

const PRIMARY: ModelPair = { provider: 'claude', model: 'claude-sonnet-4-6' }

function policy(overrides: Partial<LlmPolicy> = {}): LlmPolicy {
  return {
    cooldownSeconds: 300,
    triggerOn: ['insufficient_quota', 'auth', 'provider_unavailable', 'rate_limited'],
    fallbacks: [{ provider: 'openai', model: 'gpt-5.4' }],
    ...overrides,
  }
}

/** A classifier that reads `LlmError` fields (as the agent glue does). */
function classify(err: unknown): { code: LlmErrorCode; retryable: boolean } | null {
  if (err instanceof LlmError) return { code: err.code as LlmErrorCode, retryable: err.retryable }
  return null
}

function llmError(code: LlmErrorCode, retryable: boolean): LlmError {
  return new LlmError('boom', 'claude', code, retryable)
}

describe('FailoverEngine', () => {
  let metricInc: Mock<(labels: { from: string; to: string; reason: string }) => void>
  let onSwitch: Mock<(event: FailoverSwitchEvent) => void>
  let now: number

  beforeEach(() => {
    metricInc = vi.fn()
    onSwitch = vi.fn()
    now = 1_000_000
  })

  function engine(p: LlmPolicy = policy()): FailoverEngine {
    return new FailoverEngine(p, {
      now: () => now,
      onSwitch: e => onSwitch(e),
      metricInc: l => metricInc(l),
    })
  }

  it('serves the primary and does not switch when the call succeeds', async () => {
    const e = engine()
    const build = vi.fn(t => () => Promise.resolve(`ok:${t.kind}`))
    const res = await e.run(PRIMARY, build, classify)
    expect(res).toBe('ok:primary')
    expect(metricInc).not.toHaveBeenCalled()
    expect(e.servedBy()).toEqual({ ...PRIMARY, fallback: false })
  })

  it.each([
    [LlmErrorCode.InsufficientQuota, false, 'insufficient_quota'],
    [LlmErrorCode.AuthenticationFailed, false, 'auth'],
    [LlmErrorCode.RateLimited, true, 'rate_limited'],
    [LlmErrorCode.ModelOverloaded, true, 'provider_unavailable'],
    [LlmErrorCode.ApiCallFailed, true, 'provider_unavailable'],
  ])(
    'each eligible class (%s) triggers a switch to the fallback',
    async (code, retryable, reason) => {
      const e = engine()
      const res = await e.run(
        PRIMARY,
        t =>
          t.kind === 'primary'
            ? () => Promise.reject(llmError(code, retryable))
            : () => Promise.resolve('fallback-served'),
        classify
      )
      expect(res).toBe('fallback-served')
      expect(metricInc).toHaveBeenCalledWith({
        from: 'claude/claude-sonnet-4-6',
        to: 'openai/gpt-5.4',
        reason,
      })
      expect(onSwitch).toHaveBeenCalledWith({
        from: PRIMARY,
        to: { provider: 'openai', model: 'gpt-5.4' },
        reason,
      })
      expect(e.servedBy()).toEqual({ provider: 'openai', model: 'gpt-5.4', fallback: true })
    }
  )

  it('does NOT trigger on a non-eligible error (400/validation) — propagates', async () => {
    const e = engine()
    const err = llmError(LlmErrorCode.ApiCallFailed, false) // 400 → not retryable
    await expect(
      e.run(
        PRIMARY,
        t => (t.kind === 'primary' ? () => Promise.reject(err) : () => Promise.resolve('nope')),
        classify
      )
    ).rejects.toBe(err)
    expect(metricInc).not.toHaveBeenCalled()
  })

  it('keeps the primary eligible after a local admission failure', async () => {
    const e = engine()
    const local = llmError(LlmErrorCode.ApiCallFailed, false)
    const firstBuild = vi.fn(() => () => Promise.reject(local))
    await expect(e.run(PRIMARY, firstBuild, classify)).rejects.toBe(local)
    expect(firstBuild).toHaveBeenCalledTimes(1)
    expect(onSwitch).not.toHaveBeenCalled()
    expect(metricInc).not.toHaveBeenCalled()

    const nextBuild = vi.fn(target => () => Promise.resolve(target.kind))
    await expect(e.run(PRIMARY, nextBuild, classify)).resolves.toBe('primary')
    expect(nextBuild).toHaveBeenCalledTimes(1)
    expect(e.servedBy()).toEqual({ ...PRIMARY, fallback: false })
  })

  it('respects a restricted triggerOn: an out-of-set class propagates', async () => {
    const e = engine(policy({ triggerOn: ['auth'] }))
    const err = llmError(LlmErrorCode.RateLimited, true) // rate_limited not in triggerOn
    await expect(
      e.run(
        PRIMARY,
        t => (t.kind === 'primary' ? () => Promise.reject(err) : () => Promise.resolve('nope')),
        classify
      )
    ).rejects.toBe(err)
    expect(metricInc).not.toHaveBeenCalled()
  })

  it('advances through an ordered list when the first fallback also fails', async () => {
    const e = engine(
      policy({
        fallbacks: [
          { provider: 'openai', model: 'gpt-5.4' },
          { provider: 'zai', model: 'glm-5.1' },
        ],
      })
    )
    const res = await e.run(
      PRIMARY,
      t => {
        if (t.kind === 'primary')
          return () => Promise.reject(llmError(LlmErrorCode.RateLimited, true))
        if (t.kind === 'fallback' && t.index === 0)
          return () => Promise.reject(llmError(LlmErrorCode.ModelOverloaded, true))
        return () => Promise.resolve('second-fallback')
      },
      classify
    )
    expect(res).toBe('second-fallback')
    expect(metricInc).toHaveBeenCalledTimes(2)
    expect(metricInc).toHaveBeenNthCalledWith(1, {
      from: 'claude/claude-sonnet-4-6',
      to: 'openai/gpt-5.4',
      reason: 'rate_limited',
    })
    expect(metricInc).toHaveBeenNthCalledWith(2, {
      from: 'openai/gpt-5.4',
      to: 'zai/glm-5.1',
      reason: 'provider_unavailable',
    })
    expect(e.servedBy()).toEqual({ provider: 'zai', model: 'glm-5.1', fallback: true })
  })

  it('skips an unconstructible fallback (null builder) without a phantom switch', async () => {
    const e = engine(
      policy({
        fallbacks: [
          { provider: 'openai', model: 'gpt-5.4' }, // unconstructible
          { provider: 'zai', model: 'glm-5.1' },
        ],
      })
    )
    const res = await e.run(
      PRIMARY,
      t => {
        if (t.kind === 'primary')
          return () => Promise.reject(llmError(LlmErrorCode.RateLimited, true))
        if (t.kind === 'fallback' && t.index === 0) return null // skip
        return () => Promise.resolve('third')
      },
      classify
    )
    expect(res).toBe('third')
    // Exactly one switch: primary → the SECOND fallback (the skipped one emits none).
    expect(metricInc).toHaveBeenCalledTimes(1)
    expect(metricInc).toHaveBeenCalledWith({
      from: 'claude/claude-sonnet-4-6',
      to: 'zai/glm-5.1',
      reason: 'rate_limited',
    })
  })

  describe('Codex request limits', () => {
    // The primary runs the real Codex provider behind the port adapter, so the
    // error the engine sees is produced by CodexSubscriptionProvider.classifyError.
    function codexPrimary(stream: ReturnType<typeof vi.fn>) {
      const adapter = new LlmPortAdapter(
        new CodexSubscriptionProvider('gpt-5.3-codex', {
          authorizer: {
            authorize: vi.fn().mockResolvedValue({
              providerAttemptId: 'attempt-1',
              requestHash: 'a'.repeat(64),
              executionTicket: 'ticket-123456',
              expiresAt: '2026-08-20T10:00:00.000Z',
            }),
          },
          proxy: { stream },
          attemptContext: () => ({
            policyRevision: 1,
            policyHash: 'b'.repeat(64),
            hostRef: 'chatllm',
          }),
        } as never),
        'gpt-5.3-codex',
        'codex-subscription'
      )
      const thrown: unknown[] = []
      const call = vi.fn(
        (): Promise<unknown> =>
          adapter
            .completeWithTools({
              messages: [{ role: 'user', content: 'hi' }],
              tools: [{ name: 'echo', description: 'echo', parameters: {} }],
            })
            .catch((err: unknown) => {
              thrown.push(err)
              throw err
            })
      )
      return { call, thrown }
    }

    it('propagates a tool-call limit without switching or setting a cooldown', async () => {
      const stream = vi.fn().mockResolvedValue({
        text: '',
        outcome: 'success',
        toolCalls: Array.from({ length: 257 }, (_, index) => ({
          id: `call-${index}`,
          name: 'echo',
          arguments: {},
        })),
      })
      const primary = codexPrimary(stream)
      const fallbackBuild = vi.fn(() => () => Promise.resolve('fallback-served'))
      const e = engine()

      const failure = await e
        .run(PRIMARY, t => (t.kind === 'primary' ? primary.call : fallbackBuild()), classify)
        .catch((err: unknown) => err)

      // Liveness witness: the primary ran once and really reached the proxy.
      expect(primary.call).toHaveBeenCalledTimes(1)
      expect(stream).toHaveBeenCalledTimes(1)
      expect(primary.thrown).toHaveLength(1)
      expect(failure).toBe(primary.thrown[0])
      expect(failure).toBeInstanceOf(LlmError)
      expect(failure).toMatchObject({
        code: LlmErrorCode.ToolCallLimitExceeded,
        retryable: false,
      })
      expect(fallbackBuild).not.toHaveBeenCalled()
      expect(metricInc).not.toHaveBeenCalled()

      // No cooldown was set: the next call starts at the primary again.
      const nextPrimary = vi.fn(() => Promise.resolve('primary-again'))
      const next = await e.run(
        PRIMARY,
        t => (t.kind === 'primary' ? nextPrimary : () => Promise.resolve('fallback')),
        classify
      )
      expect(next).toBe('primary-again')
      expect(nextPrimary).toHaveBeenCalledTimes(1)
    })

    it('still fails over a Codex provider outage to the fallback', async () => {
      const stream = vi
        .fn()
        .mockRejectedValue(new CodexProxyError('provider_unavailable', 'upstream 5xx'))
      const primary = codexPrimary(stream)
      const e = engine()

      const res = await e.run(
        PRIMARY,
        t => (t.kind === 'primary' ? primary.call : () => Promise.resolve('fallback-served')),
        classify
      )

      expect(primary.call).toHaveBeenCalledTimes(1)
      expect(primary.thrown[0]).toMatchObject({
        code: LlmErrorCode.ModelOverloaded,
        retryable: true,
      })
      expect(res).toBe('fallback-served')
      expect(metricInc).toHaveBeenCalledWith({
        from: 'claude/claude-sonnet-4-6',
        to: 'openai/gpt-5.4',
        reason: 'provider_unavailable',
      })
    })

    // A stream cut at its total cap spent the attempt's whole budget; the
    // fallback would spend it again on the same request.
    it('propagates a stream duration cap without switching or setting a cooldown', async () => {
      const stream = vi
        .fn()
        .mockRejectedValue(
          new CodexProxyError(
            'stream_duration_exceeded',
            'proxy stream failed with 504 (stream_duration_exceeded)'
          )
        )
      const primary = codexPrimary(stream)
      const fallbackBuild = vi.fn(() => () => Promise.resolve('fallback-served'))
      const e = engine()

      const failure = await e
        .run(PRIMARY, t => (t.kind === 'primary' ? primary.call : fallbackBuild()), classify)
        .catch((err: unknown) => err)

      // Liveness witness: the primary ran once and really reached the proxy.
      expect(primary.call).toHaveBeenCalledTimes(1)
      expect(stream).toHaveBeenCalledTimes(1)
      expect(failure).toBe(primary.thrown[0])
      expect(failure).toMatchObject({
        code: LlmErrorCode.StreamDurationExceeded,
        retryable: false,
      })
      expect(fallbackBuild).not.toHaveBeenCalled()
      expect(metricInc).not.toHaveBeenCalled()

      // No cooldown was set: the next call starts at the primary again.
      const nextPrimary = vi.fn(() => Promise.resolve('primary-again'))
      const next = await e.run(
        PRIMARY,
        t => (t.kind === 'primary' ? nextPrimary : () => Promise.resolve('fallback')),
        classify
      )
      expect(next).toBe('primary-again')
      expect(nextPrimary).toHaveBeenCalledTimes(1)
    })

    // The same claim for Grok. This is issue #680 in its most direct form:
    // before the taxonomy, the limit classified as ModelOverloaded/retryable,
    // failover fired, and the fallback answered from partial tool results.
    it('propagates a Grok tool-call limit without switching or setting a cooldown', async () => {
      const stream = vi.fn().mockResolvedValue({
        text: '',
        outcome: 'success',
        toolCalls: Array.from({ length: 257 }, (_, index) => ({
          id: `call-${index}`,
          name: 'echo',
          arguments: {},
        })),
      })
      const adapter = new LlmPortAdapter(
        new GrokSubscriptionProvider('grok-4.6', {
          authorizer: {
            authorize: vi.fn().mockResolvedValue({
              providerAttemptId: 'attempt-1',
              requestHash: 'a'.repeat(64),
              executionTicket: 'ticket-123456',
              expiresAt: '2026-08-20T10:00:00.000Z',
            }),
          },
          proxy: { stream },
          attemptContext: () => ({
            policyRevision: 1,
            policyHash: 'b'.repeat(64),
            hostRef: 'chatllm',
          }),
        } as never),
        'grok-4.6',
        'grok-subscription'
      )
      const thrown: unknown[] = []
      const call = vi.fn(
        (): Promise<unknown> =>
          adapter
            .completeWithTools({
              messages: [{ role: 'user', content: 'hi' }],
              tools: [{ name: 'echo', description: 'echo', parameters: {} }],
            })
            .catch((err: unknown) => {
              thrown.push(err)
              throw err
            })
      )
      const fallbackBuild = vi.fn(() => () => Promise.resolve('fallback-served'))
      const e = engine()

      const failure = await e
        .run(PRIMARY, t => (t.kind === 'primary' ? call : fallbackBuild()), classify)
        .catch((err: unknown) => err)

      // Liveness witness: the primary ran once and really reached the proxy.
      expect(call).toHaveBeenCalledTimes(1)
      expect(stream).toHaveBeenCalledTimes(1)
      expect(thrown).toHaveLength(1)
      expect(failure).toBe(thrown[0])
      expect(failure).toBeInstanceOf(LlmError)
      expect(failure).toMatchObject({
        code: LlmErrorCode.ToolCallLimitExceeded,
        retryable: false,
      })
      expect(fallbackBuild).not.toHaveBeenCalled()
      expect(metricInc).not.toHaveBeenCalled()
    })
  })

  it('is sticky: while cooling, the next call starts at the fallback (no primary retry)', async () => {
    const e = engine()
    // Call 1: primary fails eligibly → cooldown set, fallback serves.
    await e.run(
      PRIMARY,
      t =>
        t.kind === 'primary'
          ? () => Promise.reject(llmError(LlmErrorCode.RateLimited, true))
          : () => Promise.resolve('fb1'),
      classify
    )
    // Call 2 within the cooldown window: primary must NOT be attempted.
    const primaryBuild = vi.fn(() => () => Promise.resolve('primary-should-not-run'))
    const res = await e.run(
      PRIMARY,
      t => (t.kind === 'primary' ? primaryBuild() : () => Promise.resolve('fb2')),
      classify
    )
    expect(res).toBe('fb2')
    expect(primaryBuild).not.toHaveBeenCalled()
    // No new switch recorded on call 2 (already on the fallback).
    expect(metricInc).toHaveBeenCalledTimes(1)
  })

  it('lazy recovery: after the cooldown expires the primary is retried', async () => {
    const e = engine(policy({ cooldownSeconds: 300 }))
    await e.run(
      PRIMARY,
      t =>
        t.kind === 'primary'
          ? () => Promise.reject(llmError(LlmErrorCode.RateLimited, true))
          : () => Promise.resolve('fb'),
      classify
    )
    expect(e.servedBy()?.fallback).toBe(true)
    // Advance past the cooldown.
    now += 300_000 + 1
    const res = await e.run(PRIMARY, () => () => Promise.resolve('primary-recovered'), classify)
    expect(res).toBe('primary-recovered')
    expect(e.servedBy()).toEqual({ ...PRIMARY, fallback: false })
  })

  it('exhausted list propagates the last error (canned fallback is the last resort)', async () => {
    const e = engine()
    const lastErr = llmError(LlmErrorCode.ModelOverloaded, true)
    await expect(
      e.run(
        PRIMARY,
        t =>
          t.kind === 'primary'
            ? () => Promise.reject(llmError(LlmErrorCode.RateLimited, true))
            : () => Promise.reject(lastErr),
        classify
      )
    ).rejects.toBe(lastErr)
  })

  it('clearCooldown lets the next call retry the primary immediately (key recovery)', async () => {
    const e = engine()
    await e.run(
      PRIMARY,
      t =>
        t.kind === 'primary'
          ? () => Promise.reject(llmError(LlmErrorCode.AuthenticationFailed, false))
          : () => Promise.resolve('fb'),
      classify
    )
    // Without clearing, the primary would be skipped (cooling) — clear it.
    e.clearCooldown()
    const res = await e.run(PRIMARY, () => () => Promise.resolve('primary-immediately'), classify)
    expect(res).toBe('primary-immediately')
    expect(e.servedBy()).toEqual({ ...PRIMARY, fallback: false })
  })

  it('passes the correct FULL-list index to the builder as it advances (index alignment)', async () => {
    const e = engine(
      policy({
        fallbacks: [
          { provider: 'openai', model: 'gpt-5.4' }, // index 0 — unconstructible
          { provider: 'zai', model: 'glm-5.1' }, // index 1 — serves
        ],
      })
    )
    const seenIndexes: number[] = []
    const res = await e.run(
      PRIMARY,
      t => {
        if (t.kind === 'primary')
          return () => Promise.reject(llmError(LlmErrorCode.RateLimited, true))
        seenIndexes.push(t.index)
        if (t.index === 0) return null // unconstructible → skip
        return () => Promise.resolve(`served-${t.index}`)
      },
      classify
    )
    expect(res).toBe('served-1')
    // The builder saw both full-list indices in order (0 skipped, 1 served).
    expect(seenIndexes).toEqual([0, 1])
  })

  it('honours an Attempt.servedModel override for servedBy + the metric to-label', async () => {
    // FIX-2: a same-provider fallback serves the SESSION model, not entry.model.
    // The builder returns { run, servedModel } and the engine must use that model
    // for servedBy and the `to` metric label instead of the target's entry model.
    const e = engine(policy({ fallbacks: [{ provider: 'claude', model: 'claude-haiku-4-5' }] }))
    const res = await e.run(
      PRIMARY,
      t =>
        t.kind === 'primary'
          ? () => Promise.reject(llmError(LlmErrorCode.RateLimited, true))
          : { run: () => Promise.resolve('served'), servedModel: 'claude-sonnet-4-6' },
      classify
    )
    expect(res).toBe('served')
    expect(e.servedBy()).toEqual({ provider: 'claude', model: 'claude-sonnet-4-6', fallback: true })
    expect(metricInc).toHaveBeenCalledWith({
      from: 'claude/claude-sonnet-4-6',
      to: 'claude/claude-sonnet-4-6',
      reason: 'rate_limited',
    })
  })

  it('setPolicy resets the sticky cooldown', async () => {
    const e = engine()
    await e.run(
      PRIMARY,
      t =>
        t.kind === 'primary'
          ? () => Promise.reject(llmError(LlmErrorCode.RateLimited, true))
          : () => Promise.resolve('fb'),
      classify
    )
    e.setPolicy(policy())
    expect(e.servedBy()).toBeNull()
    // After reset the primary is attempted again immediately.
    const res = await e.run(PRIMARY, () => () => Promise.resolve('primary'), classify)
    expect(res).toBe('primary')
  })

  it('assigns a distinct providerAttemptIndex to each physical attempt', async () => {
    const e = engine(
      policy({
        fallbacks: [
          { provider: 'openai', model: 'gpt-5.4' },
          { provider: 'zai', model: 'glm-5.1' },
        ],
      })
    )
    const indexes: number[] = []
    await e.run(
      PRIMARY,
      (t, physical) => {
        indexes.push(physical.providerAttemptIndex)
        if (t.kind === 'primary')
          return () => Promise.reject(llmError(LlmErrorCode.RateLimited, true))
        if (t.kind === 'fallback' && t.index === 0)
          return () => Promise.reject(llmError(LlmErrorCode.ModelOverloaded, true))
        return () => Promise.resolve('ok')
      },
      classify
    )
    expect(indexes).toEqual([1, 2, 3])
  })

  it('treats budget_denied as terminal unless the policy opts in', async () => {
    const err = new LlmError(
      'budget denied',
      'codex-subscription',
      LlmErrorCode.InsufficientQuota,
      false,
      undefined,
      undefined,
      'budget_denied'
    )
    const classifyWithCode = (caught: unknown): ClassifiedLike | null => {
      if (caught instanceof LlmError) {
        return {
          code: caught.code as LlmErrorCode,
          retryable: caught.retryable,
          providerCode: caught.providerCode,
        }
      }
      return classify(caught)
    }
    const denied = engine()
    await expect(
      denied.run(
        PRIMARY,
        t => (t.kind === 'primary' ? () => Promise.reject(err) : () => Promise.resolve('nope')),
        classifyWithCode
      )
    ).rejects.toBe(err)

    const optedIn = engine(policy({ budgetDeniedFailover: true }))
    await expect(
      optedIn.run(
        PRIMARY,
        t => (t.kind === 'primary' ? () => Promise.reject(err) : () => Promise.resolve('fallback')),
        classifyWithCode
      )
    ).resolves.toBe('fallback')
  })
})