Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript
8.0 KB
Raw Blame
import { describe, expect, it, vi } from 'vitest'
import { LlmErrorCode } from '../core/errors'
import { FinishReason } from '../core/types'
import type { CompletionResponse, ToolCompletionResponse } from '../core/types'
import type { SingleTurnProvider } from '../llm'
import {
  FailoverSingleTurnProvider,
  isFailoverProvider,
  parseWorkflowLlmPolicy,
} from './failoverProvider'

/**
 * Provider-fallback (R5 F6) — workflow-step glue.
 */

const TEXT: CompletionResponse = {
  content: 'done',
  usage: { input_tokens: 1, output_tokens: 2, total_tokens: 3 },
  finish_reason: FinishReason.Stop,
}
const TOOLS: ToolCompletionResponse = {
  content: 'done',
  tool_calls: [],
  usage: { input_tokens: 1, output_tokens: 2, total_tokens: 3 },
  finish_reason: FinishReason.Stop,
}

function fakeProvider(opts: {
  onWithTools?: () => Promise<ToolCompletionResponse>
  classify?: { code: LlmErrorCode; retryable: boolean }
}): SingleTurnProvider {
  const classify = opts.classify ?? { code: LlmErrorCode.RateLimited, retryable: true }
  return {
    completeSingleTurn: vi.fn(async () => TEXT),
    completeSingleTurnWithTools: vi.fn(opts.onWithTools ?? (async () => TOOLS)),
    getProviderType: () => 'openai',
    classifyError: () => ({ code: classify.code, retryable: classify.retryable, message: 'x' }),
  } as unknown as SingleTurnProvider
}

describe('parseWorkflowLlmPolicy', () => {
  it('returns null for absent / malformed / empty policies (no failover)', () => {
    expect(parseWorkflowLlmPolicy(undefined)).toBeNull()
    expect(parseWorkflowLlmPolicy({})).toBeNull()
    expect(parseWorkflowLlmPolicy({ fallbacks: [] })).toBeNull()
    // fallback missing apiKey → dropped → zero valid → null
    expect(parseWorkflowLlmPolicy({ fallbacks: [{ provider: 'openai', model: 'm' }] })).toBeNull()
  })

  it('parses a valid policy with defaults for cooldown/triggerOn', () => {
    const policy = parseWorkflowLlmPolicy({
      fallbacks: [{ provider: 'claude', model: 'claude-haiku-4-5', apiKey: 'sk-fb' }],
    })
    expect(policy).not.toBeNull()
    expect(policy!.cooldownSeconds).toBe(300)
    expect(policy!.triggerOn).toEqual([
      'insufficient_quota',
      'auth',
      'provider_unavailable',
      'rate_limited',
    ])
    expect(policy!.fallbacks).toEqual([
      { provider: 'claude', model: 'claude-haiku-4-5', apiKey: 'sk-fb' },
    ])
  })

  it('honours explicit cooldown + triggerOn and drops unknown classes', () => {
    const policy = parseWorkflowLlmPolicy({
      cooldownSeconds: 30,
      triggerOn: ['rate_limited', 'bogus'],
      fallbacks: [{ provider: 'openai', model: 'gpt-x', apiKey: 'sk' }],
    })
    expect(policy!.cooldownSeconds).toBe(30)
    expect(policy!.triggerOn).toEqual(['rate_limited'])
  })

  it('preserves cooldownSeconds: 0 (immediate expiry), not coerced to 300', () => {
    const policy = parseWorkflowLlmPolicy({
      cooldownSeconds: 0,
      fallbacks: [{ provider: 'openai', model: 'gpt-x', apiKey: 'sk' }],
    })
    expect(policy!.cooldownSeconds).toBe(0)
  })

  it('ignores a negative / non-integer cooldown, keeping the default', () => {
    expect(
      parseWorkflowLlmPolicy({
        cooldownSeconds: -5,
        fallbacks: [{ provider: 'openai', model: 'gpt-x', apiKey: 'sk' }],
      })!.cooldownSeconds
    ).toBe(300)
    expect(
      parseWorkflowLlmPolicy({
        cooldownSeconds: 1.5,
        fallbacks: [{ provider: 'openai', model: 'gpt-x', apiKey: 'sk' }],
      })!.cooldownSeconds
    ).toBe(300)
  })
})

describe('FailoverSingleTurnProvider', () => {
  it('switches to the fallback on an eligible failure and reports the served pair', async () => {
    const primary = fakeProvider({
      onWithTools: () => Promise.reject(new Error('429')),
      classify: { code: LlmErrorCode.RateLimited, retryable: true },
    })
    const fallback = fakeProvider({ onWithTools: () => Promise.resolve(TOOLS) })
    const buildFallback = vi.fn(() => fallback)

    const provider = new FailoverSingleTurnProvider({
      primary,
      primaryPair: { provider: 'openai', model: 'gpt-primary' },
      policy: {
        cooldownSeconds: 300,
        triggerOn: ['rate_limited'],
        fallbacks: [{ provider: 'claude', model: 'claude-fb', apiKey: 'sk-fb' }],
      },
      buildFallback,
    })

    const res = await provider.completeSingleTurnWithTools([], [])
    expect(res.content).toBe('done')
    expect(buildFallback).toHaveBeenCalledTimes(1)
    expect(provider.servedBy()).toEqual({ provider: 'claude', model: 'claude-fb' })
    expect(isFailoverProvider(provider)).toBe(true)
  })

  it('does NOT switch on a non-eligible failure and rethrows the ORIGINAL error', async () => {
    const original = new Error('400 bad request')
    const primary = fakeProvider({
      onWithTools: () => Promise.reject(original),
      classify: { code: LlmErrorCode.ApiCallFailed, retryable: false }, // 400 → not eligible
    })
    const buildFallback = vi.fn(() => fakeProvider({}))

    const provider = new FailoverSingleTurnProvider({
      primary,
      primaryPair: { provider: 'openai', model: 'gpt-primary' },
      policy: {
        cooldownSeconds: 300,
        triggerOn: ['rate_limited', 'provider_unavailable'],
        fallbacks: [{ provider: 'claude', model: 'claude-fb', apiKey: 'sk-fb' }],
      },
      buildFallback,
    })

    await expect(provider.completeSingleTurnWithTools([], [])).rejects.toBe(original)
    expect(buildFallback).not.toHaveBeenCalled()
    // Still serving the primary (never switched).
    expect(provider.servedBy()).toEqual({ provider: 'openai', model: 'gpt-primary' })
  })

  it('serves the primary when it succeeds', async () => {
    const primary = fakeProvider({ onWithTools: () => Promise.resolve(TOOLS) })
    const provider = new FailoverSingleTurnProvider({
      primary,
      primaryPair: { provider: 'openai', model: 'gpt-primary' },
      policy: {
        cooldownSeconds: 300,
        triggerOn: ['rate_limited'],
        fallbacks: [{ provider: 'claude', model: 'claude-fb', apiKey: 'sk-fb' }],
      },
      buildFallback: () => fakeProvider({}),
    })

    await provider.completeSingleTurnWithTools([], [])
    expect(provider.servedBy()).toEqual({ provider: 'openai', model: 'gpt-primary' })
  })

  it('skips an unconstructible fallback (builder returns null) and propagates the last error', async () => {
    const err = new Error('429')
    const primary = fakeProvider({
      onWithTools: () => Promise.reject(err),
      classify: { code: LlmErrorCode.RateLimited, retryable: true },
    })
    const provider = new FailoverSingleTurnProvider({
      primary,
      primaryPair: { provider: 'openai', model: 'gpt-primary' },
      policy: {
        cooldownSeconds: 300,
        triggerOn: ['rate_limited'],
        fallbacks: [{ provider: 'claude', model: 'claude-fb', apiKey: 'sk-fb' }],
      },
      buildFallback: () => null, // credential absent → skip
    })

    await expect(provider.completeSingleTurnWithTools([], [])).rejects.toBe(err)
  })

  it('does NOT fail over on a cooperative abort (step deadline) — propagates + no switch', async () => {
    const abort = new Error('step-timeout')
    const controller = new AbortController()
    controller.abort('step-timeout')
    const primary = fakeProvider({
      onWithTools: () => Promise.reject(abort),
      // Would be eligible if classified — but the aborted signal short-circuits.
      classify: { code: LlmErrorCode.ApiCallFailed, retryable: true },
    })
    const buildFallback = vi.fn(() => fakeProvider({}))

    const provider = new FailoverSingleTurnProvider({
      primary,
      primaryPair: { provider: 'openai', model: 'gpt-primary' },
      policy: {
        cooldownSeconds: 300,
        triggerOn: ['provider_unavailable'],
        fallbacks: [{ provider: 'claude', model: 'claude-fb', apiKey: 'sk-fb' }],
      },
      buildFallback,
    })

    await expect(
      provider.completeSingleTurnWithTools([], [], { signal: controller.signal })
    ).rejects.toBe(abort)
    expect(buildFallback).not.toHaveBeenCalled()
    expect(provider.servedBy()).toEqual({ provider: 'openai', model: 'gpt-primary' })
  })
})