Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript
5.3 KB
Raw Blame
/**
 * Observable-behavior regression tests for the G2 gap (spec 02, Pieza A):
 * a provider 404 (retired / inaccessible model) MUST classify as
 * `ModelNotAvailable` (non-retryable) so that, observably:
 *
 *   1. the failover engine does NOT silently divert traffic to another
 *      provider (ModelNotAvailable → no failover class), and
 *   2. the tool-use loop does NOT retry it as a transient transport error.
 *
 * These assert the OBSERVABLE outcome (T4) — did we failover? did we retry?
 * what code surfaces? — not `classifyByHttpStatus` in isolation. The
 * `ClassifiedError` / `LlmError` are DERIVED from the real provider classifiers
 * (T1), never hand-built.
 *
 * Coverage spans all four classifier arms: OpenAI-compatible, Claude, Vertex
 * (google driver), and Bedrock.
 */
import { describe, expect, it, vi } from 'vitest'
import { LlmError, LlmErrorCode } from '../../core/errors'
import {
  buildTestConfig,
  createMockReasoning,
} from '../../core/orchestration/__tests__/toolUseLoopRetryableTestUtils'
import { runToolUseLoop } from '../../core/orchestration/toolUseLoop'
import { ClaudeProvider } from '../claude'
import { classifyBedrockError } from '../drivers/bedrockConverse'
import { classifyGoogleError } from '../drivers/googleGenerative'
import { FailoverEngine } from '../failover/engine'
import type { LlmPolicy, ModelPair } from '../failover/types'
import { OpenAICompatibleProvider } from '../openaiCompatible'
import type { ClassifiedError } from '../types'
import {
  anthropicApiError,
  bedrockResourceNotFound,
  openaiApiError,
  vertexApiError,
} from './sdkErrorFixtures'

const PRIMARY: ModelPair = { provider: 'claude', model: 'claude-sonnet-4-6' }

function policy(): LlmPolicy {
  return {
    cooldownSeconds: 300,
    triggerOn: ['insufficient_quota', 'auth', 'provider_unavailable', 'rate_limited'],
    fallbacks: [{ provider: 'openai', model: 'gpt-5.4' }],
  }
}

type Classifier = (err: unknown) => ClassifiedError

const zai = new OpenAICompatibleProvider(
  { id: 'zai', baseURL: 'https://example.invalid', defaultModel: 'glm-4' },
  'fake-key'
)
const claude = new ClaudeProvider('fake-key', 'claude-sonnet-4-6')

// Real SDK-shaped 404 payloads (the boundary these classifiers consume), one
// per arm. All four arms derive from the real SDK error constructor (T1):
// OpenAI/Anthropic via `*.APIError.generate`, Vertex via `new ApiError`
// (@google/genai), Bedrock via `new ResourceNotFoundException`
// (@aws-sdk/client-bedrock-runtime). The ClassifiedError/LlmError below are
// produced by the real classifiers, not hand-written.
const CASES: ReadonlyArray<[string, Classifier, string, unknown]> = [
  [
    'openai-compat (zai)',
    (e: unknown) => zai.classifyError(e),
    'zai',
    openaiApiError(404, { code: '1211', message: 'The model does not exist' }),
  ],
  [
    'claude',
    (e: unknown) => claude.classifyError(e),
    'claude',
    anthropicApiError(404, 'not_found_error', 'model: claude-x not found'),
  ],
  [
    'vertex (google)',
    classifyGoogleError,
    'vertex',
    vertexApiError(404, {
      error: { code: 404, message: 'Publisher Model `x` was not found', status: 'NOT_FOUND' },
    }),
  ],
  [
    'bedrock',
    classifyBedrockError,
    'bedrock',
    bedrockResourceNotFound('The provided model identifier is invalid'),
  ],
]

function toLlmError(classify: Classifier, provider: string, err: unknown): LlmError {
  // Mirrors LlmPortAdapter.handleProviderError construction.
  const c = classify(err)
  return new LlmError(
    c.message,
    provider,
    c.code,
    c.retryable,
    err as Error,
    c.httpStatus,
    c.providerCode
  )
}

describe('404 → ModelNotAvailable: does NOT trigger cross-provider failover', () => {
  it.each(CASES)('%s', async (_label, classify, _provider, err404) => {
    // Sanity: the real classifier produces the ambiguous, non-retryable class.
    expect(classify(err404).code).toBe(LlmErrorCode.ModelNotAvailable)
    expect(classify(err404).retryable).toBe(false)

    const engine = new FailoverEngine(policy())
    const fallbackRun = vi.fn()
    const run = engine.run(
      PRIMARY,
      target =>
        target.kind === 'primary'
          ? () => Promise.reject(err404)
          : () => {
              fallbackRun()
              return Promise.resolve('fallback-served')
            },
      classify
    )

    // Propagates the original error — never masked by a fallback.
    await expect(run).rejects.toBe(err404)
    // The observable proof of "no silent failover": the fallback never ran.
    expect(fallbackRun).not.toHaveBeenCalled()
  })
})

describe('404 → ModelNotAvailable: is NOT retried by the tool-use loop', () => {
  it.each(CASES)('%s', async (_label, classify, provider, err404) => {
    const reasoning = createMockReasoning([
      { type: 'error', error: toLlmError(classify, provider, err404) },
    ])

    const result = await runToolUseLoop(buildTestConfig(reasoning, []), [
      { role: 'user', content: 'hi' },
    ])

    expect(result.type).toBe('error')
    if (result.type === 'error') {
      expect((result.error as LlmError).code).toBe(LlmErrorCode.ModelNotAvailable)
    }
    // A retryable transport error would call respondWithTools twice; a
    // non-retryable ModelNotAvailable must be surfaced after a single call.
    expect(reasoning.respondWithTools).toHaveBeenCalledTimes(1)
  })
})