Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
/**
* Provider-fallback (R5) — the reusable failover engine.
*
* AGENT-AGNOSTIC by design (spec §3-R5.6/R5.7): it holds ONLY the failover
* policy + the sticky in-memory state (primary cooldown, current served pair),
* and orchestrates the ordered attempt loop. It knows nothing about `LlmPort`,
* the agent, workflows or promptBridge — F5 (agent) wires it through
* `core/adapters/failoverLlmPort.ts`; F6 will wire the same engine into the
* workflow step runner and the promptBridge `LlmBridge`.
*
* Semantics (spec §3-R5.1):
* - A call that fails with an ELIGIBLE error (`triggerOn ∋ class`) advances to
* the next constructible fallback in order; the primary enters a sticky
* cooldown (default 300 s).
* - While the primary is cooling, fresh calls start directly at the fallbacks
* (no immediate retry of the downed primary — anti-flapping, V13).
* - Lazy recovery: once the cooldown expires, the next call attempts the
* primary again (no active health-probe). If it fails again, cooldown renews.
* - A NON-eligible error (e.g. 400/validation) propagates immediately without
* failover (never mask bugs).
* - When the ordered list is exhausted the last error propagates — the agent
* loop's canned workflow fallback remains the last resort (precedence: the
* engine switches BELOW that layer, so it wins when a policy is configured).
*
* One pod = one agent, so the state lives in memory here (spec §3-R5.9).
*/
import { LlmErrorCode } from '../../core/errors'
import { classifyFailoverClass } from './classify'
import { llmFallbackTotal } from './metrics'
import type {
FailoverClass,
FailoverSwitchEvent,
FailoverTarget,
LlmPolicy,
ModelPair,
ServedBy,
} from './types'
/** Minimal shape the engine needs to classify a thrown provider error. */
export interface ClassifiedLike {
code: LlmErrorCode
retryable: boolean
/** Provider-native code (e.g. `budget_denied`) used for terminal gates. */
providerCode?: string
/**
* Mirrors `ClassifiedError.providerDispatched`: `false` only when the
* classifier proved no call left the process. `undefined` is unknown and
* must be read as dispatched.
*/
providerDispatched?: boolean
}
export interface FailoverEngineOptions {
/** Injectable clock for tests. Defaults to `Date.now`. */
now?: () => number
/** Structured-WARN sink for each switch (no secrets). */
onSwitch?: (event: FailoverSwitchEvent) => void
/**
* Metric sink for each switch. Defaults to the shared
* `clerum_llm_fallback_total`. Injectable so tests can assert without the
* global registry.
*/
metricInc?: (labels: { from: string; to: string; reason: string }) => void
}
/**
* A concrete attempt for a target: the call thunk, optionally paired with the
* model the attempt will REALLY serve. The engine uses `servedModel` (when
* present) for `servedBy` + the `clerum_llm_fallback_total{to}` metric instead
* of the target's declared `model`. This matters for a SAME-provider fallback:
* the runtime serves the SESSION model (per R5.7), not the entry's `model`, so
* the caller reports the real served model here.
*/
export interface Attempt<T> {
run: () => Promise<T>
/** The model actually served by this attempt (overrides `target.model`). */
servedModel?: string
}
/**
* Builds the concrete attempt for a target, or returns `null` when that target
* is not constructible right now (e.g. a fallback whose credential slot is
* absent) — the engine then skips it. May return a bare `() => Promise<T>`
* thunk (served model = `target.model`) or an {@link Attempt} carrying an
* explicit `servedModel` override.
*/
export type PhysicalAttempt = {
/** 1-based index of this physical attempt in the engine's lifetime. */
providerAttemptIndex: number
}
export type AttemptBuilder<T> = (
target: FailoverTarget,
physicalAttempt: PhysicalAttempt
) => (() => Promise<T>) | Attempt<T> | null
function pairLabel(p: ModelPair): string {
return `${p.provider}/${p.model}`
}
export class FailoverEngine {
private cooldownUntil: number | null = null
private served: ServedBy | null = null
private nextProviderAttemptIndex = 1
constructor(
private policy: LlmPolicy,
private readonly opts: FailoverEngineOptions = {}
) {}
/** The pair currently serving (for the status projection). */
servedBy(): ServedBy | null {
return this.served
}
/**
* Replace the policy (Host CR edit). Resets the sticky state: a changed
* fallback list / cooldown invalidates the old cursor + cooldown clock.
*/
setPolicy(policy: LlmPolicy): void {
this.policy = policy
this.cooldownUntil = null
this.served = null
this.nextProviderAttemptIndex = 1
}
/** Is this `(code, retryable)` eligible for fallback under the current policy? */
isEligible(code: LlmErrorCode, retryable: boolean): boolean {
const cls = classifyFailoverClass(code, retryable)
return cls !== null && this.policy.triggerOn.includes(cls)
}
/** Record that `pair` is now serving via the primary (recovered / normal). */
noteServedPrimary(pair: ModelPair): void {
this.served = { provider: pair.provider, model: pair.model, fallback: false }
}
/**
* Clear any active primary cooldown so the very next call retries the primary.
* Called when the primary becomes constructible again out-of-band (e.g. the
* LLM key is rotated/fixed) — the primary was never truly "down", so the
* sticky cooldown should not delay its recovery.
*/
clearCooldown(): void {
this.cooldownUntil = null
}
/** Record that `pair` (a fallback) is now serving — used at boot too. */
noteServedFallback(pair: ModelPair): void {
this.served = { provider: pair.provider, model: pair.model, fallback: true }
}
private now(): number {
return this.opts.now ? this.opts.now() : Date.now()
}
/**
* Ordered targets to try for a fresh call, honoring the sticky primary
* cooldown + lazy recovery. Reads (and expires) the cooldown clock.
*/
planTargets(primary: ModelPair): FailoverTarget[] {
let primaryCooling = false
if (this.cooldownUntil !== null) {
if (this.now() < this.cooldownUntil) primaryCooling = true
else this.cooldownUntil = null // lazy recovery — expire and retry primary
}
const targets: FailoverTarget[] = []
if (!primaryCooling) targets.push({ kind: 'primary', ...primary })
this.policy.fallbacks.forEach((entry, index) => {
targets.push({ kind: 'fallback', index, provider: entry.provider, model: entry.model })
})
return targets
}
private recordSwitch(from: ModelPair, to: ModelPair, reason: FailoverClass): void {
const inc = this.opts.metricInc ?? (labels => llmFallbackTotal.inc(labels))
inc({ from: pairLabel(from), to: pairLabel(to), reason })
this.opts.onSwitch?.({ from, to, reason })
}
/**
* Run one logical LLM call with failover. `primary` is the effective pair the
* call would normally use (per-session resolved model, for the agent); `build`
* produces the concrete attempt for each ordered target (or `null` to skip an
* unconstructible one); `classify` extracts `(code, retryable)` from a thrown
* error (or `null` when it is not a recognised provider error).
*/
async run<T>(
primary: ModelPair,
build: AttemptBuilder<T>,
classify: (err: unknown) => ClassifiedLike | null
): Promise<T> {
const targets = this.planTargets(primary)
let lastErr: unknown
let prevFailure: { pair: ModelPair; reason: FailoverClass } | null = null
for (const target of targets) {
const physicalAttempt = { providerAttemptIndex: this.nextProviderAttemptIndex }
const built = build(target, physicalAttempt)
if (!built) continue // not constructible → skip this target
this.nextProviderAttemptIndex += 1
const attempt = typeof built === 'function' ? built : built.run
// Report the model REALLY served (a same-provider fallback serves the
// session model, not `entry.model`) so `servedBy` + the metric's `to`
// label reflect it; bare-thunk builders fall back to the target model.
const servedModel =
typeof built === 'function' ? target.model : (built.servedModel ?? target.model)
const pair: ModelPair = { provider: target.provider, model: servedModel }
// Only count a switch once we actually attempt a DIFFERENT constructible
// pair following an eligible failure (skipped/unconstructible targets in
// between do not emit a phantom switch).
if (prevFailure) this.recordSwitch(prevFailure.pair, pair, prevFailure.reason)
try {
const result = await attempt()
if (target.kind === 'fallback') this.noteServedFallback(pair)
else this.noteServedPrimary(pair)
return result
} catch (err) {
lastErr = err
const classified = classify(err)
if (classified?.providerCode === 'budget_denied' && !this.policy.budgetDeniedFailover) {
throw err
}
const failClass = classified
? classifyFailoverClass(classified.code, classified.retryable)
: null
const eligible = failClass !== null && this.policy.triggerOn.includes(failClass)
if (!eligible) throw err // non-eligible → propagate, never mask (e.g. 400)
if (target.kind === 'primary') {
this.cooldownUntil = this.now() + this.policy.cooldownSeconds * 1000
}
prevFailure = { pair, reason: failClass }
// fall through to next target
}
}
// Exhausted (or nothing constructible): propagate the last error so the
// caller's own recovery (agent loop's canned fallback) can act as last resort.
throw lastErr ?? new Error('[Failover] no attemptable target')
}
}