Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
feat(codex): reauthorize fallback and Prompt Bridge attempts
Alfredo Lopez8b09d6007c38
diff --git a/control-api/src/services/budgets/definitions.ts b/control-api/src/services/budgets/definitions.tsindex 12a7afe1b..2f8b22236 100644--- a/control-api/src/services/budgets/definitions.ts+++ b/control-api/src/services/budgets/definitions.ts@@ -43,6 +43,21 @@ export class BudgetValidationError extends Error {}}+const CODEX_SUBSCRIPTION_PROVIDER = 'codex-subscription'++export function scopeIncludesCodexSubscription(scope?: BudgetScope | null): boolean {+ return Array.isArray(scope?.provider) && scope.provider.includes(CODEX_SUBSCRIPTION_PROVIDER)+}++export function assertCodexBudgetUnit(+ unit: BudgetUnit | undefined,+ scope?: BudgetScope | null+): void {+ if (unit === 'cost' && scopeIncludesCodexSubscription(scope)) {+ throw new BudgetValidationError('Codex subscription budgets must use unit tokens, not cost.')+ }+}+/** A scoped (provider, model) target that has no active price. provider is null* when the scope pins a model without pinning a provider (any provider). */export type UnpricedScopeTarget = { provider: string | null; model: string }@@ -318,6 +333,7 @@ export async function createBudget(db: DbClient = pool): Promise<TokenBudget> {const currency = input.currency ?? null+ assertCodexBudgetUnit(input.unit, input.scope)if (input.unit === 'cost') {const unpriced = await findUnpricedScopedModels(input.scope, db)if (unpriced.length > 0) throw new BudgetUnpricedModelsError(unpriced)@@ -383,6 +399,7 @@ export async function updateBudget(}}if (effectiveUnit === 'cost' && effectiveScope) {+ assertCodexBudgetUnit(effectiveUnit, effectiveScope)const unpriced = await findUnpricedScopedModels(effectiveScope, db)if (unpriced.length > 0) throw new BudgetUnpricedModelsError(unpriced)}diff --git a/control-api/test/services.budgets.test.ts b/control-api/test/services.budgets.test.tsindex 8b82bd4d5..446bbe205 100644--- a/control-api/test/services.budgets.test.ts+++ b/control-api/test/services.budgets.test.ts@@ -357,6 +357,36 @@ describe('createBudget cost-model pricing guard', () => {const pricingCall = query.mock.calls.find(c => /llm_model_prices/.test(String(c[0])))expect(pricingCall).toBeUndefined()})++ it('rejects unit=cost when the scope includes Codex subscription', async () => {+ const query = vi.fn()+ const input = createBudgetSchema.parse({+ name: 'codex cost',+ unit: 'cost',+ currency: 'USD',+ limit_amount: 25,+ period: 'monthly',+ scope: { provider: ['codex-subscription'] },+ })+ await expect(createBudget(input, fakeDb(query))).rejects.toMatchObject({+ name: 'BudgetValidationError',+ message: /unit tokens, not cost/,+ })+ expect(query).not.toHaveBeenCalled()+ })++ it('allows a tokens budget scoped to Codex subscription', async () => {+ const query = vi.fn().mockResolvedValue({ rows: [BASE_BUDGET], rowCount: 1 })+ const input = createBudgetSchema.parse({+ name: 'codex tokens',+ unit: 'tokens',+ limit_amount: 10000,+ period: 'monthly',+ scope: { provider: ['codex-subscription'] },+ })+ await createBudget(input, fakeDb(query))+ expect(query).toHaveBeenCalled()+ })})describe('updateBudget cost-model pricing guard (merged effective row)', () => {diff --git a/mcp-host/src/core/adapters/failoverLlmPort.ts b/mcp-host/src/core/adapters/failoverLlmPort.tsindex 406f02724..4d1492aab 100644--- a/mcp-host/src/core/adapters/failoverLlmPort.ts+++ b/mcp-host/src/core/adapters/failoverLlmPort.ts@@ -31,7 +31,11 @@ import type {/** Classify a thrown error for the engine. Only real `LlmError`s are eligible. */function classifyLlmError(err: unknown): ClassifiedLike | null {if (err instanceof LlmError) {- return { code: err.code as LlmErrorCode, retryable: err.retryable }+ return {+ code: err.code as LlmErrorCode,+ retryable: err.retryable,+ ...(err.providerCode ? { providerCode: err.providerCode } : {}),+ }}return null}diff --git a/mcp-host/src/llm/__tests__/codexSubscription.test.ts b/mcp-host/src/llm/__tests__/codexSubscription.test.tsindex 2c917c3a2..d9451f6d0 100644--- a/mcp-host/src/llm/__tests__/codexSubscription.test.ts+++ b/mcp-host/src/llm/__tests__/codexSubscription.test.ts@@ -91,6 +91,26 @@ describe('CodexSubscriptionProvider', () => {expect(wired.stream).not.toHaveBeenCalled()})+ it('issues a new authorize attempt index on every physical call', async () => {+ const wired = deps()+ const provider = new CodexSubscriptionProvider('gpt-5.3-codex', wired as never)+ await provider.completeSingleTurn([{ role: 'user', content: 'one' }])+ await provider.completeSingleTurn([{ role: 'user', content: 'two' }])+ expect(wired.authorize.mock.calls[0][0].providerAttemptIndex).toBe(1)+ expect(wired.authorize.mock.calls[1][0].providerAttemptIndex).toBe(2)+ expect(wired.authorize.mock.calls[0][0].request.requestId).not.toBe(+ wired.authorize.mock.calls[1][0].request.requestId+ )+ })++ it('keeps unknown usage unknown when the proxy omits token counts', async () => {+ const wired = deps()+ const provider = new CodexSubscriptionProvider('gpt-5.3-codex', wired as never)+ const result = await provider.completeSingleTurn([{ role: 'user', content: 'hi' }])+ expect(result.usage_reported).toBe(false)+ expect(result.usage).toEqual({ input_tokens: 0, output_tokens: 0, total_tokens: 0 })+ })+it('keeps insufficient_scope distinguishable', () => {const provider = new CodexSubscriptionProvider('gpt-5.3-codex', deps() as never)const classified = provider.classifyError(diff --git a/mcp-host/src/llm/codexLlmProxyClient.ts b/mcp-host/src/llm/codexLlmProxyClient.tsindex 8e8babdb4..b65fdd577 100644--- a/mcp-host/src/llm/codexLlmProxyClient.ts+++ b/mcp-host/src/llm/codexLlmProxyClient.ts@@ -19,6 +19,8 @@ export type CodexProxyStreamResult = {text: stringtoolCalls: Array<{ id: string; name: string; arguments: Record<string, unknown> }>outcome: 'success' | 'canceled' | 'error' | 'unknown'+ /** Present only when the proxy forwarded real upstream token counts. */+ usage?: { inputTokens: number; outputTokens: number }}export type CodexLlmProxyClientOptions = {diff --git a/mcp-host/src/llm/codexSubscription.ts b/mcp-host/src/llm/codexSubscription.tsindex c90e4d9c0..111b3c696 100644--- a/mcp-host/src/llm/codexSubscription.ts+++ b/mcp-host/src/llm/codexSubscription.ts@@ -29,6 +29,29 @@ export type CodexAttemptContext = {userId?: string}+function mapCodexUsage(usage?: { inputTokens: number; outputTokens: number }): {+ usage: { input_tokens: number; output_tokens: number; total_tokens: number }+ usage_reported: boolean+} {+ const inputTokens = usage?.inputTokens+ const outputTokens = usage?.outputTokens+ const usageKnown =+ typeof inputTokens === 'number' &&+ typeof outputTokens === 'number' &&+ Number.isInteger(inputTokens) &&+ Number.isInteger(outputTokens) &&+ inputTokens >= 0 &&+ outputTokens >= 0+ return {+ usage: {+ input_tokens: usageKnown ? inputTokens : 0,+ output_tokens: usageKnown ? outputTokens : 0,+ total_tokens: usageKnown ? inputTokens + outputTokens : 0,+ },+ usage_reported: usageKnown,+ }+}+export type CodexSubscriptionDeps = {authorizer: ProviderAttemptAuthorizerproxy: CodexLlmProxyClient@@ -36,6 +59,8 @@ export type CodexSubscriptionDeps = {}export class CodexSubscriptionProvider implements SingleTurnProvider {+ private nextProviderAttemptIndex = 1+constructor(private readonly model: string,private readonly deps: CodexSubscriptionDeps@@ -92,10 +117,11 @@ export class CodexSubscriptionProvider implements SingleTurnProvider {options?: { max_tokens?: number; temperature?: number; signal?: AbortSignal }): Promise<CompletionResponse> {const result = await this.execute(messages, undefined, options)+ const usage = mapCodexUsage(result.usage)return {content: result.text,- usage: { input_tokens: 0, output_tokens: 0, total_tokens: 0 },- usage_reported: false,+ usage: usage.usage,+ usage_reported: usage.usage_reported,finish_reason: result.outcome === 'canceled' ? FinishReason.Unknown : FinishReason.Stop,}}@@ -111,6 +137,7 @@ export class CodexSubscriptionProvider implements SingleTurnProvider {}): Promise<ToolCompletionResponse> {const result = await this.execute(messages, tools, options)+ const usage = mapCodexUsage(result.usage)return {content: result.text || null,tool_calls:@@ -121,8 +148,8 @@ export class CodexSubscriptionProvider implements SingleTurnProvider {arguments: call.arguments,})): null,- usage: { input_tokens: 0, output_tokens: 0, total_tokens: 0 },- usage_reported: false,+ usage: usage.usage,+ usage_reported: usage.usage_reported,finish_reason: result.toolCalls.length > 0 ? FinishReason.ToolUse : FinishReason.Stop,}}@@ -148,7 +175,7 @@ export class CodexSubscriptionProvider implements SingleTurnProvider {requestHash,invocationId: context.invocationId ?? request.requestId,attemptGeneration: context.attemptGeneration ?? 1,- providerAttemptIndex: context.providerAttemptIndex ?? 1,+ providerAttemptIndex: context.providerAttemptIndex ?? this.nextProviderAttemptIndex++,policyRevision: context.policyRevision,policyHash: context.policyHash,hostRef: context.hostRef,diff --git a/mcp-host/src/llm/failover/__tests__/engine.test.ts b/mcp-host/src/llm/failover/__tests__/engine.test.tsindex db8ae02fe..bb72466c0 100644--- a/mcp-host/src/llm/failover/__tests__/engine.test.ts+++ b/mcp-host/src/llm/failover/__tests__/engine.test.ts@@ -313,4 +313,64 @@ describe('FailoverEngine', () => {const res = await e.run(PRIMARY, () => () => Promise.resolve('primary'), classify)expect(res).toBe('primary')})++ it('assigns a distinct providerAttemptIndex to each physical attempt', async () => {+ const e = engine(+ policy({+ fallbacks: [+ { provider: 'openai', model: 'gpt-5.4' },+ { provider: 'zai', model: 'glm-5.1' },+ ],+ })+ )+ const indexes: number[] = []+ await e.run(+ PRIMARY,+ (t, physical) => {+ indexes.push(physical.providerAttemptIndex)+ if (t.kind === 'primary')+ return () => Promise.reject(llmError(LlmErrorCode.RateLimited, true))+ if (t.kind === 'fallback' && t.index === 0)+ return () => Promise.reject(llmError(LlmErrorCode.ModelOverloaded, true))+ return () => Promise.resolve('ok')+ },+ classify+ )+ expect(indexes).toEqual([1, 2, 3])+ })++ it('treats budget_denied as terminal unless the policy opts in', async () => {+ const err = new LlmError(+ 'budget denied',+ 'codex-subscription',+ LlmErrorCode.InsufficientQuota,+ false,+ undefined,+ undefined,+ 'budget_denied'+ )+ const classifyWithCode = (caught: unknown) => {+ if (caught instanceof LlmError) {+ return { code: caught.code, retryable: caught.retryable, providerCode: caught.providerCode }+ }+ return classify(caught)+ }+ const denied = engine()+ await expect(+ denied.run(+ PRIMARY,+ t => (t.kind === 'primary' ? () => Promise.reject(err) : () => Promise.resolve('nope')),+ classifyWithCode+ )+ ).rejects.toBe(err)++ const optedIn = engine(policy({ budgetDeniedFailover: true }))+ await expect(+ optedIn.run(+ PRIMARY,+ t => (t.kind === 'primary' ? () => Promise.reject(err) : () => Promise.resolve('fallback')),+ classifyWithCode+ )+ ).resolves.toBe('fallback')+ })})diff --git a/mcp-host/src/llm/failover/engine.ts b/mcp-host/src/llm/failover/engine.tsindex c5c8646aa..68857b632 100644--- a/mcp-host/src/llm/failover/engine.ts+++ b/mcp-host/src/llm/failover/engine.ts@@ -40,6 +40,8 @@ import type {export interface ClassifiedLike {code: LlmErrorCoderetryable: boolean+ /** Provider-native code (e.g. `budget_denied`) used for terminal gates. */+ providerCode?: string}export interface FailoverEngineOptions {@@ -76,7 +78,15 @@ export interface Attempt<T> {* thunk (served model = `target.model`) or an {@link Attempt} carrying an* explicit `servedModel` override.*/-export type AttemptBuilder<T> = (target: FailoverTarget) => (() => Promise<T>) | Attempt<T> | null+export type PhysicalAttempt = {+ /** 1-based index of this physical attempt in the engine's lifetime. */+ providerAttemptIndex: number+}++export type AttemptBuilder<T> = (+ target: FailoverTarget,+ physicalAttempt: PhysicalAttempt+) => (() => Promise<T>) | Attempt<T> | nullfunction pairLabel(p: ModelPair): string {return `${p.provider}/${p.model}`@@ -85,6 +95,7 @@ function pairLabel(p: ModelPair): string {export class FailoverEngine {private cooldownUntil: number | null = nullprivate served: ServedBy | null = null+ private nextProviderAttemptIndex = 1constructor(private policy: LlmPolicy,@@ -104,6 +115,7 @@ export class FailoverEngine {this.policy = policythis.cooldownUntil = nullthis.served = null+ this.nextProviderAttemptIndex = 1}/** Is this `(code, retryable)` eligible for fallback under the current policy? */@@ -177,8 +189,10 @@ export class FailoverEngine {let prevFailure: { pair: ModelPair; reason: FailoverClass } | null = nullfor (const target of targets) {- const built = build(target)+ const physicalAttempt = { providerAttemptIndex: this.nextProviderAttemptIndex }+ const built = build(target, physicalAttempt)if (!built) continue // not constructible → skip this target+ this.nextProviderAttemptIndex += 1const attempt = typeof built === 'function' ? built : built.run// Report the model REALLY served (a same-provider fallback serves the// session model, not `entry.model`) so `servedBy` + the metric's `to`@@ -198,6 +212,9 @@ export class FailoverEngine {} catch (err) {lastErr = errconst classified = classify(err)+ if (classified?.providerCode === 'budget_denied' && !this.policy.budgetDeniedFailover) {+ throw err+ }const failClass = classified? classifyFailoverClass(classified.code, classified.retryable): nulldiff --git a/mcp-host/src/llm/failover/policy.ts b/mcp-host/src/llm/failover/policy.tsindex 226779ee1..da087a8fe 100644--- a/mcp-host/src/llm/failover/policy.ts+++ b/mcp-host/src/llm/failover/policy.ts@@ -80,5 +80,10 @@ export function parseLlmPolicy(raw: unknown): LlmPolicy | null {if (fallbacks.length === 0) return nullconst { cooldownSeconds, triggerOn } = parseCooldownAndTriggers(rec)- return { cooldownSeconds, triggerOn, fallbacks }+ return {+ cooldownSeconds,+ triggerOn,+ fallbacks,+ ...(rec.budgetDeniedFailover === true ? { budgetDeniedFailover: true } : {}),+ }}diff --git a/mcp-host/src/llm/failover/types.ts b/mcp-host/src/llm/failover/types.tsindex f99f91b85..eff49ce35 100644--- a/mcp-host/src/llm/failover/types.ts+++ b/mcp-host/src/llm/failover/types.ts@@ -53,6 +53,12 @@ export interface LlmPolicy {triggerOn: FailoverClass[]/** Ordered fallback list, tried in order. Non-empty. */fallbacks: FallbackEntry[]+ /**+ * Token-budget denials (`providerCode=budget_denied`) are terminal by+ * default. Set true only when the operator explicitly opts a Host into+ * failing over after a Codex/control-plane budget block.+ */+ budgetDeniedFailover?: boolean}/** A target the engine attempts on a given call. */diff --git a/mcp-host/src/pluginWorkloadSdk/promptBridge/llmBridge.failover.test.ts b/mcp-host/src/pluginWorkloadSdk/promptBridge/llmBridge.failover.test.tsindex c6dad89c1..71f8dda5f 100644--- a/mcp-host/src/pluginWorkloadSdk/promptBridge/llmBridge.failover.test.ts+++ b/mcp-host/src/pluginWorkloadSdk/promptBridge/llmBridge.failover.test.ts@@ -573,4 +573,32 @@ describe('LlmBridge authorized multi-provider fallback', () => {)?.value).toBe(1)})++ it('authorizes a Codex target without redeeming a credential ticket or leaking secrets', async () => {+ const codex: PromptBridgeTarget = {+ targetRef: 'codex-primary',+ provider: 'codex-subscription',+ model: 'gpt-5.1',+ credentialSlot: '',+ }+ const provider = new FakeProvider(async () => ({+ ...OK,+ usage_reported: false,+ usage: { input_tokens: 0, output_tokens: 0, total_tokens: 0 },+ }))+ const { bridge, credentialCalls, resolver } = makeBridge({ 'gpt-5.1': provider })+ const issuer = { issue: vi.fn() }+ const result = await bridge.complete({+ ...request,+ targets: [{ target: codex }],+ credentialTicketIssuer: issuer,+ })+ expect(result.servedTarget).toEqual(codex)+ expect(result.llmSecretName).toBe('')+ expect(result.content).toBe('ok')+ expect(issuer.issue).not.toHaveBeenCalled()+ expect(resolver.resolve).not.toHaveBeenCalled()+ expect(credentialCalls).toEqual([])+ expect(JSON.stringify(result)).not.toMatch(/ticket|authorization|refreshToken|accessToken/i)+ })})diff --git a/mcp-host/src/pluginWorkloadSdk/promptBridge/llmBridge.ts b/mcp-host/src/pluginWorkloadSdk/promptBridge/llmBridge.tsindex 2438f34bb..e80c986c6 100644--- a/mcp-host/src/pluginWorkloadSdk/promptBridge/llmBridge.ts+++ b/mcp-host/src/pluginWorkloadSdk/promptBridge/llmBridge.ts@@ -2,7 +2,7 @@ import { LlmErrorCode } from '../../core/errors'import { type ChatMessage, FinishReason } from '../../core/types'import { type SingleTurnProvider, createLLMProvider } from '../../llm'import { type ClassifiedLike, type FailoverClass, classifyFailoverClass } from '../../llm/failover'-import { isLlmProvider } from '../../llm/registryCore'+import { descriptorFor, isLlmProvider } from '../../llm/registryCore'import { CircuitBreaker } from '../domain/circuitBreaker'import { PluginWorkloadError, type PluginWorkloadProviderAttemptContext } from '../domain/errors'import type { PromptBridgeTarget } from '../domain/types'@@ -293,6 +293,44 @@ export class LlmBridge {}const breaker = this.breakerFor(authorized.target)+ if (+ isLlmProvider(authorized.target.provider) &&+ descriptorFor(authorized.target.provider).authMode === 'oauth-broker'+ ) {+ try {+ const completion = await this.attempt(+ { target: authorized.target, keys: {}, llmSecretName: '' },+ { ...request, timeoutMs: remainingTimeoutMs(deadlineAt) },+ breaker+ )+ return {+ ...completion,+ servedTarget: authorized.target,+ fallbackUsed: index > 0,+ attemptCount: index + 1,+ llmSecretName: '',+ providerAttemptAcknowledgement: 'owned_by_finalizer',+ }+ } catch (error) {+ if (+ error instanceof ClassifiedProviderError &&+ error.classified.providerCode === 'budget_denied'+ ) {+ throw error.toPluginWorkloadError()+ }+ if (!(error instanceof ClassifiedProviderError)) throw error+ lastProviderError = error+ const failureClass = classifyFailoverClass(+ error.classified.code,+ error.classified.retryable+ )+ const eligible = failureClass !== null && triggerOn.has(failureClass)+ if (!eligible || index === request.targets.length - 1) {+ throw error.toPluginWorkloadError()+ }+ continue+ }+ }if (!breaker.allow()) {// A circuit-open target still consumes an ordered, auditable physical// attempt. Reserve and close it as `skipped` before advancing; without@@ -571,8 +609,9 @@ export class LlmBridge {}const inputTokens = response.usage?.input_tokens ?? 0const outputTokens = response.usage?.output_tokens ?? 0+ const brokerBacked = descriptorFor(providerId).authMode === 'oauth-broker'if (- response.usage_reported === false ||+ (!brokerBacked && response.usage_reported === false) ||inputTokens < 0 ||outputTokens < 0 ||!Number.isInteger(inputTokens) ||@@ -611,7 +650,11 @@ export class LlmBridge {const reason =classifyFailoverClass(classified.code, classified.retryable) ?? 'provider_unavailable'throw new ClassifiedProviderError(- { code: classified.code, retryable: classified.retryable },+ {+ code: classified.code,+ retryable: classified.retryable,+ ...(classified.providerCode ? { providerCode: classified.providerCode } : {}),+ },new PluginWorkloadError('provider_unavailable',`LLM provider error: ${classified.code}`,