Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
fix(mcp-host): retry a 429 once after its Retry-After
Alfredo Lopez9ca0e61fdd0f
diff --git a/mcp-host/package-lock.json b/mcp-host/package-lock.jsonindex 88ca0c35a..89c3be23a 100644--- a/mcp-host/package-lock.json+++ b/mcp-host/package-lock.json@@ -1,12 +1,12 @@{"name": "clerum-mcp-host",- "version": "0.1.447",+ "version": "0.1.448","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "clerum-mcp-host",- "version": "0.1.447",+ "version": "0.1.448","license": "SEE LICENSE IN LICENSE","dependencies": {"@anthropic-ai/sdk": "^0.32.0",diff --git a/mcp-host/package.json b/mcp-host/package.jsonindex 3f2614c66..1afe24588 100644--- a/mcp-host/package.json+++ b/mcp-host/package.json@@ -1,6 +1,6 @@{"name": "clerum-mcp-host",- "version": "0.1.447",+ "version": "0.1.448","description": "MCP Host - reads Host CRD and provides LLM access via OpenAI/Claude","main": "dist/main.js","engines": {diff --git a/mcp-host/src/llm/codexSubscription.ts b/mcp-host/src/llm/codexSubscription.tsindex 483cfc6e0..f8273364d 100644--- a/mcp-host/src/llm/codexSubscription.ts+++ b/mcp-host/src/llm/codexSubscription.ts@@ -20,6 +20,7 @@ import { logger } from '../logger'import { CodexLlmProxyClient, CodexProxyError } from './codexLlmProxyClient'import { classifyUnknown } from './errorClassification'import { CodexAuthorizeError, ProviderAttemptAuthorizer } from './providerAttemptAuthorizer'+import { rateLimitRetryDelayMs, waitBeforeRetry } from './rateLimitRetry'import { type LlmProvider, descriptorFor } from './registryCore'import type { ClassifiedError, SingleTurnProvider } from './types'@@ -616,43 +617,58 @@ export class CodexSubscriptionProvider implements SingleTurnProvider {) {throw new CodexAuthorizeError('no_grant', 'Codex catalog policy binding is missing')}- const providerAttemptIndex = context.providerAttemptIndex ?? this.nextProviderAttemptIndex++- const authorized = await this.deps.authorizer.authorize(- {+ const attempt = async (providerAttemptIndex: number) => {+ const authorized = await this.deps.authorizer.authorize(+ {+ request: wireRequest,+ requestHash,+ invocationId: context.invocationId ?? wireRequest.requestId,+ attemptGeneration: context.attemptGeneration ?? 1,+ providerAttemptIndex,+ policyRevision: context.policyRevision,+ policyHash: context.policyHash,+ hostRef: context.hostRef,+ recipeNamespace: context.recipeNamespace,+ recipeName: context.recipeName,+ userId: context.userId,+ ...(context.pluginWorkloadSdkProviderAttemptId+ ? { pluginWorkloadSdkProviderAttemptId: context.pluginWorkloadSdkProviderAttemptId }+ : {}),+ ...(context.targetRef ? { targetRef: context.targetRef } : {}),+ },+ // Authorize shares the caller's deadline. Without this the attempt could+ // keep a cancelled request alive on the control-api hop while the bridge+ // has already given up on it.+ { ...(options?.signal ? { signal: options.signal } : {}) }+ )+ if (!('accessToken' in authorized)) {+ // Bound: authorize returns ticket material only.+ }+ const streamed = await this.deps.proxy.stream({+ executionTicket: authorized.executionTicket,+ requestHash: authorized.requestHash,request: wireRequest,- requestHash,- invocationId: context.invocationId ?? wireRequest.requestId,- attemptGeneration: context.attemptGeneration ?? 1,+ signal: options?.signal,+ })+ return {+ ...streamed,+ providerAttemptId: authorized.providerAttemptId,providerAttemptIndex,- policyRevision: context.policyRevision,- policyHash: context.policyHash,- hostRef: context.hostRef,- recipeNamespace: context.recipeNamespace,- recipeName: context.recipeName,- userId: context.userId,- ...(context.pluginWorkloadSdkProviderAttemptId- ? { pluginWorkloadSdkProviderAttemptId: context.pluginWorkloadSdkProviderAttemptId }- : {}),- ...(context.targetRef ? { targetRef: context.targetRef } : {}),- },- // Authorize shares the caller's deadline. Without this the attempt could- // keep a cancelled request alive on the control-api hop while the bridge- // has already given up on it.- { ...(options?.signal ? { signal: options.signal } : {}) }- )- if (!('accessToken' in authorized)) {- // Bound: authorize returns ticket material only.- }- const streamed = await this.deps.proxy.stream({- executionTicket: authorized.executionTicket,- requestHash: authorized.requestHash,- request: wireRequest,- signal: options?.signal,- })- return {- ...streamed,- providerAttemptId: authorized.providerAttemptId,- providerAttemptIndex,+ }+ }+ try {+ return await attempt(context.providerAttemptIndex ?? this.nextProviderAttemptIndex++)+ } catch (err) {+ // G1-9 (#720): a 429 that advised a short Retry-After is retried once,+ // after that delay, under a new authorize: a redeemed ticket cannot be+ // reused. A caller that pins the attempt index owns its own retries.+ const waitMs =+ context.providerAttemptIndex === undefined && err instanceof CodexProxyError+ ? rateLimitRetryDelayMs(err.code, err.retryAfterMs)+ : undefined+ if (waitMs === undefined) throw err+ await waitBeforeRetry(waitMs, options?.signal)+ return attempt(this.nextProviderAttemptIndex++)}}diff --git a/mcp-host/src/llm/grokSubscription.ts b/mcp-host/src/llm/grokSubscription.tsindex 838bb2f59..9eae0436d 100644--- a/mcp-host/src/llm/grokSubscription.ts+++ b/mcp-host/src/llm/grokSubscription.ts@@ -16,6 +16,7 @@ import { logger } from '../logger'import { classifyUnknown } from './errorClassification'import { GrokLlmProxyClient, GrokProxyError } from './grokLlmProxyClient'import { CodexAuthorizeError, ProviderAttemptAuthorizer } from './providerAttemptAuthorizer'+import { rateLimitRetryDelayMs, waitBeforeRetry } from './rateLimitRetry'import type { LlmProvider } from './registryCore'import type { ClassifiedError, SingleTurnProvider } from './types'@@ -465,43 +466,58 @@ export class GrokSubscriptionProvider implements SingleTurnProvider {) {throw new CodexAuthorizeError('no_grant', 'Grok catalog policy binding is missing')}- const providerAttemptIndex = context.providerAttemptIndex ?? this.nextProviderAttemptIndex++- const authorized = await this.deps.authorizer.authorize(- {+ const attempt = async (providerAttemptIndex: number) => {+ const authorized = await this.deps.authorizer.authorize(+ {+ request,+ requestHash,+ invocationId: context.invocationId ?? request.requestId,+ attemptGeneration: context.attemptGeneration ?? 1,+ providerAttemptIndex,+ policyRevision: context.policyRevision,+ policyHash: context.policyHash,+ hostRef: context.hostRef,+ recipeNamespace: context.recipeNamespace,+ recipeName: context.recipeName,+ userId: context.userId,+ ...(context.pluginWorkloadSdkProviderAttemptId+ ? { pluginWorkloadSdkProviderAttemptId: context.pluginWorkloadSdkProviderAttemptId }+ : {}),+ ...(context.targetRef ? { targetRef: context.targetRef } : {}),+ },+ // Authorize shares the caller's deadline. Without this the attempt could+ // keep a cancelled request alive on the control-api hop while the bridge+ // has already given up on it.+ { ...(options?.signal ? { signal: options.signal } : {}) }+ )+ if (!('accessToken' in authorized)) {+ // Bound: authorize returns ticket material only.+ }+ const streamed = await this.deps.proxy.stream({+ executionTicket: authorized.executionTicket,+ requestHash: authorized.requestHash,request,- requestHash,- invocationId: context.invocationId ?? request.requestId,- attemptGeneration: context.attemptGeneration ?? 1,+ signal: options?.signal,+ })+ return {+ ...streamed,+ providerAttemptId: authorized.providerAttemptId,providerAttemptIndex,- policyRevision: context.policyRevision,- policyHash: context.policyHash,- hostRef: context.hostRef,- recipeNamespace: context.recipeNamespace,- recipeName: context.recipeName,- userId: context.userId,- ...(context.pluginWorkloadSdkProviderAttemptId- ? { pluginWorkloadSdkProviderAttemptId: context.pluginWorkloadSdkProviderAttemptId }- : {}),- ...(context.targetRef ? { targetRef: context.targetRef } : {}),- },- // Authorize shares the caller's deadline. Without this the attempt could- // keep a cancelled request alive on the control-api hop while the bridge- // has already given up on it.- { ...(options?.signal ? { signal: options.signal } : {}) }- )- if (!('accessToken' in authorized)) {- // Bound: authorize returns ticket material only.+ }}- const streamed = await this.deps.proxy.stream({- executionTicket: authorized.executionTicket,- requestHash: authorized.requestHash,- request,- signal: options?.signal,- })- return {- ...streamed,- providerAttemptId: authorized.providerAttemptId,- providerAttemptIndex,+ try {+ return await attempt(context.providerAttemptIndex ?? this.nextProviderAttemptIndex++)+ } catch (err) {+ // G1-9 (#720): a 429 that advised a short Retry-After is retried once,+ // after that delay, under a new authorize: a redeemed ticket cannot be+ // reused. A caller that pins the attempt index owns its own retries.+ const waitMs =+ context.providerAttemptIndex === undefined && err instanceof GrokProxyError+ ? rateLimitRetryDelayMs(err.code, err.retryAfterMs)+ : undefined+ if (waitMs === undefined) throw err+ await waitBeforeRetry(waitMs, options?.signal)+ return attempt(this.nextProviderAttemptIndex++)}}diff --git a/mcp-host/src/llm/rateLimitRetry.ts b/mcp-host/src/llm/rateLimitRetry.tsnew file mode 100644index 000000000..c52c247e1--- /dev/null+++ b/mcp-host/src/llm/rateLimitRetry.ts@@ -0,0 +1,33 @@+import { withAbort } from '../core/adapters/abortableLlmPort'++/**+ * G1-9 (#720): the longest Retry-After a subscription provider waits before its+ * single retry of a 429. Longer advice is thrown as `RateLimited`, so failover+ * and its cooldown decide instead of the turn being held open.+ */+export const RATE_LIMIT_RETRY_MAX_WAIT_MS = 30_000++/**+ * The wait before the single retry of a proxy error, or `undefined` when it is+ * not retried: only a `rate_limited` that carried a Retry-After within+ * {@link RATE_LIMIT_RETRY_MAX_WAIT_MS}. `control_plane_unavailable` and every+ * other code are never retried here.+ */+export function rateLimitRetryDelayMs(+ code: string,+ retryAfterMs: number | undefined+): number | undefined {+ if (code !== 'rate_limited' || retryAfterMs === undefined) return undefined+ return retryAfterMs <= RATE_LIMIT_RETRY_MAX_WAIT_MS ? retryAfterMs : undefined+}++/** Waits `ms`; an abort of `signal` rejects at once with the abort reason. */+export function waitBeforeRetry(ms: number, signal: AbortSignal | undefined): Promise<void> {+ let timer: ReturnType<typeof setTimeout> | undefined+ const wait = () =>+ new Promise<void>(resolve => {+ timer = setTimeout(resolve, ms)+ })+ if (!signal) return wait()+ return withAbort(wait, signal).finally(() => clearTimeout(timer))+}