Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

fix(mcp-host): retry a 429 once after its Retry-After

Alfredo Lopez9ca0e61fdd0f
diff --git a/mcp-host/package-lock.json b/mcp-host/package-lock.json
index 88ca0c35a..89c3be23a 100644
--- a/mcp-host/package-lock.json
+++ b/mcp-host/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "clerum-mcp-host",
- "version": "0.1.447",
+ "version": "0.1.448",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "clerum-mcp-host",
- "version": "0.1.447",
+ "version": "0.1.448",
"license": "SEE LICENSE IN LICENSE",
"dependencies": {
"@anthropic-ai/sdk": "^0.32.0",
diff --git a/mcp-host/package.json b/mcp-host/package.json
index 3f2614c66..1afe24588 100644
--- a/mcp-host/package.json
+++ b/mcp-host/package.json
@@ -1,6 +1,6 @@
{
"name": "clerum-mcp-host",
- "version": "0.1.447",
+ "version": "0.1.448",
"description": "MCP Host - reads Host CRD and provides LLM access via OpenAI/Claude",
"main": "dist/main.js",
"engines": {
diff --git a/mcp-host/src/llm/codexSubscription.ts b/mcp-host/src/llm/codexSubscription.ts
index 483cfc6e0..f8273364d 100644
--- a/mcp-host/src/llm/codexSubscription.ts
+++ b/mcp-host/src/llm/codexSubscription.ts
@@ -20,6 +20,7 @@ import { logger } from '../logger'
import { CodexLlmProxyClient, CodexProxyError } from './codexLlmProxyClient'
import { classifyUnknown } from './errorClassification'
import { CodexAuthorizeError, ProviderAttemptAuthorizer } from './providerAttemptAuthorizer'
+import { rateLimitRetryDelayMs, waitBeforeRetry } from './rateLimitRetry'
import { type LlmProvider, descriptorFor } from './registryCore'
import type { ClassifiedError, SingleTurnProvider } from './types'
@@ -616,43 +617,58 @@ export class CodexSubscriptionProvider implements SingleTurnProvider {
) {
throw new CodexAuthorizeError('no_grant', 'Codex catalog policy binding is missing')
}
- const providerAttemptIndex = context.providerAttemptIndex ?? this.nextProviderAttemptIndex++
- const authorized = await this.deps.authorizer.authorize(
- {
+ const attempt = async (providerAttemptIndex: number) => {
+ const authorized = await this.deps.authorizer.authorize(
+ {
+ request: wireRequest,
+ requestHash,
+ invocationId: context.invocationId ?? wireRequest.requestId,
+ attemptGeneration: context.attemptGeneration ?? 1,
+ providerAttemptIndex,
+ policyRevision: context.policyRevision,
+ policyHash: context.policyHash,
+ hostRef: context.hostRef,
+ recipeNamespace: context.recipeNamespace,
+ recipeName: context.recipeName,
+ userId: context.userId,
+ ...(context.pluginWorkloadSdkProviderAttemptId
+ ? { pluginWorkloadSdkProviderAttemptId: context.pluginWorkloadSdkProviderAttemptId }
+ : {}),
+ ...(context.targetRef ? { targetRef: context.targetRef } : {}),
+ },
+ // Authorize shares the caller's deadline. Without this the attempt could
+ // keep a cancelled request alive on the control-api hop while the bridge
+ // has already given up on it.
+ { ...(options?.signal ? { signal: options.signal } : {}) }
+ )
+ if (!('accessToken' in authorized)) {
+ // Bound: authorize returns ticket material only.
+ }
+ const streamed = await this.deps.proxy.stream({
+ executionTicket: authorized.executionTicket,
+ requestHash: authorized.requestHash,
request: wireRequest,
- requestHash,
- invocationId: context.invocationId ?? wireRequest.requestId,
- attemptGeneration: context.attemptGeneration ?? 1,
+ signal: options?.signal,
+ })
+ return {
+ ...streamed,
+ providerAttemptId: authorized.providerAttemptId,
providerAttemptIndex,
- policyRevision: context.policyRevision,
- policyHash: context.policyHash,
- hostRef: context.hostRef,
- recipeNamespace: context.recipeNamespace,
- recipeName: context.recipeName,
- userId: context.userId,
- ...(context.pluginWorkloadSdkProviderAttemptId
- ? { pluginWorkloadSdkProviderAttemptId: context.pluginWorkloadSdkProviderAttemptId }
- : {}),
- ...(context.targetRef ? { targetRef: context.targetRef } : {}),
- },
- // Authorize shares the caller's deadline. Without this the attempt could
- // keep a cancelled request alive on the control-api hop while the bridge
- // has already given up on it.
- { ...(options?.signal ? { signal: options.signal } : {}) }
- )
- if (!('accessToken' in authorized)) {
- // Bound: authorize returns ticket material only.
- }
- const streamed = await this.deps.proxy.stream({
- executionTicket: authorized.executionTicket,
- requestHash: authorized.requestHash,
- request: wireRequest,
- signal: options?.signal,
- })
- return {
- ...streamed,
- providerAttemptId: authorized.providerAttemptId,
- providerAttemptIndex,
+ }
+ }
+ try {
+ return await attempt(context.providerAttemptIndex ?? this.nextProviderAttemptIndex++)
+ } catch (err) {
+ // G1-9 (#720): a 429 that advised a short Retry-After is retried once,
+ // after that delay, under a new authorize: a redeemed ticket cannot be
+ // reused. A caller that pins the attempt index owns its own retries.
+ const waitMs =
+ context.providerAttemptIndex === undefined && err instanceof CodexProxyError
+ ? rateLimitRetryDelayMs(err.code, err.retryAfterMs)
+ : undefined
+ if (waitMs === undefined) throw err
+ await waitBeforeRetry(waitMs, options?.signal)
+ return attempt(this.nextProviderAttemptIndex++)
}
}
diff --git a/mcp-host/src/llm/grokSubscription.ts b/mcp-host/src/llm/grokSubscription.ts
index 838bb2f59..9eae0436d 100644
--- a/mcp-host/src/llm/grokSubscription.ts
+++ b/mcp-host/src/llm/grokSubscription.ts
@@ -16,6 +16,7 @@ import { logger } from '../logger'
import { classifyUnknown } from './errorClassification'
import { GrokLlmProxyClient, GrokProxyError } from './grokLlmProxyClient'
import { CodexAuthorizeError, ProviderAttemptAuthorizer } from './providerAttemptAuthorizer'
+import { rateLimitRetryDelayMs, waitBeforeRetry } from './rateLimitRetry'
import type { LlmProvider } from './registryCore'
import type { ClassifiedError, SingleTurnProvider } from './types'
@@ -465,43 +466,58 @@ export class GrokSubscriptionProvider implements SingleTurnProvider {
) {
throw new CodexAuthorizeError('no_grant', 'Grok catalog policy binding is missing')
}
- const providerAttemptIndex = context.providerAttemptIndex ?? this.nextProviderAttemptIndex++
- const authorized = await this.deps.authorizer.authorize(
- {
+ const attempt = async (providerAttemptIndex: number) => {
+ const authorized = await this.deps.authorizer.authorize(
+ {
+ request,
+ requestHash,
+ invocationId: context.invocationId ?? request.requestId,
+ attemptGeneration: context.attemptGeneration ?? 1,
+ providerAttemptIndex,
+ policyRevision: context.policyRevision,
+ policyHash: context.policyHash,
+ hostRef: context.hostRef,
+ recipeNamespace: context.recipeNamespace,
+ recipeName: context.recipeName,
+ userId: context.userId,
+ ...(context.pluginWorkloadSdkProviderAttemptId
+ ? { pluginWorkloadSdkProviderAttemptId: context.pluginWorkloadSdkProviderAttemptId }
+ : {}),
+ ...(context.targetRef ? { targetRef: context.targetRef } : {}),
+ },
+ // Authorize shares the caller's deadline. Without this the attempt could
+ // keep a cancelled request alive on the control-api hop while the bridge
+ // has already given up on it.
+ { ...(options?.signal ? { signal: options.signal } : {}) }
+ )
+ if (!('accessToken' in authorized)) {
+ // Bound: authorize returns ticket material only.
+ }
+ const streamed = await this.deps.proxy.stream({
+ executionTicket: authorized.executionTicket,
+ requestHash: authorized.requestHash,
request,
- requestHash,
- invocationId: context.invocationId ?? request.requestId,
- attemptGeneration: context.attemptGeneration ?? 1,
+ signal: options?.signal,
+ })
+ return {
+ ...streamed,
+ providerAttemptId: authorized.providerAttemptId,
providerAttemptIndex,
- policyRevision: context.policyRevision,
- policyHash: context.policyHash,
- hostRef: context.hostRef,
- recipeNamespace: context.recipeNamespace,
- recipeName: context.recipeName,
- userId: context.userId,
- ...(context.pluginWorkloadSdkProviderAttemptId
- ? { pluginWorkloadSdkProviderAttemptId: context.pluginWorkloadSdkProviderAttemptId }
- : {}),
- ...(context.targetRef ? { targetRef: context.targetRef } : {}),
- },
- // Authorize shares the caller's deadline. Without this the attempt could
- // keep a cancelled request alive on the control-api hop while the bridge
- // has already given up on it.
- { ...(options?.signal ? { signal: options.signal } : {}) }
- )
- if (!('accessToken' in authorized)) {
- // Bound: authorize returns ticket material only.
+ }
}
- const streamed = await this.deps.proxy.stream({
- executionTicket: authorized.executionTicket,
- requestHash: authorized.requestHash,
- request,
- signal: options?.signal,
- })
- return {
- ...streamed,
- providerAttemptId: authorized.providerAttemptId,
- providerAttemptIndex,
+ try {
+ return await attempt(context.providerAttemptIndex ?? this.nextProviderAttemptIndex++)
+ } catch (err) {
+ // G1-9 (#720): a 429 that advised a short Retry-After is retried once,
+ // after that delay, under a new authorize: a redeemed ticket cannot be
+ // reused. A caller that pins the attempt index owns its own retries.
+ const waitMs =
+ context.providerAttemptIndex === undefined && err instanceof GrokProxyError
+ ? rateLimitRetryDelayMs(err.code, err.retryAfterMs)
+ : undefined
+ if (waitMs === undefined) throw err
+ await waitBeforeRetry(waitMs, options?.signal)
+ return attempt(this.nextProviderAttemptIndex++)
}
}
diff --git a/mcp-host/src/llm/rateLimitRetry.ts b/mcp-host/src/llm/rateLimitRetry.ts
new file mode 100644
index 000000000..c52c247e1
--- /dev/null
+++ b/mcp-host/src/llm/rateLimitRetry.ts
@@ -0,0 +1,33 @@
+import { withAbort } from '../core/adapters/abortableLlmPort'
+
+/**
+ * G1-9 (#720): the longest Retry-After a subscription provider waits before its
+ * single retry of a 429. Longer advice is thrown as `RateLimited`, so failover
+ * and its cooldown decide instead of the turn being held open.
+ */
+export const RATE_LIMIT_RETRY_MAX_WAIT_MS = 30_000
+
+/**
+ * The wait before the single retry of a proxy error, or `undefined` when it is
+ * not retried: only a `rate_limited` that carried a Retry-After within
+ * {@link RATE_LIMIT_RETRY_MAX_WAIT_MS}. `control_plane_unavailable` and every
+ * other code are never retried here.
+ */
+export function rateLimitRetryDelayMs(
+ code: string,
+ retryAfterMs: number | undefined
+): number | undefined {
+ if (code !== 'rate_limited' || retryAfterMs === undefined) return undefined
+ return retryAfterMs <= RATE_LIMIT_RETRY_MAX_WAIT_MS ? retryAfterMs : undefined
+}
+
+/** Waits `ms`; an abort of `signal` rejects at once with the abort reason. */
+export function waitBeforeRetry(ms: number, signal: AbortSignal | undefined): Promise<void> {
+ let timer: ReturnType<typeof setTimeout> | undefined
+ const wait = () =>
+ new Promise<void>(resolve => {
+ timer = setTimeout(resolve, ms)
+ })
+ if (!signal) return wait()
+ return withAbort(wait, signal).finally(() => clearTimeout(timer))
+}