Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

test(mcp-host): require one bounded retry of a 429 after its Retry-After

Alfredo Lopez5280f1b962d3
diff --git a/mcp-host/package-lock.json b/mcp-host/package-lock.json
index d32a2ca19..88ca0c35a 100644
--- a/mcp-host/package-lock.json
+++ b/mcp-host/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "clerum-mcp-host",
- "version": "0.1.446",
+ "version": "0.1.447",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "clerum-mcp-host",
- "version": "0.1.446",
+ "version": "0.1.447",
"license": "SEE LICENSE IN LICENSE",
"dependencies": {
"@anthropic-ai/sdk": "^0.32.0",
diff --git a/mcp-host/package.json b/mcp-host/package.json
index 0f9eb847c..3f2614c66 100644
--- a/mcp-host/package.json
+++ b/mcp-host/package.json
@@ -1,6 +1,6 @@
{
"name": "clerum-mcp-host",
- "version": "0.1.446",
+ "version": "0.1.447",
"description": "MCP Host - reads Host CRD and provides LLM access via OpenAI/Claude",
"main": "dist/main.js",
"engines": {
diff --git a/mcp-host/src/llm/__tests__/codexSubscription.test.ts b/mcp-host/src/llm/__tests__/codexSubscription.test.ts
index d98849f2c..ed3a28615 100644
--- a/mcp-host/src/llm/__tests__/codexSubscription.test.ts
+++ b/mcp-host/src/llm/__tests__/codexSubscription.test.ts
@@ -1,4 +1,4 @@
-import { describe, expect, it, vi } from 'vitest'
+import { afterEach, describe, expect, it, vi } from 'vitest'
import { LIMITS } from '@clerum/llm-provider-attempt-contract'
import {
VISUAL_LIMITS,
@@ -1447,3 +1447,172 @@ describe('CodexSubscriptionProvider', () => {
expect(JSON.stringify(request)).not.toContain('contentParts')
})
})
+
+describe('CodexSubscriptionProvider Retry-After retry (G1-9, #720)', () => {
+ afterEach(() => {
+ vi.useRealTimers()
+ })
+
+ const ok = { text: 'after the wait', toolCalls: [], outcome: 'success' }
+ const limited = (retryAfterMs?: number) =>
+ new CodexProxyError(
+ 'rate_limited',
+ 'proxy stream failed with 429 (rate_limited)',
+ true,
+ retryAfterMs
+ )
+ const authorizeTwice = () =>
+ vi
+ .fn()
+ .mockResolvedValueOnce({
+ providerAttemptId: 'attempt-1',
+ requestHash,
+ executionTicket: 'ticket-first',
+ expiresAt: '2026-08-20T10:00:00.000Z',
+ })
+ .mockResolvedValueOnce({
+ providerAttemptId: 'attempt-2',
+ requestHash,
+ executionTicket: 'ticket-second',
+ expiresAt: '2026-08-20T10:00:00.000Z',
+ })
+
+ it('G1-9a: waits the advised delay, re-authorizes with the next index and dispatches again', async () => {
+ vi.useFakeTimers()
+ const stream = vi.fn().mockRejectedValueOnce(limited(2000)).mockResolvedValueOnce(ok)
+ const wired = deps({ authorize: authorizeTwice(), stream })
+ const provider = new CodexSubscriptionProvider('gpt-5.3-codex', wired as never)
+
+ const turn = provider.completeSingleTurn([{ role: 'user', content: 'hi' }])
+ await vi.advanceTimersByTimeAsync(1999)
+ // The advised delay is honoured: nothing is re-sent before it elapses.
+ expect(wired.authorize).toHaveBeenCalledTimes(1)
+ expect(wired.stream).toHaveBeenCalledTimes(1)
+ await vi.advanceTimersByTimeAsync(1)
+ const result = await turn
+
+ expect(result.content).toBe('after the wait')
+ expect(wired.authorize).toHaveBeenCalledTimes(2)
+ const [first, second] = wired.authorize.mock.calls.map(call => call[0])
+ expect(second.invocationId).toBe(first.invocationId)
+ expect(second.requestHash).toBe(first.requestHash)
+ expect(first.providerAttemptIndex).toBe(1)
+ expect(second.providerAttemptIndex).toBe(2)
+ expect(wired.stream).toHaveBeenCalledTimes(2)
+ expect(wired.stream.mock.calls[1][0].executionTicket).toBe('ticket-second')
+ })
+
+ it('G1-9a: retries at exactly the 30 s cap', async () => {
+ vi.useFakeTimers()
+ const stream = vi.fn().mockRejectedValueOnce(limited(30_000)).mockResolvedValueOnce(ok)
+ const wired = deps({ authorize: authorizeTwice(), stream })
+ const provider = new CodexSubscriptionProvider('gpt-5.3-codex', wired as never)
+
+ const turn = provider.completeSingleTurn([{ role: 'user', content: 'hi' }])
+ await vi.advanceTimersByTimeAsync(30_000)
+ await expect(turn).resolves.toMatchObject({ content: 'after the wait' })
+ expect(wired.authorize).toHaveBeenCalledTimes(2)
+ })
+
+ it.each([
+ ['G1-9b: without Retry-After', undefined],
+ ['G1-9c: over the 30 s cap', 31_000],
+ ])('%s the 429 is thrown with no retry', async (_label, retryAfterMs) => {
+ vi.useFakeTimers()
+ const err = limited(retryAfterMs)
+ const wired = deps({ stream: vi.fn().mockRejectedValueOnce(err).mockResolvedValueOnce(ok) })
+ const provider = new CodexSubscriptionProvider('gpt-5.3-codex', wired as never)
+
+ const turn = provider.completeSingleTurn([{ role: 'user', content: 'hi' }])
+ const settled = expect(turn).rejects.toBe(err)
+ await vi.advanceTimersByTimeAsync(60_000)
+ await settled
+ // Witness: the first attempt ran; the negative is that no second one did.
+ expect(wired.authorize).toHaveBeenCalledTimes(1)
+ expect(wired.stream).toHaveBeenCalledTimes(1)
+ expect(provider.classifyError(err).code).toBe(LlmErrorCode.RateLimited)
+ })
+
+ it('G1-9d: a second 429 is thrown as RateLimited with no third attempt', async () => {
+ vi.useFakeTimers()
+ const second = limited(1000)
+ const stream = vi.fn().mockRejectedValueOnce(limited(1000)).mockRejectedValueOnce(second)
+ const wired = deps({ authorize: authorizeTwice(), stream })
+ const provider = new CodexSubscriptionProvider('gpt-5.3-codex', wired as never)
+
+ const turn = provider.completeSingleTurn([{ role: 'user', content: 'hi' }])
+ const settled = expect(turn).rejects.toBe(second)
+ await vi.advanceTimersByTimeAsync(60_000)
+ await settled
+ expect(wired.authorize).toHaveBeenCalledTimes(2)
+ expect(wired.stream).toHaveBeenCalledTimes(2)
+ expect(provider.classifyError(second)).toMatchObject({
+ code: LlmErrorCode.RateLimited,
+ retryable: true,
+ })
+ })
+
+ it('G1-9e: a caller-pinned attempt index is never retried here', async () => {
+ vi.useFakeTimers()
+ const err = limited(1000)
+ const wired = deps({ stream: vi.fn().mockRejectedValueOnce(err).mockResolvedValueOnce(ok) })
+ wired.attemptContext = vi.fn(() => ({
+ policyRevision: 1,
+ policyHash: 'b'.repeat(64),
+ hostRef: 'chatllm',
+ providerAttemptIndex: 3,
+ }))
+ const provider = new CodexSubscriptionProvider('gpt-5.3-codex', wired as never)
+
+ const turn = provider.completeSingleTurn([{ role: 'user', content: 'hi' }])
+ const settled = expect(turn).rejects.toBe(err)
+ await vi.advanceTimersByTimeAsync(60_000)
+ await settled
+ expect(wired.authorize).toHaveBeenCalledTimes(1)
+ expect(wired.authorize.mock.calls[0][0].providerAttemptIndex).toBe(3)
+ expect(wired.stream).toHaveBeenCalledTimes(1)
+ })
+
+ it('G1-9f: an abort during the wait rejects with the abort reason and sends nothing more', async () => {
+ vi.useFakeTimers()
+ const wired = deps({
+ authorize: authorizeTwice(),
+ stream: vi.fn().mockRejectedValueOnce(limited(5000)).mockResolvedValueOnce(ok),
+ })
+ const provider = new CodexSubscriptionProvider('gpt-5.3-codex', wired as never)
+ const controller = new AbortController()
+ const reason = new Error('caller gave up')
+
+ const turn = provider.completeSingleTurn([{ role: 'user', content: 'hi' }], {
+ signal: controller.signal,
+ })
+ const settled = expect(turn).rejects.toBe(reason)
+ await vi.advanceTimersByTimeAsync(1000)
+ expect(wired.stream).toHaveBeenCalledTimes(1)
+ controller.abort(reason)
+ await settled
+ await vi.advanceTimersByTimeAsync(60_000)
+ expect(wired.authorize).toHaveBeenCalledTimes(1)
+ expect(wired.stream).toHaveBeenCalledTimes(1)
+ })
+
+ it('G1-9g: control_plane_unavailable is never retried here', async () => {
+ vi.useFakeTimers()
+ // Carries a delay on purpose, so only the code keeps it from being retried.
+ const err = new CodexProxyError(
+ 'control_plane_unavailable',
+ 'proxy could not be reached',
+ true,
+ 1000
+ )
+ const wired = deps({ stream: vi.fn().mockRejectedValueOnce(err).mockResolvedValueOnce(ok) })
+ const provider = new CodexSubscriptionProvider('gpt-5.3-codex', wired as never)
+
+ const turn = provider.completeSingleTurn([{ role: 'user', content: 'hi' }])
+ const settled = expect(turn).rejects.toBe(err)
+ await vi.advanceTimersByTimeAsync(60_000)
+ await settled
+ expect(wired.authorize).toHaveBeenCalledTimes(1)
+ expect(wired.stream).toHaveBeenCalledTimes(1)
+ })
+})
diff --git a/mcp-host/src/llm/__tests__/grokSubscription.test.ts b/mcp-host/src/llm/__tests__/grokSubscription.test.ts
index 73c5ee63a..d7d313993 100644
--- a/mcp-host/src/llm/__tests__/grokSubscription.test.ts
+++ b/mcp-host/src/llm/__tests__/grokSubscription.test.ts
@@ -1,4 +1,4 @@
-import { describe, expect, it, vi } from 'vitest'
+import { afterEach, describe, expect, it, vi } from 'vitest'
import { LIMITS } from '@clerum/grok-provider-attempt-contract'
import { minifiedMcpResult } from '../../__tests__/fixtures/minifiedMcpResult'
import { LlmErrorCode } from '../../core/errors'
@@ -9,13 +9,18 @@ import { CodexAuthorizeError } from '../providerAttemptAuthorizer'
const requestHash = 'a'.repeat(64)
-function deps(overrides?: { stream?: ReturnType<typeof vi.fn> }) {
- const authorize = vi.fn().mockResolvedValue({
- providerAttemptId: 'attempt-1',
- requestHash,
- executionTicket: 'ticket-123456',
- expiresAt: '2026-08-20T10:00:00.000Z',
- })
+function deps(overrides?: {
+ authorize?: ReturnType<typeof vi.fn>
+ stream?: ReturnType<typeof vi.fn>
+}) {
+ const authorize =
+ overrides?.authorize ??
+ vi.fn().mockResolvedValue({
+ providerAttemptId: 'attempt-1',
+ requestHash,
+ executionTicket: 'ticket-123456',
+ expiresAt: '2026-08-20T10:00:00.000Z',
+ })
const stream =
overrides?.stream ??
vi.fn().mockResolvedValue({
@@ -827,3 +832,172 @@ describe('GrokSubscriptionProvider', () => {
expect(wired.authorizer.authorize).toHaveBeenNthCalledWith(2, expect.any(Object), {})
})
})
+
+describe('GrokSubscriptionProvider Retry-After retry (G1-9, #720)', () => {
+ afterEach(() => {
+ vi.useRealTimers()
+ })
+
+ const ok = { text: 'after the wait', toolCalls: [], outcome: 'success' }
+ const limited = (retryAfterMs?: number) =>
+ new GrokProxyError(
+ 'rate_limited',
+ 'proxy stream failed with 429 (rate_limited)',
+ true,
+ retryAfterMs
+ )
+ const authorizeTwice = () =>
+ vi
+ .fn()
+ .mockResolvedValueOnce({
+ providerAttemptId: 'attempt-1',
+ requestHash,
+ executionTicket: 'ticket-first',
+ expiresAt: '2026-08-20T10:00:00.000Z',
+ })
+ .mockResolvedValueOnce({
+ providerAttemptId: 'attempt-2',
+ requestHash,
+ executionTicket: 'ticket-second',
+ expiresAt: '2026-08-20T10:00:00.000Z',
+ })
+
+ it('G1-9a: waits the advised delay, re-authorizes with the next index and dispatches again', async () => {
+ vi.useFakeTimers()
+ const stream = vi.fn().mockRejectedValueOnce(limited(2000)).mockResolvedValueOnce(ok)
+ const wired = deps({ authorize: authorizeTwice(), stream })
+ const provider = new GrokSubscriptionProvider('grok-4.6', wired as never)
+
+ const turn = provider.completeSingleTurn([{ role: 'user', content: 'hi' }])
+ await vi.advanceTimersByTimeAsync(1999)
+ // The advised delay is honoured: nothing is re-sent before it elapses.
+ expect(wired.authorizer.authorize).toHaveBeenCalledTimes(1)
+ expect(wired.proxy.stream).toHaveBeenCalledTimes(1)
+ await vi.advanceTimersByTimeAsync(1)
+ const result = await turn
+
+ expect(result.content).toBe('after the wait')
+ expect(wired.authorizer.authorize).toHaveBeenCalledTimes(2)
+ const [first, second] = wired.authorizer.authorize.mock.calls.map(call => call[0])
+ expect(second.invocationId).toBe(first.invocationId)
+ expect(second.requestHash).toBe(first.requestHash)
+ expect(first.providerAttemptIndex).toBe(1)
+ expect(second.providerAttemptIndex).toBe(2)
+ expect(wired.proxy.stream).toHaveBeenCalledTimes(2)
+ expect(wired.proxy.stream.mock.calls[1][0].executionTicket).toBe('ticket-second')
+ })
+
+ it('G1-9a: retries at exactly the 30 s cap', async () => {
+ vi.useFakeTimers()
+ const stream = vi.fn().mockRejectedValueOnce(limited(30_000)).mockResolvedValueOnce(ok)
+ const wired = deps({ authorize: authorizeTwice(), stream })
+ const provider = new GrokSubscriptionProvider('grok-4.6', wired as never)
+
+ const turn = provider.completeSingleTurn([{ role: 'user', content: 'hi' }])
+ await vi.advanceTimersByTimeAsync(30_000)
+ await expect(turn).resolves.toMatchObject({ content: 'after the wait' })
+ expect(wired.authorizer.authorize).toHaveBeenCalledTimes(2)
+ })
+
+ it.each([
+ ['G1-9b: without Retry-After', undefined],
+ ['G1-9c: over the 30 s cap', 31_000],
+ ])('%s the 429 is thrown with no retry', async (_label, retryAfterMs) => {
+ vi.useFakeTimers()
+ const err = limited(retryAfterMs)
+ const wired = deps({ stream: vi.fn().mockRejectedValueOnce(err).mockResolvedValueOnce(ok) })
+ const provider = new GrokSubscriptionProvider('grok-4.6', wired as never)
+
+ const turn = provider.completeSingleTurn([{ role: 'user', content: 'hi' }])
+ const settled = expect(turn).rejects.toBe(err)
+ await vi.advanceTimersByTimeAsync(60_000)
+ await settled
+ // Witness: the first attempt ran; the negative is that no second one did.
+ expect(wired.authorizer.authorize).toHaveBeenCalledTimes(1)
+ expect(wired.proxy.stream).toHaveBeenCalledTimes(1)
+ expect(provider.classifyError(err).code).toBe(LlmErrorCode.RateLimited)
+ })
+
+ it('G1-9d: a second 429 is thrown as RateLimited with no third attempt', async () => {
+ vi.useFakeTimers()
+ const second = limited(1000)
+ const stream = vi.fn().mockRejectedValueOnce(limited(1000)).mockRejectedValueOnce(second)
+ const wired = deps({ authorize: authorizeTwice(), stream })
+ const provider = new GrokSubscriptionProvider('grok-4.6', wired as never)
+
+ const turn = provider.completeSingleTurn([{ role: 'user', content: 'hi' }])
+ const settled = expect(turn).rejects.toBe(second)
+ await vi.advanceTimersByTimeAsync(60_000)
+ await settled
+ expect(wired.authorizer.authorize).toHaveBeenCalledTimes(2)
+ expect(wired.proxy.stream).toHaveBeenCalledTimes(2)
+ expect(provider.classifyError(second)).toMatchObject({
+ code: LlmErrorCode.RateLimited,
+ retryable: true,
+ })
+ })
+
+ it('G1-9e: a caller-pinned attempt index is never retried here', async () => {
+ vi.useFakeTimers()
+ const err = limited(1000)
+ const wired = deps({ stream: vi.fn().mockRejectedValueOnce(err).mockResolvedValueOnce(ok) })
+ wired.attemptContext = vi.fn(() => ({
+ policyRevision: 1,
+ policyHash: 'b'.repeat(64),
+ hostRef: 'chatllm',
+ providerAttemptIndex: 3,
+ }))
+ const provider = new GrokSubscriptionProvider('grok-4.6', wired as never)
+
+ const turn = provider.completeSingleTurn([{ role: 'user', content: 'hi' }])
+ const settled = expect(turn).rejects.toBe(err)
+ await vi.advanceTimersByTimeAsync(60_000)
+ await settled
+ expect(wired.authorizer.authorize).toHaveBeenCalledTimes(1)
+ expect(wired.authorizer.authorize.mock.calls[0][0].providerAttemptIndex).toBe(3)
+ expect(wired.proxy.stream).toHaveBeenCalledTimes(1)
+ })
+
+ it('G1-9f: an abort during the wait rejects with the abort reason and sends nothing more', async () => {
+ vi.useFakeTimers()
+ const wired = deps({
+ authorize: authorizeTwice(),
+ stream: vi.fn().mockRejectedValueOnce(limited(5000)).mockResolvedValueOnce(ok),
+ })
+ const provider = new GrokSubscriptionProvider('grok-4.6', wired as never)
+ const controller = new AbortController()
+ const reason = new Error('caller gave up')
+
+ const turn = provider.completeSingleTurn([{ role: 'user', content: 'hi' }], {
+ signal: controller.signal,
+ })
+ const settled = expect(turn).rejects.toBe(reason)
+ await vi.advanceTimersByTimeAsync(1000)
+ expect(wired.proxy.stream).toHaveBeenCalledTimes(1)
+ controller.abort(reason)
+ await settled
+ await vi.advanceTimersByTimeAsync(60_000)
+ expect(wired.authorizer.authorize).toHaveBeenCalledTimes(1)
+ expect(wired.proxy.stream).toHaveBeenCalledTimes(1)
+ })
+
+ it('G1-9g: control_plane_unavailable is never retried here', async () => {
+ vi.useFakeTimers()
+ // Carries a delay on purpose, so only the code keeps it from being retried.
+ const err = new GrokProxyError(
+ 'control_plane_unavailable',
+ 'proxy could not be reached',
+ true,
+ 1000
+ )
+ const wired = deps({ stream: vi.fn().mockRejectedValueOnce(err).mockResolvedValueOnce(ok) })
+ const provider = new GrokSubscriptionProvider('grok-4.6', wired as never)
+
+ const turn = provider.completeSingleTurn([{ role: 'user', content: 'hi' }])
+ const settled = expect(turn).rejects.toBe(err)
+ await vi.advanceTimersByTimeAsync(60_000)
+ await settled
+ expect(wired.authorizer.authorize).toHaveBeenCalledTimes(1)
+ expect(wired.proxy.stream).toHaveBeenCalledTimes(1)
+ })
+})