Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

fix(authorize): retry a capacity refusal once in the same provider

Alfredo Lopez62649a512b16
diff --git a/control-api/package-lock.json b/control-api/package-lock.json
index 17ea1b018..4c05778ab 100644
--- a/control-api/package-lock.json
+++ b/control-api/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "clerum-control-api",
- "version": "0.1.720",
+ "version": "0.1.721",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "clerum-control-api",
- "version": "0.1.720",
+ "version": "0.1.721",
"dependencies": {
"@clerum/display-field": "file:../packages/display-field",
"@clerum/grok-provider-attempt-contract": "file:../packages/grok-provider-attempt-contract",
diff --git a/control-api/package.json b/control-api/package.json
index a3cbb2738..1b5e6ed95 100644
--- a/control-api/package.json
+++ b/control-api/package.json
@@ -1,6 +1,6 @@
{
"name": "clerum-control-api",
- "version": "0.1.720",
+ "version": "0.1.721",
"description": "Internal Clerum control plane REST API + UI for managing CRDs and secrets",
"main": "dist/main.js",
"engines": {
diff --git a/control-api/src/middleware/llmProviderAttemptAdmissionLimits.ts b/control-api/src/middleware/llmProviderAttemptAdmissionLimits.ts
index 30a09fbde..d8b45dd29 100644
--- a/control-api/src/middleware/llmProviderAttemptAdmissionLimits.ts
+++ b/control-api/src/middleware/llmProviderAttemptAdmissionLimits.ts
@@ -15,6 +15,11 @@ import type { AuthorizeBodyAdmissionPolicy } from './llmProviderAttemptBodyAdmis
// share stops one principal holding all three positions. queueWaitMs is the
// longest a healthy holder can keep the unit (its read deadline plus its work
// deadline), so a waiter is refused only when the holder outlives both.
+//
+// retryAfterSeconds is the Retry-After on a capacity refusal. The Host retries
+// once in the same provider after it, re-uploading the body, and the second
+// refusal is terminal. It equals the read deadline, the bound on a slow
+// upload, and stays below the Host's 30 s retry ceiling. Uncertified (#813).
const READ_DEADLINE_MS = 10_000
const WORK_DEADLINE_MS = 30_000
@@ -26,4 +31,5 @@ export const LLM_PROVIDER_ATTEMPT_ADMISSION_POLICY: AuthorizeBodyAdmissionPolicy
readDeadlineMs: READ_DEADLINE_MS,
workDeadlineMs: WORK_DEADLINE_MS,
closeGraceMs: 250,
+ retryAfterSeconds: READ_DEADLINE_MS / 1000,
})
diff --git a/control-api/src/middleware/llmProviderAttemptBodyAdmission.ts b/control-api/src/middleware/llmProviderAttemptBodyAdmission.ts
index ab17e454e..fa6c79213 100644
--- a/control-api/src/middleware/llmProviderAttemptBodyAdmission.ts
+++ b/control-api/src/middleware/llmProviderAttemptBodyAdmission.ts
@@ -10,6 +10,8 @@ export type AuthorizeBodyAdmissionPolicy = Readonly<{
readDeadlineMs: number
workDeadlineMs: number
closeGraceMs: number
+ /** `Retry-After` on a capacity refusal; the Host retries once after it. */
+ retryAfterSeconds: number
}>
// The route supplies these claims only after requireMcpHostJwt succeeds.
@@ -75,6 +77,14 @@ export class AuthorizeBodyAdmission {
throw new TypeError('authorize timers must be positive signed-32-bit integers')
}
}
+ // The Host honours 1-3600 s (mcp-host `retryAfter.ts`).
+ if (
+ !Number.isSafeInteger(policy.retryAfterSeconds) ||
+ policy.retryAfterSeconds <= 0 ||
+ policy.retryAfterSeconds > 3600
+ ) {
+ throw new TypeError('retryAfterSeconds must be an integer from 1 to 3600')
+ }
this.policy = Object.freeze({
maxInFlight: policy.maxInFlight,
maxQueued: policy.maxQueued,
@@ -83,6 +93,7 @@ export class AuthorizeBodyAdmission {
readDeadlineMs: policy.readDeadlineMs,
workDeadlineMs: policy.workDeadlineMs,
closeGraceMs: policy.closeGraceMs,
+ retryAfterSeconds: policy.retryAfterSeconds,
})
}
@@ -311,6 +322,9 @@ export class AuthorizeBodyAdmission {
res.once('finish', stopRejectedBody)
res.once('close', stopRejectedBody)
res.setHeader('Connection', 'close')
+ // Nothing was read or recorded, so the Host may retry the same attempt
+ // once after this delay (same provider, no failover).
+ res.setHeader('Retry-After', String(this.policy.retryAfterSeconds))
requestLog(req).warn(
{
event: 'llm_provider_attempt_admission_refused',
diff --git a/control-api/test/llmProviderAttemptBodyAdmission.test.ts b/control-api/test/llmProviderAttemptBodyAdmission.test.ts
index 5057250ea..89293e49c 100644
--- a/control-api/test/llmProviderAttemptBodyAdmission.test.ts
+++ b/control-api/test/llmProviderAttemptBodyAdmission.test.ts
@@ -152,6 +152,7 @@ describe('AuthorizeBodyAdmission policy and idempotence', () => {
readDeadlineMs: 20,
workDeadlineMs: 20,
closeGraceMs: 5,
+ retryAfterSeconds: 1,
}
const gate = new AuthorizeBodyAdmission(valid)
expect(() => {
@@ -183,6 +184,13 @@ describe('AuthorizeBodyAdmission policy and idempotence', () => {
TypeError
)
}
+ // The Host only honours a Retry-After of 1-3600 whole seconds.
+ expect(() => new AuthorizeBodyAdmission({ ...valid, retryAfterSeconds: 3600 })).not.toThrow()
+ for (const value of [0, 1.5, 3601]) {
+ expect(() => new AuthorizeBodyAdmission({ ...valid, retryAfterSeconds: value })).toThrow(
+ TypeError
+ )
+ }
})
it('an old release cannot free the slot occupied by a newer request', () => {
@@ -194,6 +202,7 @@ describe('AuthorizeBodyAdmission policy and idempotence', () => {
readDeadlineMs: 100,
workDeadlineMs: 100,
closeGraceMs: 10,
+ retryAfterSeconds: 1,
})
const releaseA = gate.tryAcquire(PRINCIPAL)
expect(releaseA).not.toBeNull()
@@ -218,6 +227,7 @@ describe('AuthorizeBodyAdmission policy and idempotence', () => {
readDeadlineMs: 100,
workDeadlineMs: 100,
closeGraceMs: 10,
+ retryAfterSeconds: 1,
})
const req = fakeRequest()
const res = fakeResponse()
@@ -248,6 +258,7 @@ describe('AuthorizeBodyAdmission FIFO and verified-principal share', () => {
readDeadlineMs: 1000,
workDeadlineMs: 1000,
closeGraceMs: 5,
+ retryAfterSeconds: 1,
...overrides,
})
}
@@ -334,6 +345,7 @@ describe('AuthorizeBodyAdmission FIFO and verified-principal share', () => {
}
)
expect(fullRes.body).toEqual({ error: 'authorize_capacity_exceeded' })
+ expect(fullRes.setHeader).toHaveBeenCalledWith('Retry-After', '1')
expect(fullParser).not.toHaveBeenCalled()
await refused
fullRes.emit('finish')
@@ -385,6 +397,8 @@ describe('AuthorizeBodyAdmission FIFO and verified-principal share', () => {
{ sub: 'same-sub', hostRefs: ['host-b', 'host-a'] }
)
expect(rejectedRes.body).toEqual({ error: 'authorize_capacity_exceeded' })
+ // principal_share invites the Host's single retry like every capacity refusal.
+ expect(rejectedRes.setHeader).toHaveBeenCalledWith('Retry-After', '1')
expect(rejectedParser).not.toHaveBeenCalled()
await rejected
rejectedRes.emit('finish')
@@ -834,6 +848,7 @@ describe('AuthorizeBodyAdmission parser lifecycle', () => {
readDeadlineMs: 100,
workDeadlineMs: 100,
closeGraceMs: 10,
+ retryAfterSeconds: 1,
})
const req = fakeRequest()
const res = fakeResponse()
@@ -865,6 +880,7 @@ describe('AuthorizeBodyAdmission parser lifecycle', () => {
readDeadlineMs: 15,
workDeadlineMs: 100,
closeGraceMs: 5,
+ retryAfterSeconds: 1,
})
const req = fakeRequest()
const requestLogger = fakeRequestLogger()
@@ -907,6 +923,7 @@ describe('AuthorizeBodyAdmission parser lifecycle', () => {
readDeadlineMs: 100,
workDeadlineMs: 100,
closeGraceMs: 5,
+ retryAfterSeconds: 1,
})
const holder = gate.tryAcquire(PRINCIPAL)
expect(holder).not.toBeNull()
@@ -957,6 +974,7 @@ describe('AuthorizeBodyAdmission parser lifecycle', () => {
readDeadlineMs: 100,
workDeadlineMs: 100,
closeGraceMs: 10,
+ retryAfterSeconds: 1,
})
const req = fakeRequest()
const res = fakeResponse()
@@ -986,6 +1004,7 @@ describe('AuthorizeBodyAdmission work cancellation', () => {
readDeadlineMs: 100,
workDeadlineMs: 15,
closeGraceMs: 10,
+ retryAfterSeconds: 1,
})
const req = fakeRequest()
const res = fakeResponse()
@@ -1016,6 +1035,7 @@ describe('AuthorizeBodyAdmission work cancellation', () => {
readDeadlineMs: 1000,
workDeadlineMs: 5000,
closeGraceMs: 10,
+ retryAfterSeconds: 1,
})
const req = fakeRequest()
const res = fakeResponse()
@@ -1075,6 +1095,7 @@ describe('AuthorizeBodyAdmission HTTP fixture', () => {
readDeadlineMs: 1000,
workDeadlineMs: 1000,
closeGraceMs: 10,
+ retryAfterSeconds: 1,
})
const parser = express.json({ limit: '1kb', inflate: false })
const work = vi.fn(async (req: express.Request, res: Response, _signal: AbortSignal) => {
@@ -1112,6 +1133,7 @@ describe('AuthorizeBodyAdmission HTTP fixture', () => {
readDeadlineMs: 1000,
workDeadlineMs: 5000,
closeGraceMs: 10,
+ retryAfterSeconds: 1,
})
const parser = express.json({ limit: '1kb', inflate: false })
const workStarted = deferred()
@@ -1175,6 +1197,7 @@ describe('AuthorizeBodyAdmission HTTP fixture', () => {
expect(refused.status).toBe(503)
expect(JSON.parse(refused.body)).toEqual({ error: 'authorize_capacity_exceeded' })
expect(refused.headers.toLowerCase()).toContain('connection: close')
+ expect(refused.headers.toLowerCase()).toMatch(/\r\nretry-after: 1(\r\n|$)/)
expect(work).toHaveBeenCalledOnce()
first.destroy()
@@ -1214,6 +1237,7 @@ describe('AuthorizeBodyAdmission HTTP fixture', () => {
readDeadlineMs: 5000,
workDeadlineMs: 1000,
closeGraceMs: 10,
+ retryAfterSeconds: 1,
})
const parser = express.json({ limit: '1kb', inflate: false })
const parserStarted = deferred()
diff --git a/control-api/test/routes.mcp-host.llmProviderAttempts.bodyAdmission.test.ts b/control-api/test/routes.mcp-host.llmProviderAttempts.bodyAdmission.test.ts
index 00b678e9a..3d21f5dbe 100644
--- a/control-api/test/routes.mcp-host.llmProviderAttempts.bodyAdmission.test.ts
+++ b/control-api/test/routes.mcp-host.llmProviderAttempts.bodyAdmission.test.ts
@@ -274,6 +274,9 @@ describe('createApp retained authorize-body ownership', () => {
expect(blocked.status).toBe(503)
expect(JSON.parse(blocked.body)).toEqual({ error: 'authorize_capacity_exceeded' })
expect(blocked.headers.connection).toBe('close')
+ expect(blocked.headers['retry-after']).toBe(
+ String(LLM_PROVIDER_ATTEMPT_ADMISSION_POLICY.retryAfterSeconds)
+ )
expect(otherApp.observations[0]).toMatchObject({
bodyDataSubscriptions: 0,
readBytesBeforeResponse: 0,
@@ -602,6 +605,7 @@ describe('createApp retained authorize-body ownership', () => {
expect(expired.status).toBe(503)
expect(JSON.parse(expired.body)).toEqual({ error: 'authorize_capacity_exceeded' })
expect(expired.headers.connection).toBe('close')
+ expect(expired.headers['retry-after']).toBe('10')
expect(app.observations[1]).toMatchObject({
bodyDataSubscriptions: 0,
readBytesBeforeResponse: 0,
@@ -863,6 +867,8 @@ describe('createApp retained authorize-body ownership', () => {
const response = await outcome
expect(response.status).toBe(503)
expect(JSON.parse(response.body)).toEqual({ error: 'authorize_timeout' })
+ // Work may have run: only a capacity refusal invites a retry.
+ expect(response.headers['retry-after']).toBeUndefined()
expect((app.observations[0].request as Request).body).toBeUndefined()
const recovered = await send(app.url, headers('after-timeout-host'), '{}').response
expect(recovered.status).toBe(200)
diff --git a/docs/architecture/codex-subscription-transport-contract.md b/docs/architecture/codex-subscription-transport-contract.md
index 5648a55f8..0d19c1f48 100644
--- a/docs/architecture/codex-subscription-transport-contract.md
+++ b/docs/architecture/codex-subscription-transport-contract.md
@@ -123,11 +123,17 @@ control-api's authorize route has two local outcomes of its own, also
closed wire identities read only from the `error` field: 503
`authorize_capacity_exceeded` (the retained-body admission is full, or a
queued request waited past its bound) and 503 `authorize_timeout` (authorize
-work outlived its 30 s deadline). Both classify as non-retryable
-`LLM_API_CALL_FAILED`, do not fail over or install a cooldown, and send
-nothing to the provider. A capacity refusal is answered before the body is
-read, with `Connection: close`, so no attempt is recorded. Only the retained
-path takes a unit. A request whose declared `Content-Length` is at or below
+work outlived its 30 s deadline). Neither sends anything to the provider,
+fails over or installs a cooldown. A capacity refusal is answered before the
+body is read, with `Connection: close` and `Retry-After: 10`, so no attempt is
+recorded. The Host retries it once in the same provider after that delay, with
+the next attempt index, and re-uploads the body. A second refusal, or a
+refusal without `Retry-After` (an older control-api), is terminal: a
+non-retryable `LLM_API_CALL_FAILED`. A caller-pinned attempt index is never
+retried. The retry adds at most 50 s (a 40 s queue refusal plus the 10 s wait)
+before the second attempt's own 80 s. `authorize_timeout` carries no
+`Retry-After` and is terminal at once. Only the retained path takes a unit. A
+request whose declared `Content-Length` is at or below
the text authorize envelope (`maxRequestBodyBytes` +
`ENVELOPE_ALLOWANCE_BYTES`, 8404992 bytes, the larger of the two contracts)
is parsed with that limit and never queues. A larger declared length, a
diff --git a/docs/architecture/grok-subscription-transport-contract.md b/docs/architecture/grok-subscription-transport-contract.md
index 3916cafe8..9182948a6 100644
--- a/docs/architecture/grok-subscription-transport-contract.md
+++ b/docs/architecture/grok-subscription-transport-contract.md
@@ -247,11 +247,17 @@ Errors:
closed wire identities read only from the `error` field: 503
`authorize_capacity_exceeded` (the retained-body admission is full, or a
queued request waited past its bound) and 503 `authorize_timeout` (authorize
- work outlived its 30 s deadline). Both classify as non-retryable
- `LLM_API_CALL_FAILED`, do not fail over or install a cooldown, and send
- nothing to the provider. A capacity refusal is answered before the body is
- read, with `Connection: close`, so no attempt is recorded. Only the retained
- path takes a unit. A request whose declared `Content-Length` is at or below
+ work outlived its 30 s deadline). Neither sends anything to the provider,
+ fails over or installs a cooldown. A capacity refusal is answered before the
+ body is read, with `Connection: close` and `Retry-After: 10`, so no attempt
+ is recorded. The Host retries it once in the same provider after that delay,
+ with the next attempt index, and re-uploads the body. A second refusal, or a
+ refusal without `Retry-After` (an older control-api), is terminal: a
+ non-retryable `LLM_API_CALL_FAILED`. A caller-pinned attempt index is never
+ retried. The retry adds at most 50 s (a 40 s queue refusal plus the 10 s
+ wait) before the second attempt's own 80 s. `authorize_timeout` carries no
+ `Retry-After` and is terminal at once. Only the retained path takes a unit. A
+ request whose declared `Content-Length` is at or below
the text authorize envelope (`maxRequestBodyBytes` +
`ENVELOPE_ALLOWANCE_BYTES`, 8404992 bytes, the larger of the two contracts)
is parsed with that limit and never queues. A larger declared length, a
diff --git a/mcp-host/package-lock.json b/mcp-host/package-lock.json
index 1a3db01f1..5084bf3fe 100644
--- a/mcp-host/package-lock.json
+++ b/mcp-host/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "clerum-mcp-host",
- "version": "0.1.490",
+ "version": "0.1.491",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "clerum-mcp-host",
- "version": "0.1.490",
+ "version": "0.1.491",
"license": "SEE LICENSE IN LICENSE",
"dependencies": {
"@anthropic-ai/sdk": "^0.32.0",
diff --git a/mcp-host/package.json b/mcp-host/package.json
index 5f1ace456..b4e1ff97d 100644
--- a/mcp-host/package.json
+++ b/mcp-host/package.json
@@ -1,6 +1,6 @@
{
"name": "clerum-mcp-host",
- "version": "0.1.490",
+ "version": "0.1.491",
"description": "MCP Host - reads Host CRD and provides LLM access via OpenAI/Claude",
"main": "dist/main.js",
"engines": {
diff --git a/mcp-host/src/llm/__tests__/codexSubscription.test.ts b/mcp-host/src/llm/__tests__/codexSubscription.test.ts
index 0bfb929ea..9dcc40d9d 100644
--- a/mcp-host/src/llm/__tests__/codexSubscription.test.ts
+++ b/mcp-host/src/llm/__tests__/codexSubscription.test.ts
@@ -2012,6 +2012,174 @@ describe('CodexSubscriptionProvider Retry-After retry (G1-9, #720)', () => {
expect(wired.stream).toHaveBeenCalledTimes(0)
})
+ // M-A (review 5426789128): control-api's capacity refusal is answered
+ // before any read and carries Retry-After. The provider retries it once in
+ // place; the second refusal is terminal, with no failover or cooldown.
+ const capacityRefusal = (seconds?: number) =>
+ Response.json(
+ { error: 'authorize_capacity_exceeded' },
+ seconds === undefined
+ ? { status: 503 }
+ : { status: 503, headers: { 'retry-after': String(seconds) } }
+ )
+
+ it('M-A1: a capacity refusal on authorize is retried once after its Retry-After', async () => {
+ vi.useFakeTimers()
+ const fetchFn = vi
+ .fn()
+ .mockResolvedValueOnce(capacityRefusal(10))
+ .mockResolvedValueOnce(authorizedSecond())
+ const wired = deps({ authorize: realAuthorize(fetchFn), stream: vi.fn().mockResolvedValue(ok) })
+ const provider = new CodexSubscriptionProvider('gpt-5.3-codex', wired as never)
+
+ const turn = provider.completeSingleTurn([{ role: 'user', content: 'hi' }])
+ await vi.advanceTimersByTimeAsync(9999)
+ // Witness: the first authorize ran; nothing was dispatched on its refusal.
+ expect(fetchFn).toHaveBeenCalledTimes(1)
+ expect(wired.proxy.stream).toHaveBeenCalledTimes(0)
+ await vi.advanceTimersByTimeAsync(1)
+ await expect(turn).resolves.toMatchObject({ content: 'after the wait' })
+
+ expect(fetchFn).toHaveBeenCalledTimes(2)
+ const [first, second] = fetchFn.mock.calls.map(call => JSON.parse(call[1].body as string))
+ expect(second.invocationId).toBe(first.invocationId)
+ expect([first.providerAttemptIndex, second.providerAttemptIndex]).toEqual([1, 2])
+ expect(wired.proxy.stream).toHaveBeenCalledTimes(1)
+ expect(wired.proxy.stream.mock.calls[0][0].executionTicket).toBe('ticket-second')
+ })
+
+ it('M-A2: a second capacity refusal is terminal and never fails over', async () => {
+ vi.useFakeTimers()
+ const fetchFn = vi
+ .fn()
+ .mockResolvedValueOnce(capacityRefusal(10))
+ .mockResolvedValueOnce(capacityRefusal(10))
+ .mockResolvedValueOnce(authorizedSecond())
+ const wired = deps({ authorize: realAuthorize(fetchFn) })
+ const provider = new CodexSubscriptionProvider('gpt-5.3-codex', wired as never)
+
+ const turn = provider.completeSingleTurn([{ role: 'user', content: 'hi' }])
+ const caught = turn.then(
+ () => undefined,
+ (e: unknown) => e
+ )
+ await vi.advanceTimersByTimeAsync(60_000)
+ const err = await caught
+
+ expect(err).toBeInstanceOf(CodexAuthorizeError)
+ expect(err).toMatchObject({ code: 'authorize_capacity_exceeded', retryAfterMs: 10_000 })
+ const classified = provider.classifyError(err)
+ expect(classified).toMatchObject({
+ code: LlmErrorCode.ApiCallFailed,
+ retryable: false,
+ providerDispatched: false,
+ })
+ expect(classifyFailoverClass(classified.code, classified.retryable)).toBeNull()
+ // Exactly two authorizes: the retry ran once and the third answer was never asked for.
+ expect(fetchFn).toHaveBeenCalledTimes(2)
+ expect(wired.proxy.stream).toHaveBeenCalledTimes(0)
+ })
+
+ it('M-A3: a capacity refusal without Retry-After is not retried', async () => {
+ vi.useFakeTimers()
+ const fetchFn = vi
+ .fn()
+ .mockResolvedValueOnce(capacityRefusal())
+ .mockResolvedValueOnce(authorizedSecond())
+ const wired = deps({ authorize: realAuthorize(fetchFn) })
+ const provider = new CodexSubscriptionProvider('gpt-5.3-codex', wired as never)
+
+ const turn = provider.completeSingleTurn([{ role: 'user', content: 'hi' }])
+ const caught = turn.then(
+ () => undefined,
+ (e: unknown) => e
+ )
+ await vi.advanceTimersByTimeAsync(60_000)
+ const err = await caught
+
+ expect(err).toBeInstanceOf(CodexAuthorizeError)
+ expect(err).toMatchObject({ code: 'authorize_capacity_exceeded' })
+ expect((err as CodexAuthorizeError).retryAfterMs).toBeUndefined()
+ // Witness: the first authorize ran; the negative is that no second one did.
+ expect(fetchFn).toHaveBeenCalledTimes(1)
+ expect(wired.proxy.stream).toHaveBeenCalledTimes(0)
+ })
+
+ it('M-A4: a capacity refusal on a caller-pinned attempt index is not retried', async () => {
+ vi.useFakeTimers()
+ const fetchFn = vi
+ .fn()
+ .mockResolvedValueOnce(capacityRefusal(10))
+ .mockResolvedValueOnce(authorizedSecond())
+ const wired = deps({ authorize: realAuthorize(fetchFn) })
+ wired.attemptContext = vi.fn(() => ({
+ policyRevision: 1,
+ policyHash: 'b'.repeat(64),
+ hostRef: 'chatllm',
+ providerAttemptIndex: 3,
+ }))
+ const provider = new CodexSubscriptionProvider('gpt-5.3-codex', wired as never)
+
+ const turn = provider.completeSingleTurn([{ role: 'user', content: 'hi' }])
+ const caught = turn.then(
+ () => undefined,
+ (e: unknown) => e
+ )
+ await vi.advanceTimersByTimeAsync(60_000)
+ const err = await caught
+
+ expect(err).toMatchObject({ code: 'authorize_capacity_exceeded', retryAfterMs: 10_000 })
+ // Witness: the pinned authorize ran; the negative is that no second one did.
+ expect(fetchFn).toHaveBeenCalledTimes(1)
+ expect(JSON.parse(fetchFn.mock.calls[0][1].body as string).providerAttemptIndex).toBe(3)
+ expect(wired.proxy.stream).toHaveBeenCalledTimes(0)
+ })
+
+ it('M-A5: a proxy error named authorize_capacity_exceeded is not retried', async () => {
+ vi.useFakeTimers()
+ const err = new CodexProxyError('authorize_capacity_exceeded', 'proxy refused', {
+ retryAfterMs: 1000,
+ })
+ const wired = deps({
+ authorize: authorizeTwice(),
+ stream: vi.fn().mockRejectedValueOnce(err).mockResolvedValueOnce(ok),
+ })
+ const provider = new CodexSubscriptionProvider('gpt-5.3-codex', wired as never)
+
+ const turn = provider.completeSingleTurn([{ role: 'user', content: 'hi' }])
+ const settled = expect(turn).rejects.toBe(err)
+ await vi.advanceTimersByTimeAsync(60_000)
+ await settled
+ // Witness: the attempt was dispatched once; the negative is that no retry followed.
+ expect(wired.authorizer.authorize).toHaveBeenCalledTimes(1)
+ expect(wired.proxy.stream).toHaveBeenCalledTimes(1)
+ })
+
+ it('M-A6: an abort during the capacity wait rejects with the abort reason and sends nothing more', async () => {
+ vi.useFakeTimers()
+ const controller = new AbortController()
+ const reason = new Error('caller gave up')
+ const fetchFn = vi
+ .fn()
+ .mockResolvedValueOnce(capacityRefusal(10))
+ .mockResolvedValueOnce(authorizedSecond())
+ const wired = deps({ authorize: realAuthorize(fetchFn), stream: vi.fn().mockResolvedValue(ok) })
+ const provider = new CodexSubscriptionProvider('gpt-5.3-codex', wired as never)
+
+ const turn = provider.completeSingleTurn([{ role: 'user', content: 'hi' }], {
+ signal: controller.signal,
+ })
+ const settled = expect(turn).rejects.toBe(reason)
+ await vi.advanceTimersByTimeAsync(5000)
+ // Witness: the first authorize ran and the wait is pending.
+ expect(fetchFn).toHaveBeenCalledTimes(1)
+ controller.abort(reason)
+ await settled
+ await vi.advanceTimersByTimeAsync(60_000)
+ expect(fetchFn).toHaveBeenCalledTimes(1)
+ expect(wired.proxy.stream).toHaveBeenCalledTimes(0)
+ })
+
it('G1-12: an abort after the wait resolves and before the second authorize sends nothing more', async () => {
vi.useFakeTimers()
const controller = new AbortController()
diff --git a/mcp-host/src/llm/__tests__/grokSubscription.test.ts b/mcp-host/src/llm/__tests__/grokSubscription.test.ts
index 14d9581ec..0758337da 100644
--- a/mcp-host/src/llm/__tests__/grokSubscription.test.ts
+++ b/mcp-host/src/llm/__tests__/grokSubscription.test.ts
@@ -1888,6 +1888,174 @@ describe('GrokSubscriptionProvider Retry-After retry (G1-9, #720)', () => {
expect(wired.proxy.stream).toHaveBeenCalledTimes(0)
})
+ // M-A (review 5426789128): control-api's capacity refusal is answered
+ // before any read and carries Retry-After. The provider retries it once in
+ // place; the second refusal is terminal, with no failover or cooldown.
+ const capacityRefusal = (seconds?: number) =>
+ Response.json(
+ { error: 'authorize_capacity_exceeded' },
+ seconds === undefined
+ ? { status: 503 }
+ : { status: 503, headers: { 'retry-after': String(seconds) } }
+ )
+
+ it('M-A1: a capacity refusal on authorize is retried once after its Retry-After', async () => {
+ vi.useFakeTimers()
+ const fetchFn = vi
+ .fn()
+ .mockResolvedValueOnce(capacityRefusal(10))
+ .mockResolvedValueOnce(authorizedSecond())
+ const wired = deps({ authorize: realAuthorize(fetchFn), stream: vi.fn().mockResolvedValue(ok) })
+ const provider = new GrokSubscriptionProvider('grok-4.6', wired as never)
+
+ const turn = provider.completeSingleTurn([{ role: 'user', content: 'hi' }])
+ await vi.advanceTimersByTimeAsync(9999)
+ // Witness: the first authorize ran; nothing was dispatched on its refusal.
+ expect(fetchFn).toHaveBeenCalledTimes(1)
+ expect(wired.proxy.stream).toHaveBeenCalledTimes(0)
+ await vi.advanceTimersByTimeAsync(1)
+ await expect(turn).resolves.toMatchObject({ content: 'after the wait' })
+
+ expect(fetchFn).toHaveBeenCalledTimes(2)
+ const [first, second] = fetchFn.mock.calls.map(call => JSON.parse(call[1].body as string))
+ expect(second.invocationId).toBe(first.invocationId)
+ expect([first.providerAttemptIndex, second.providerAttemptIndex]).toEqual([1, 2])
+ expect(wired.proxy.stream).toHaveBeenCalledTimes(1)
+ expect(wired.proxy.stream.mock.calls[0][0].executionTicket).toBe('ticket-second')
+ })
+
+ it('M-A2: a second capacity refusal is terminal and never fails over', async () => {
+ vi.useFakeTimers()
+ const fetchFn = vi
+ .fn()
+ .mockResolvedValueOnce(capacityRefusal(10))
+ .mockResolvedValueOnce(capacityRefusal(10))
+ .mockResolvedValueOnce(authorizedSecond())
+ const wired = deps({ authorize: realAuthorize(fetchFn) })
+ const provider = new GrokSubscriptionProvider('grok-4.6', wired as never)
+
+ const turn = provider.completeSingleTurn([{ role: 'user', content: 'hi' }])
+ const caught = turn.then(
+ () => undefined,
+ (e: unknown) => e
+ )
+ await vi.advanceTimersByTimeAsync(60_000)
+ const err = await caught
+
+ expect(err).toBeInstanceOf(CodexAuthorizeError)
+ expect(err).toMatchObject({ code: 'authorize_capacity_exceeded', retryAfterMs: 10_000 })
+ const classified = provider.classifyError(err)
+ expect(classified).toMatchObject({
+ code: LlmErrorCode.ApiCallFailed,
+ retryable: false,
+ providerDispatched: false,
+ })
+ expect(classifyFailoverClass(classified.code, classified.retryable)).toBeNull()
+ // Exactly two authorizes: the retry ran once and the third answer was never asked for.
+ expect(fetchFn).toHaveBeenCalledTimes(2)
+ expect(wired.proxy.stream).toHaveBeenCalledTimes(0)
+ })
+
+ it('M-A3: a capacity refusal without Retry-After is not retried', async () => {
+ vi.useFakeTimers()
+ const fetchFn = vi
+ .fn()
+ .mockResolvedValueOnce(capacityRefusal())
+ .mockResolvedValueOnce(authorizedSecond())
+ const wired = deps({ authorize: realAuthorize(fetchFn) })
+ const provider = new GrokSubscriptionProvider('grok-4.6', wired as never)
+
+ const turn = provider.completeSingleTurn([{ role: 'user', content: 'hi' }])
+ const caught = turn.then(
+ () => undefined,
+ (e: unknown) => e
+ )
+ await vi.advanceTimersByTimeAsync(60_000)
+ const err = await caught
+
+ expect(err).toBeInstanceOf(CodexAuthorizeError)
+ expect(err).toMatchObject({ code: 'authorize_capacity_exceeded' })
+ expect((err as CodexAuthorizeError).retryAfterMs).toBeUndefined()
+ // Witness: the first authorize ran; the negative is that no second one did.
+ expect(fetchFn).toHaveBeenCalledTimes(1)
+ expect(wired.proxy.stream).toHaveBeenCalledTimes(0)
+ })
+
+ it('M-A4: a capacity refusal on a caller-pinned attempt index is not retried', async () => {
+ vi.useFakeTimers()
+ const fetchFn = vi
+ .fn()
+ .mockResolvedValueOnce(capacityRefusal(10))
+ .mockResolvedValueOnce(authorizedSecond())
+ const wired = deps({ authorize: realAuthorize(fetchFn) })
+ wired.attemptContext = vi.fn(() => ({
+ policyRevision: 1,
+ policyHash: 'b'.repeat(64),
+ hostRef: 'chatllm',
+ providerAttemptIndex: 3,
+ }))
+ const provider = new GrokSubscriptionProvider('grok-4.6', wired as never)
+
+ const turn = provider.completeSingleTurn([{ role: 'user', content: 'hi' }])
+ const caught = turn.then(
+ () => undefined,
+ (e: unknown) => e
+ )
+ await vi.advanceTimersByTimeAsync(60_000)
+ const err = await caught
+
+ expect(err).toMatchObject({ code: 'authorize_capacity_exceeded', retryAfterMs: 10_000 })
+ // Witness: the pinned authorize ran; the negative is that no second one did.
+ expect(fetchFn).toHaveBeenCalledTimes(1)
+ expect(JSON.parse(fetchFn.mock.calls[0][1].body as string).providerAttemptIndex).toBe(3)
+ expect(wired.proxy.stream).toHaveBeenCalledTimes(0)
+ })
+
+ it('M-A5: a proxy error named authorize_capacity_exceeded is not retried', async () => {
+ vi.useFakeTimers()
+ const err = new GrokProxyError('authorize_capacity_exceeded', 'proxy refused', {
+ retryAfterMs: 1000,
+ })
+ const wired = deps({
+ authorize: authorizeTwice(),
+ stream: vi.fn().mockRejectedValueOnce(err).mockResolvedValueOnce(ok),
+ })
+ const provider = new GrokSubscriptionProvider('grok-4.6', wired as never)
+
+ const turn = provider.completeSingleTurn([{ role: 'user', content: 'hi' }])
+ const settled = expect(turn).rejects.toBe(err)
+ await vi.advanceTimersByTimeAsync(60_000)
+ await settled
+ // Witness: the attempt was dispatched once; the negative is that no retry followed.
+ expect(wired.authorizer.authorize).toHaveBeenCalledTimes(1)
+ expect(wired.proxy.stream).toHaveBeenCalledTimes(1)
+ })
+
+ it('M-A6: an abort during the capacity wait rejects with the abort reason and sends nothing more', async () => {
+ vi.useFakeTimers()
+ const controller = new AbortController()
+ const reason = new Error('caller gave up')
+ const fetchFn = vi
+ .fn()
+ .mockResolvedValueOnce(capacityRefusal(10))
+ .mockResolvedValueOnce(authorizedSecond())
+ const wired = deps({ authorize: realAuthorize(fetchFn), stream: vi.fn().mockResolvedValue(ok) })
+ const provider = new GrokSubscriptionProvider('grok-4.6', wired as never)
+
+ const turn = provider.completeSingleTurn([{ role: 'user', content: 'hi' }], {
+ signal: controller.signal,
+ })
+ const settled = expect(turn).rejects.toBe(reason)
+ await vi.advanceTimersByTimeAsync(5000)
+ // Witness: the first authorize ran and the wait is pending.
+ expect(fetchFn).toHaveBeenCalledTimes(1)
+ controller.abort(reason)
+ await settled
+ await vi.advanceTimersByTimeAsync(60_000)
+ expect(fetchFn).toHaveBeenCalledTimes(1)
+ expect(wired.proxy.stream).toHaveBeenCalledTimes(0)
+ })
+
it('G1-12: an abort after the wait resolves and before the second authorize sends nothing more', async () => {
vi.useFakeTimers()
const controller = new AbortController()
diff --git a/mcp-host/src/llm/__tests__/providerAttemptAuthorizer.test.ts b/mcp-host/src/llm/__tests__/providerAttemptAuthorizer.test.ts
index 053282149..4de97005f 100644
--- a/mcp-host/src/llm/__tests__/providerAttemptAuthorizer.test.ts
+++ b/mcp-host/src/llm/__tests__/providerAttemptAuthorizer.test.ts
@@ -619,6 +619,53 @@ describe('ProviderAttemptAuthorizer', () => {
}
)
+ // M-A (review 5426789128): control-api's capacity refusal carries
+ // Retry-After, and only that coded 503 has it read.
+ it.each([
+ [
+ 'a capacity refusal with Retry-After',
+ () =>
+ Response.json(
+ { error: 'authorize_capacity_exceeded' },
+ { status: 503, headers: { 'retry-after': '10' } }
+ ),
+ 'authorize_capacity_exceeded',
+ 10_000,
+ ],
+ [
+ 'a capacity refusal without Retry-After',
+ () => Response.json({ error: 'authorize_capacity_exceeded' }, { status: 503 }),
+ 'authorize_capacity_exceeded',
+ undefined,
+ ],
+ [
+ 'an authorize_timeout with Retry-After',
+ () =>
+ Response.json(
+ { error: 'authorize_timeout' },
+ { status: 503, headers: { 'retry-after': '10' } }
+ ),
+ 'authorize_timeout',
+ undefined,
+ ],
+ [
+ 'an uncoded 503 with Retry-After',
+ () =>
+ new Response('<html>503 Service Temporarily Unavailable</html>', {
+ status: 503,
+ headers: { 'content-type': 'text/html', 'retry-after': '10' },
+ }),
+ 'provider_unavailable',
+ undefined,
+ ],
+ ] as const)('M-A reads Retry-After for %s', async (_label, response, code, retryAfter) => {
+ const { err, fetchFn } = await authorizeFailure(response())
+ expect(fetchFn).toHaveBeenCalledTimes(1)
+ expect(err).toBeInstanceOf(CodexAuthorizeError)
+ expect(err).toMatchObject({ code })
+ expect((err as CodexAuthorizeError).retryAfterMs).toBe(retryAfter)
+ })
+
// G1-6 (#720): a limiter in front of control-api can answer 429 with no JSON
// code. It is a rate limit, not a provider outage.
it('G1-6d reads an HTML 429 from the gateway as rate_limited', async () => {
diff --git a/mcp-host/src/llm/codexSubscription.ts b/mcp-host/src/llm/codexSubscription.ts
index 788c6611e..1f47f724b 100644
--- a/mcp-host/src/llm/codexSubscription.ts
+++ b/mcp-host/src/llm/codexSubscription.ts
@@ -22,7 +22,7 @@ import { CodexLlmProxyClient, CodexProxyError } from './codexLlmProxyClient'
import { classifyUnknown } from './errorClassification'
import { IMAGE_SOURCE_INVALID, canonicalRefusalCode, projectMessage } from './imageSource'
import { CodexAuthorizeError, ProviderAttemptAuthorizer } from './providerAttemptAuthorizer'
-import { rateLimitRetryDelayMs, waitBeforeRetry } from './rateLimitRetry'
+import { authorizeRetryDelayMs, rateLimitRetryDelayMs, waitBeforeRetry } from './rateLimitRetry'
import { type LlmProvider, descriptorFor } from './registryCore'
import type { ClassifiedError, SingleTurnProvider } from './types'
@@ -528,12 +528,17 @@ export class CodexSubscriptionProvider implements SingleTurnProvider {
// after that delay, under a new authorize: a redeemed ticket cannot be
// reused. The 429 may come from the proxy or from control-api's
// authorize limiter (G1-11). A caller that pins the attempt index owns
- // its own retries.
+ // its own retries. control-api's authorize_capacity_exceeded, refused
+ // before any read, is retried once the same way (M-A, review
+ // 5426789128); a proxy error of that name is not.
const waitMs =
- context.providerAttemptIndex === undefined &&
- (err instanceof CodexProxyError || err instanceof CodexAuthorizeError)
- ? rateLimitRetryDelayMs(err.code, err.retryAfterMs)
- : undefined
+ context.providerAttemptIndex !== undefined
+ ? undefined
+ : err instanceof CodexAuthorizeError
+ ? authorizeRetryDelayMs(err.code, err.retryAfterMs)
+ : err instanceof CodexProxyError
+ ? rateLimitRetryDelayMs(err.code, err.retryAfterMs)
+ : undefined
if (waitMs === undefined) throw err
await waitBeforeRetry(waitMs, options?.signal)
// The wait stops watching the signal once its timer fires: an abort in
diff --git a/mcp-host/src/llm/failover/__tests__/subscriptionAdmission.integration.test.ts b/mcp-host/src/llm/failover/__tests__/subscriptionAdmission.integration.test.ts
index ce8fd6183..01cb5adc7 100644
--- a/mcp-host/src/llm/failover/__tests__/subscriptionAdmission.integration.test.ts
+++ b/mcp-host/src/llm/failover/__tests__/subscriptionAdmission.integration.test.ts
@@ -8,6 +8,11 @@ import { CodexLlmProxyClient } from '../../codexLlmProxyClient'
import { CodexSubscriptionProvider } from '../../codexSubscription'
import { GrokLlmProxyClient } from '../../grokLlmProxyClient'
import { GrokSubscriptionProvider } from '../../grokSubscription'
+import {
+ CodexAuthorizeError,
+ ProviderAttemptAuthorizer,
+ resolveCodexAuthorizeUrl,
+} from '../../providerAttemptAuthorizer'
import { type ClassifiedLike, FailoverEngine } from '../engine'
import type { FailoverTarget, ModelPair } from '../types'
@@ -354,5 +359,48 @@ describe.each(['grok', 'codex'] as const)(
expect(f.engine.servedBy()).toEqual({ ...f.primary, fallback: false })
}
)
+
+ // M-A (review 5426789128): control-api's capacity refusal with
+ // Retry-After is retried once in place; the second refusal is terminal and
+ // the turn never leaves the primary or installs a cooldown.
+ it('a repeated authorize capacity refusal retries once on primary without failover', async () => {
+ const f = await fixture(variant)
+ const authorizeFetch = vi.fn(async () =>
+ Response.json(
+ { error: 'authorize_capacity_exceeded' },
+ { status: 503, headers: { 'retry-after': '1' } }
+ )
+ )
+ const authorizer = new ProviderAttemptAuthorizer({
+ authorizeUrl: resolveCodexAuthorizeUrl('http://gateway:8092'),
+ readPlatformJwt: () => 'platform-jwt',
+ fetchFn: authorizeFetch as unknown as typeof fetch,
+ })
+ f.authorize.mockImplementation(((body: never, options: never) =>
+ authorizer.authorize(body, options)) as never)
+
+ const err = await f.run('bounded text').then(
+ () => undefined,
+ (e: unknown) => e
+ )
+
+ expect(err).toBeInstanceOf(CodexAuthorizeError)
+ expect(err).toMatchObject({ code: 'authorize_capacity_exceeded', retryAfterMs: 1000 })
+ expect(f.classified()).toMatchObject({
+ code: LlmErrorCode.ApiCallFailed,
+ retryable: false,
+ providerCode: 'authorize_capacity_exceeded',
+ providerDispatched: false,
+ })
+ // Witness: authorize ran exactly twice, the attempt and its one retry.
+ expect(f.authorize).toHaveBeenCalledTimes(2)
+ expect(authorizeFetch).toHaveBeenCalledTimes(2)
+ expect(f.hostFetch).not.toHaveBeenCalled()
+ expect(f.built.map(target => target.kind)).toEqual(['primary'])
+ expect(f.fallback).not.toHaveBeenCalled()
+ expect(f.onSwitch).not.toHaveBeenCalled()
+ expect(f.metricInc).not.toHaveBeenCalled()
+ expect(f.engine.planTargets(f.primary)[0]?.kind).toBe('primary')
+ })
}
)
diff --git a/mcp-host/src/llm/grokSubscription.ts b/mcp-host/src/llm/grokSubscription.ts
index ce09bbe59..634da4788 100644
--- a/mcp-host/src/llm/grokSubscription.ts
+++ b/mcp-host/src/llm/grokSubscription.ts
@@ -23,7 +23,7 @@ import { classifyUnknown } from './errorClassification'
import { GrokLlmProxyClient, GrokProxyError } from './grokLlmProxyClient'
import { IMAGE_SOURCE_INVALID, canonicalRefusalCode, projectMessage } from './imageSource'
import { CodexAuthorizeError, ProviderAttemptAuthorizer } from './providerAttemptAuthorizer'
-import { rateLimitRetryDelayMs, waitBeforeRetry } from './rateLimitRetry'
+import { authorizeRetryDelayMs, rateLimitRetryDelayMs, waitBeforeRetry } from './rateLimitRetry'
import { type LlmProvider, descriptorFor } from './registryCore'
import type { ClassifiedError, SingleTurnProvider } from './types'
@@ -523,12 +523,17 @@ export class GrokSubscriptionProvider implements SingleTurnProvider {
// after that delay, under a new authorize: a redeemed ticket cannot be
// reused. The 429 may come from the proxy or from control-api's
// authorize limiter (G1-11). A caller that pins the attempt index owns
- // its own retries.
+ // its own retries. control-api's authorize_capacity_exceeded, refused
+ // before any read, is retried once the same way (M-A, review
+ // 5426789128); a proxy error of that name is not.
const waitMs =
- context.providerAttemptIndex === undefined &&
- (err instanceof GrokProxyError || err instanceof CodexAuthorizeError)
- ? rateLimitRetryDelayMs(err.code, err.retryAfterMs)
- : undefined
+ context.providerAttemptIndex !== undefined
+ ? undefined
+ : err instanceof CodexAuthorizeError
+ ? authorizeRetryDelayMs(err.code, err.retryAfterMs)
+ : err instanceof GrokProxyError
+ ? rateLimitRetryDelayMs(err.code, err.retryAfterMs)
+ : undefined
if (waitMs === undefined) throw err
await waitBeforeRetry(waitMs, options?.signal)
// The wait stops watching the signal once its timer fires: an abort in
diff --git a/mcp-host/src/llm/providerAttemptAuthorizer.ts b/mcp-host/src/llm/providerAttemptAuthorizer.ts
index 9cb58b799..bf5621fe3 100644
--- a/mcp-host/src/llm/providerAttemptAuthorizer.ts
+++ b/mcp-host/src/llm/providerAttemptAuthorizer.ts
@@ -221,7 +221,15 @@ export class ProviderAttemptAuthorizer {
: code === 'authorize_timeout'
? 'Request authorization timed out. Wait for active requests to finish, then try again.'
: `authorize failed with ${response.status}`,
- { retryAfterMs: response.status === 429 ? retryAfterMs(response) : undefined }
+ // Retry-After is read on a 429 and on control-api's capacity refusal,
+ // which the provider retries once after it (M-A).
+ {
+ retryAfterMs:
+ response.status === 429 ||
+ (response.status === 503 && code === 'authorize_capacity_exceeded')
+ ? retryAfterMs(response)
+ : undefined,
+ }
)
}
for (const key of LEAK_KEYS) {
diff --git a/mcp-host/src/llm/rateLimitRetry.ts b/mcp-host/src/llm/rateLimitRetry.ts
index a9e3228a8..2797c475b 100644
--- a/mcp-host/src/llm/rateLimitRetry.ts
+++ b/mcp-host/src/llm/rateLimitRetry.ts
@@ -19,7 +19,27 @@ export function rateLimitRetryDelayMs(
code: string,
retryAfterMs: number | undefined
): number | undefined {
- if (code !== 'rate_limited' || retryAfterMs === undefined) return undefined
+ return code === 'rate_limited' ? boundedRetryDelayMs(retryAfterMs) : undefined
+}
+
+/**
+ * The wait before the single retry of an authorize error: a `rate_limited`
+ * as in {@link rateLimitRetryDelayMs}, or control-api's
+ * `authorize_capacity_exceeded` with its Retry-After, on the same bound.
+ * Nothing was read or recorded for that refusal, so the retry stays with the
+ * same provider; a second refusal is terminal and never fails over.
+ */
+export function authorizeRetryDelayMs(
+ code: string,
+ retryAfterMs: number | undefined
+): number | undefined {
+ return code === 'rate_limited' || code === 'authorize_capacity_exceeded'
+ ? boundedRetryDelayMs(retryAfterMs)
+ : undefined
+}
+
+function boundedRetryDelayMs(retryAfterMs: number | undefined): number | undefined {
+ if (retryAfterMs === undefined) return undefined
return retryAfterMs > 0 && retryAfterMs <= RATE_LIMIT_RETRY_MAX_WAIT_MS ? retryAfterMs : undefined
}