Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
fix(authorize): retry a capacity refusal once in the same provider
Alfredo Lopez62649a512b16
diff --git a/control-api/package-lock.json b/control-api/package-lock.jsonindex 17ea1b018..4c05778ab 100644--- a/control-api/package-lock.json+++ b/control-api/package-lock.json@@ -1,12 +1,12 @@{"name": "clerum-control-api",- "version": "0.1.720",+ "version": "0.1.721","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "clerum-control-api",- "version": "0.1.720",+ "version": "0.1.721","dependencies": {"@clerum/display-field": "file:../packages/display-field","@clerum/grok-provider-attempt-contract": "file:../packages/grok-provider-attempt-contract",diff --git a/control-api/package.json b/control-api/package.jsonindex a3cbb2738..1b5e6ed95 100644--- a/control-api/package.json+++ b/control-api/package.json@@ -1,6 +1,6 @@{"name": "clerum-control-api",- "version": "0.1.720",+ "version": "0.1.721","description": "Internal Clerum control plane REST API + UI for managing CRDs and secrets","main": "dist/main.js","engines": {diff --git a/control-api/src/middleware/llmProviderAttemptAdmissionLimits.ts b/control-api/src/middleware/llmProviderAttemptAdmissionLimits.tsindex 30a09fbde..d8b45dd29 100644--- a/control-api/src/middleware/llmProviderAttemptAdmissionLimits.ts+++ b/control-api/src/middleware/llmProviderAttemptAdmissionLimits.ts@@ -15,6 +15,11 @@ import type { AuthorizeBodyAdmissionPolicy } from './llmProviderAttemptBodyAdmis// share stops one principal holding all three positions. queueWaitMs is the// longest a healthy holder can keep the unit (its read deadline plus its work// deadline), so a waiter is refused only when the holder outlives both.+//+// retryAfterSeconds is the Retry-After on a capacity refusal. The Host retries+// once in the same provider after it, re-uploading the body, and the second+// refusal is terminal. It equals the read deadline, the bound on a slow+// upload, and stays below the Host's 30 s retry ceiling. Uncertified (#813).const READ_DEADLINE_MS = 10_000const WORK_DEADLINE_MS = 30_000@@ -26,4 +31,5 @@ export const LLM_PROVIDER_ATTEMPT_ADMISSION_POLICY: AuthorizeBodyAdmissionPolicyreadDeadlineMs: READ_DEADLINE_MS,workDeadlineMs: WORK_DEADLINE_MS,closeGraceMs: 250,+ retryAfterSeconds: READ_DEADLINE_MS / 1000,})diff --git a/control-api/src/middleware/llmProviderAttemptBodyAdmission.ts b/control-api/src/middleware/llmProviderAttemptBodyAdmission.tsindex ab17e454e..fa6c79213 100644--- a/control-api/src/middleware/llmProviderAttemptBodyAdmission.ts+++ b/control-api/src/middleware/llmProviderAttemptBodyAdmission.ts@@ -10,6 +10,8 @@ export type AuthorizeBodyAdmissionPolicy = Readonly<{readDeadlineMs: numberworkDeadlineMs: numbercloseGraceMs: number+ /** `Retry-After` on a capacity refusal; the Host retries once after it. */+ retryAfterSeconds: number}>// The route supplies these claims only after requireMcpHostJwt succeeds.@@ -75,6 +77,14 @@ export class AuthorizeBodyAdmission {throw new TypeError('authorize timers must be positive signed-32-bit integers')}}+ // The Host honours 1-3600 s (mcp-host `retryAfter.ts`).+ if (+ !Number.isSafeInteger(policy.retryAfterSeconds) ||+ policy.retryAfterSeconds <= 0 ||+ policy.retryAfterSeconds > 3600+ ) {+ throw new TypeError('retryAfterSeconds must be an integer from 1 to 3600')+ }this.policy = Object.freeze({maxInFlight: policy.maxInFlight,maxQueued: policy.maxQueued,@@ -83,6 +93,7 @@ export class AuthorizeBodyAdmission {readDeadlineMs: policy.readDeadlineMs,workDeadlineMs: policy.workDeadlineMs,closeGraceMs: policy.closeGraceMs,+ retryAfterSeconds: policy.retryAfterSeconds,})}@@ -311,6 +322,9 @@ export class AuthorizeBodyAdmission {res.once('finish', stopRejectedBody)res.once('close', stopRejectedBody)res.setHeader('Connection', 'close')+ // Nothing was read or recorded, so the Host may retry the same attempt+ // once after this delay (same provider, no failover).+ res.setHeader('Retry-After', String(this.policy.retryAfterSeconds))requestLog(req).warn({event: 'llm_provider_attempt_admission_refused',diff --git a/control-api/test/llmProviderAttemptBodyAdmission.test.ts b/control-api/test/llmProviderAttemptBodyAdmission.test.tsindex 5057250ea..89293e49c 100644--- a/control-api/test/llmProviderAttemptBodyAdmission.test.ts+++ b/control-api/test/llmProviderAttemptBodyAdmission.test.ts@@ -152,6 +152,7 @@ describe('AuthorizeBodyAdmission policy and idempotence', () => {readDeadlineMs: 20,workDeadlineMs: 20,closeGraceMs: 5,+ retryAfterSeconds: 1,}const gate = new AuthorizeBodyAdmission(valid)expect(() => {@@ -183,6 +184,13 @@ describe('AuthorizeBodyAdmission policy and idempotence', () => {TypeError)}+ // The Host only honours a Retry-After of 1-3600 whole seconds.+ expect(() => new AuthorizeBodyAdmission({ ...valid, retryAfterSeconds: 3600 })).not.toThrow()+ for (const value of [0, 1.5, 3601]) {+ expect(() => new AuthorizeBodyAdmission({ ...valid, retryAfterSeconds: value })).toThrow(+ TypeError+ )+ }})it('an old release cannot free the slot occupied by a newer request', () => {@@ -194,6 +202,7 @@ describe('AuthorizeBodyAdmission policy and idempotence', () => {readDeadlineMs: 100,workDeadlineMs: 100,closeGraceMs: 10,+ retryAfterSeconds: 1,})const releaseA = gate.tryAcquire(PRINCIPAL)expect(releaseA).not.toBeNull()@@ -218,6 +227,7 @@ describe('AuthorizeBodyAdmission policy and idempotence', () => {readDeadlineMs: 100,workDeadlineMs: 100,closeGraceMs: 10,+ retryAfterSeconds: 1,})const req = fakeRequest()const res = fakeResponse()@@ -248,6 +258,7 @@ describe('AuthorizeBodyAdmission FIFO and verified-principal share', () => {readDeadlineMs: 1000,workDeadlineMs: 1000,closeGraceMs: 5,+ retryAfterSeconds: 1,...overrides,})}@@ -334,6 +345,7 @@ describe('AuthorizeBodyAdmission FIFO and verified-principal share', () => {})expect(fullRes.body).toEqual({ error: 'authorize_capacity_exceeded' })+ expect(fullRes.setHeader).toHaveBeenCalledWith('Retry-After', '1')expect(fullParser).not.toHaveBeenCalled()await refusedfullRes.emit('finish')@@ -385,6 +397,8 @@ describe('AuthorizeBodyAdmission FIFO and verified-principal share', () => {{ sub: 'same-sub', hostRefs: ['host-b', 'host-a'] })expect(rejectedRes.body).toEqual({ error: 'authorize_capacity_exceeded' })+ // principal_share invites the Host's single retry like every capacity refusal.+ expect(rejectedRes.setHeader).toHaveBeenCalledWith('Retry-After', '1')expect(rejectedParser).not.toHaveBeenCalled()await rejectedrejectedRes.emit('finish')@@ -834,6 +848,7 @@ describe('AuthorizeBodyAdmission parser lifecycle', () => {readDeadlineMs: 100,workDeadlineMs: 100,closeGraceMs: 10,+ retryAfterSeconds: 1,})const req = fakeRequest()const res = fakeResponse()@@ -865,6 +880,7 @@ describe('AuthorizeBodyAdmission parser lifecycle', () => {readDeadlineMs: 15,workDeadlineMs: 100,closeGraceMs: 5,+ retryAfterSeconds: 1,})const req = fakeRequest()const requestLogger = fakeRequestLogger()@@ -907,6 +923,7 @@ describe('AuthorizeBodyAdmission parser lifecycle', () => {readDeadlineMs: 100,workDeadlineMs: 100,closeGraceMs: 5,+ retryAfterSeconds: 1,})const holder = gate.tryAcquire(PRINCIPAL)expect(holder).not.toBeNull()@@ -957,6 +974,7 @@ describe('AuthorizeBodyAdmission parser lifecycle', () => {readDeadlineMs: 100,workDeadlineMs: 100,closeGraceMs: 10,+ retryAfterSeconds: 1,})const req = fakeRequest()const res = fakeResponse()@@ -986,6 +1004,7 @@ describe('AuthorizeBodyAdmission work cancellation', () => {readDeadlineMs: 100,workDeadlineMs: 15,closeGraceMs: 10,+ retryAfterSeconds: 1,})const req = fakeRequest()const res = fakeResponse()@@ -1016,6 +1035,7 @@ describe('AuthorizeBodyAdmission work cancellation', () => {readDeadlineMs: 1000,workDeadlineMs: 5000,closeGraceMs: 10,+ retryAfterSeconds: 1,})const req = fakeRequest()const res = fakeResponse()@@ -1075,6 +1095,7 @@ describe('AuthorizeBodyAdmission HTTP fixture', () => {readDeadlineMs: 1000,workDeadlineMs: 1000,closeGraceMs: 10,+ retryAfterSeconds: 1,})const parser = express.json({ limit: '1kb', inflate: false })const work = vi.fn(async (req: express.Request, res: Response, _signal: AbortSignal) => {@@ -1112,6 +1133,7 @@ describe('AuthorizeBodyAdmission HTTP fixture', () => {readDeadlineMs: 1000,workDeadlineMs: 5000,closeGraceMs: 10,+ retryAfterSeconds: 1,})const parser = express.json({ limit: '1kb', inflate: false })const workStarted = deferred()@@ -1175,6 +1197,7 @@ describe('AuthorizeBodyAdmission HTTP fixture', () => {expect(refused.status).toBe(503)expect(JSON.parse(refused.body)).toEqual({ error: 'authorize_capacity_exceeded' })expect(refused.headers.toLowerCase()).toContain('connection: close')+ expect(refused.headers.toLowerCase()).toMatch(/\r\nretry-after: 1(\r\n|$)/)expect(work).toHaveBeenCalledOnce()first.destroy()@@ -1214,6 +1237,7 @@ describe('AuthorizeBodyAdmission HTTP fixture', () => {readDeadlineMs: 5000,workDeadlineMs: 1000,closeGraceMs: 10,+ retryAfterSeconds: 1,})const parser = express.json({ limit: '1kb', inflate: false })const parserStarted = deferred()diff --git a/control-api/test/routes.mcp-host.llmProviderAttempts.bodyAdmission.test.ts b/control-api/test/routes.mcp-host.llmProviderAttempts.bodyAdmission.test.tsindex 00b678e9a..3d21f5dbe 100644--- a/control-api/test/routes.mcp-host.llmProviderAttempts.bodyAdmission.test.ts+++ b/control-api/test/routes.mcp-host.llmProviderAttempts.bodyAdmission.test.ts@@ -274,6 +274,9 @@ describe('createApp retained authorize-body ownership', () => {expect(blocked.status).toBe(503)expect(JSON.parse(blocked.body)).toEqual({ error: 'authorize_capacity_exceeded' })expect(blocked.headers.connection).toBe('close')+ expect(blocked.headers['retry-after']).toBe(+ String(LLM_PROVIDER_ATTEMPT_ADMISSION_POLICY.retryAfterSeconds)+ )expect(otherApp.observations[0]).toMatchObject({bodyDataSubscriptions: 0,readBytesBeforeResponse: 0,@@ -602,6 +605,7 @@ describe('createApp retained authorize-body ownership', () => {expect(expired.status).toBe(503)expect(JSON.parse(expired.body)).toEqual({ error: 'authorize_capacity_exceeded' })expect(expired.headers.connection).toBe('close')+ expect(expired.headers['retry-after']).toBe('10')expect(app.observations[1]).toMatchObject({bodyDataSubscriptions: 0,readBytesBeforeResponse: 0,@@ -863,6 +867,8 @@ describe('createApp retained authorize-body ownership', () => {const response = await outcomeexpect(response.status).toBe(503)expect(JSON.parse(response.body)).toEqual({ error: 'authorize_timeout' })+ // Work may have run: only a capacity refusal invites a retry.+ expect(response.headers['retry-after']).toBeUndefined()expect((app.observations[0].request as Request).body).toBeUndefined()const recovered = await send(app.url, headers('after-timeout-host'), '{}').responseexpect(recovered.status).toBe(200)diff --git a/docs/architecture/codex-subscription-transport-contract.md b/docs/architecture/codex-subscription-transport-contract.mdindex 5648a55f8..0d19c1f48 100644--- a/docs/architecture/codex-subscription-transport-contract.md+++ b/docs/architecture/codex-subscription-transport-contract.md@@ -123,11 +123,17 @@ control-api's authorize route has two local outcomes of its own, alsoclosed wire identities read only from the `error` field: 503`authorize_capacity_exceeded` (the retained-body admission is full, or aqueued request waited past its bound) and 503 `authorize_timeout` (authorize-work outlived its 30 s deadline). Both classify as non-retryable-`LLM_API_CALL_FAILED`, do not fail over or install a cooldown, and send-nothing to the provider. A capacity refusal is answered before the body is-read, with `Connection: close`, so no attempt is recorded. Only the retained-path takes a unit. A request whose declared `Content-Length` is at or below+work outlived its 30 s deadline). Neither sends anything to the provider,+fails over or installs a cooldown. A capacity refusal is answered before the+body is read, with `Connection: close` and `Retry-After: 10`, so no attempt is+recorded. The Host retries it once in the same provider after that delay, with+the next attempt index, and re-uploads the body. A second refusal, or a+refusal without `Retry-After` (an older control-api), is terminal: a+non-retryable `LLM_API_CALL_FAILED`. A caller-pinned attempt index is never+retried. The retry adds at most 50 s (a 40 s queue refusal plus the 10 s wait)+before the second attempt's own 80 s. `authorize_timeout` carries no+`Retry-After` and is terminal at once. Only the retained path takes a unit. A+request whose declared `Content-Length` is at or belowthe text authorize envelope (`maxRequestBodyBytes` +`ENVELOPE_ALLOWANCE_BYTES`, 8404992 bytes, the larger of the two contracts)is parsed with that limit and never queues. A larger declared length, adiff --git a/docs/architecture/grok-subscription-transport-contract.md b/docs/architecture/grok-subscription-transport-contract.mdindex 3916cafe8..9182948a6 100644--- a/docs/architecture/grok-subscription-transport-contract.md+++ b/docs/architecture/grok-subscription-transport-contract.md@@ -247,11 +247,17 @@ Errors:closed wire identities read only from the `error` field: 503`authorize_capacity_exceeded` (the retained-body admission is full, or aqueued request waited past its bound) and 503 `authorize_timeout` (authorize- work outlived its 30 s deadline). Both classify as non-retryable- `LLM_API_CALL_FAILED`, do not fail over or install a cooldown, and send- nothing to the provider. A capacity refusal is answered before the body is- read, with `Connection: close`, so no attempt is recorded. Only the retained- path takes a unit. A request whose declared `Content-Length` is at or below+ work outlived its 30 s deadline). Neither sends anything to the provider,+ fails over or installs a cooldown. A capacity refusal is answered before the+ body is read, with `Connection: close` and `Retry-After: 10`, so no attempt+ is recorded. The Host retries it once in the same provider after that delay,+ with the next attempt index, and re-uploads the body. A second refusal, or a+ refusal without `Retry-After` (an older control-api), is terminal: a+ non-retryable `LLM_API_CALL_FAILED`. A caller-pinned attempt index is never+ retried. The retry adds at most 50 s (a 40 s queue refusal plus the 10 s+ wait) before the second attempt's own 80 s. `authorize_timeout` carries no+ `Retry-After` and is terminal at once. Only the retained path takes a unit. A+ request whose declared `Content-Length` is at or belowthe text authorize envelope (`maxRequestBodyBytes` +`ENVELOPE_ALLOWANCE_BYTES`, 8404992 bytes, the larger of the two contracts)is parsed with that limit and never queues. A larger declared length, adiff --git a/mcp-host/package-lock.json b/mcp-host/package-lock.jsonindex 1a3db01f1..5084bf3fe 100644--- a/mcp-host/package-lock.json+++ b/mcp-host/package-lock.json@@ -1,12 +1,12 @@{"name": "clerum-mcp-host",- "version": "0.1.490",+ "version": "0.1.491","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "clerum-mcp-host",- "version": "0.1.490",+ "version": "0.1.491","license": "SEE LICENSE IN LICENSE","dependencies": {"@anthropic-ai/sdk": "^0.32.0",diff --git a/mcp-host/package.json b/mcp-host/package.jsonindex 5f1ace456..b4e1ff97d 100644--- a/mcp-host/package.json+++ b/mcp-host/package.json@@ -1,6 +1,6 @@{"name": "clerum-mcp-host",- "version": "0.1.490",+ "version": "0.1.491","description": "MCP Host - reads Host CRD and provides LLM access via OpenAI/Claude","main": "dist/main.js","engines": {diff --git a/mcp-host/src/llm/__tests__/codexSubscription.test.ts b/mcp-host/src/llm/__tests__/codexSubscription.test.tsindex 0bfb929ea..9dcc40d9d 100644--- a/mcp-host/src/llm/__tests__/codexSubscription.test.ts+++ b/mcp-host/src/llm/__tests__/codexSubscription.test.ts@@ -2012,6 +2012,174 @@ describe('CodexSubscriptionProvider Retry-After retry (G1-9, #720)', () => {expect(wired.stream).toHaveBeenCalledTimes(0)})+ // M-A (review 5426789128): control-api's capacity refusal is answered+ // before any read and carries Retry-After. The provider retries it once in+ // place; the second refusal is terminal, with no failover or cooldown.+ const capacityRefusal = (seconds?: number) =>+ Response.json(+ { error: 'authorize_capacity_exceeded' },+ seconds === undefined+ ? { status: 503 }+ : { status: 503, headers: { 'retry-after': String(seconds) } }+ )++ it('M-A1: a capacity refusal on authorize is retried once after its Retry-After', async () => {+ vi.useFakeTimers()+ const fetchFn = vi+ .fn()+ .mockResolvedValueOnce(capacityRefusal(10))+ .mockResolvedValueOnce(authorizedSecond())+ const wired = deps({ authorize: realAuthorize(fetchFn), stream: vi.fn().mockResolvedValue(ok) })+ const provider = new CodexSubscriptionProvider('gpt-5.3-codex', wired as never)++ const turn = provider.completeSingleTurn([{ role: 'user', content: 'hi' }])+ await vi.advanceTimersByTimeAsync(9999)+ // Witness: the first authorize ran; nothing was dispatched on its refusal.+ expect(fetchFn).toHaveBeenCalledTimes(1)+ expect(wired.proxy.stream).toHaveBeenCalledTimes(0)+ await vi.advanceTimersByTimeAsync(1)+ await expect(turn).resolves.toMatchObject({ content: 'after the wait' })++ expect(fetchFn).toHaveBeenCalledTimes(2)+ const [first, second] = fetchFn.mock.calls.map(call => JSON.parse(call[1].body as string))+ expect(second.invocationId).toBe(first.invocationId)+ expect([first.providerAttemptIndex, second.providerAttemptIndex]).toEqual([1, 2])+ expect(wired.proxy.stream).toHaveBeenCalledTimes(1)+ expect(wired.proxy.stream.mock.calls[0][0].executionTicket).toBe('ticket-second')+ })++ it('M-A2: a second capacity refusal is terminal and never fails over', async () => {+ vi.useFakeTimers()+ const fetchFn = vi+ .fn()+ .mockResolvedValueOnce(capacityRefusal(10))+ .mockResolvedValueOnce(capacityRefusal(10))+ .mockResolvedValueOnce(authorizedSecond())+ const wired = deps({ authorize: realAuthorize(fetchFn) })+ const provider = new CodexSubscriptionProvider('gpt-5.3-codex', wired as never)++ const turn = provider.completeSingleTurn([{ role: 'user', content: 'hi' }])+ const caught = turn.then(+ () => undefined,+ (e: unknown) => e+ )+ await vi.advanceTimersByTimeAsync(60_000)+ const err = await caught++ expect(err).toBeInstanceOf(CodexAuthorizeError)+ expect(err).toMatchObject({ code: 'authorize_capacity_exceeded', retryAfterMs: 10_000 })+ const classified = provider.classifyError(err)+ expect(classified).toMatchObject({+ code: LlmErrorCode.ApiCallFailed,+ retryable: false,+ providerDispatched: false,+ })+ expect(classifyFailoverClass(classified.code, classified.retryable)).toBeNull()+ // Exactly two authorizes: the retry ran once and the third answer was never asked for.+ expect(fetchFn).toHaveBeenCalledTimes(2)+ expect(wired.proxy.stream).toHaveBeenCalledTimes(0)+ })++ it('M-A3: a capacity refusal without Retry-After is not retried', async () => {+ vi.useFakeTimers()+ const fetchFn = vi+ .fn()+ .mockResolvedValueOnce(capacityRefusal())+ .mockResolvedValueOnce(authorizedSecond())+ const wired = deps({ authorize: realAuthorize(fetchFn) })+ const provider = new CodexSubscriptionProvider('gpt-5.3-codex', wired as never)++ const turn = provider.completeSingleTurn([{ role: 'user', content: 'hi' }])+ const caught = turn.then(+ () => undefined,+ (e: unknown) => e+ )+ await vi.advanceTimersByTimeAsync(60_000)+ const err = await caught++ expect(err).toBeInstanceOf(CodexAuthorizeError)+ expect(err).toMatchObject({ code: 'authorize_capacity_exceeded' })+ expect((err as CodexAuthorizeError).retryAfterMs).toBeUndefined()+ // Witness: the first authorize ran; the negative is that no second one did.+ expect(fetchFn).toHaveBeenCalledTimes(1)+ expect(wired.proxy.stream).toHaveBeenCalledTimes(0)+ })++ it('M-A4: a capacity refusal on a caller-pinned attempt index is not retried', async () => {+ vi.useFakeTimers()+ const fetchFn = vi+ .fn()+ .mockResolvedValueOnce(capacityRefusal(10))+ .mockResolvedValueOnce(authorizedSecond())+ const wired = deps({ authorize: realAuthorize(fetchFn) })+ wired.attemptContext = vi.fn(() => ({+ policyRevision: 1,+ policyHash: 'b'.repeat(64),+ hostRef: 'chatllm',+ providerAttemptIndex: 3,+ }))+ const provider = new CodexSubscriptionProvider('gpt-5.3-codex', wired as never)++ const turn = provider.completeSingleTurn([{ role: 'user', content: 'hi' }])+ const caught = turn.then(+ () => undefined,+ (e: unknown) => e+ )+ await vi.advanceTimersByTimeAsync(60_000)+ const err = await caught++ expect(err).toMatchObject({ code: 'authorize_capacity_exceeded', retryAfterMs: 10_000 })+ // Witness: the pinned authorize ran; the negative is that no second one did.+ expect(fetchFn).toHaveBeenCalledTimes(1)+ expect(JSON.parse(fetchFn.mock.calls[0][1].body as string).providerAttemptIndex).toBe(3)+ expect(wired.proxy.stream).toHaveBeenCalledTimes(0)+ })++ it('M-A5: a proxy error named authorize_capacity_exceeded is not retried', async () => {+ vi.useFakeTimers()+ const err = new CodexProxyError('authorize_capacity_exceeded', 'proxy refused', {+ retryAfterMs: 1000,+ })+ const wired = deps({+ authorize: authorizeTwice(),+ stream: vi.fn().mockRejectedValueOnce(err).mockResolvedValueOnce(ok),+ })+ const provider = new CodexSubscriptionProvider('gpt-5.3-codex', wired as never)++ const turn = provider.completeSingleTurn([{ role: 'user', content: 'hi' }])+ const settled = expect(turn).rejects.toBe(err)+ await vi.advanceTimersByTimeAsync(60_000)+ await settled+ // Witness: the attempt was dispatched once; the negative is that no retry followed.+ expect(wired.authorizer.authorize).toHaveBeenCalledTimes(1)+ expect(wired.proxy.stream).toHaveBeenCalledTimes(1)+ })++ it('M-A6: an abort during the capacity wait rejects with the abort reason and sends nothing more', async () => {+ vi.useFakeTimers()+ const controller = new AbortController()+ const reason = new Error('caller gave up')+ const fetchFn = vi+ .fn()+ .mockResolvedValueOnce(capacityRefusal(10))+ .mockResolvedValueOnce(authorizedSecond())+ const wired = deps({ authorize: realAuthorize(fetchFn), stream: vi.fn().mockResolvedValue(ok) })+ const provider = new CodexSubscriptionProvider('gpt-5.3-codex', wired as never)++ const turn = provider.completeSingleTurn([{ role: 'user', content: 'hi' }], {+ signal: controller.signal,+ })+ const settled = expect(turn).rejects.toBe(reason)+ await vi.advanceTimersByTimeAsync(5000)+ // Witness: the first authorize ran and the wait is pending.+ expect(fetchFn).toHaveBeenCalledTimes(1)+ controller.abort(reason)+ await settled+ await vi.advanceTimersByTimeAsync(60_000)+ expect(fetchFn).toHaveBeenCalledTimes(1)+ expect(wired.proxy.stream).toHaveBeenCalledTimes(0)+ })+it('G1-12: an abort after the wait resolves and before the second authorize sends nothing more', async () => {vi.useFakeTimers()const controller = new AbortController()diff --git a/mcp-host/src/llm/__tests__/grokSubscription.test.ts b/mcp-host/src/llm/__tests__/grokSubscription.test.tsindex 14d9581ec..0758337da 100644--- a/mcp-host/src/llm/__tests__/grokSubscription.test.ts+++ b/mcp-host/src/llm/__tests__/grokSubscription.test.ts@@ -1888,6 +1888,174 @@ describe('GrokSubscriptionProvider Retry-After retry (G1-9, #720)', () => {expect(wired.proxy.stream).toHaveBeenCalledTimes(0)})+ // M-A (review 5426789128): control-api's capacity refusal is answered+ // before any read and carries Retry-After. The provider retries it once in+ // place; the second refusal is terminal, with no failover or cooldown.+ const capacityRefusal = (seconds?: number) =>+ Response.json(+ { error: 'authorize_capacity_exceeded' },+ seconds === undefined+ ? { status: 503 }+ : { status: 503, headers: { 'retry-after': String(seconds) } }+ )++ it('M-A1: a capacity refusal on authorize is retried once after its Retry-After', async () => {+ vi.useFakeTimers()+ const fetchFn = vi+ .fn()+ .mockResolvedValueOnce(capacityRefusal(10))+ .mockResolvedValueOnce(authorizedSecond())+ const wired = deps({ authorize: realAuthorize(fetchFn), stream: vi.fn().mockResolvedValue(ok) })+ const provider = new GrokSubscriptionProvider('grok-4.6', wired as never)++ const turn = provider.completeSingleTurn([{ role: 'user', content: 'hi' }])+ await vi.advanceTimersByTimeAsync(9999)+ // Witness: the first authorize ran; nothing was dispatched on its refusal.+ expect(fetchFn).toHaveBeenCalledTimes(1)+ expect(wired.proxy.stream).toHaveBeenCalledTimes(0)+ await vi.advanceTimersByTimeAsync(1)+ await expect(turn).resolves.toMatchObject({ content: 'after the wait' })++ expect(fetchFn).toHaveBeenCalledTimes(2)+ const [first, second] = fetchFn.mock.calls.map(call => JSON.parse(call[1].body as string))+ expect(second.invocationId).toBe(first.invocationId)+ expect([first.providerAttemptIndex, second.providerAttemptIndex]).toEqual([1, 2])+ expect(wired.proxy.stream).toHaveBeenCalledTimes(1)+ expect(wired.proxy.stream.mock.calls[0][0].executionTicket).toBe('ticket-second')+ })++ it('M-A2: a second capacity refusal is terminal and never fails over', async () => {+ vi.useFakeTimers()+ const fetchFn = vi+ .fn()+ .mockResolvedValueOnce(capacityRefusal(10))+ .mockResolvedValueOnce(capacityRefusal(10))+ .mockResolvedValueOnce(authorizedSecond())+ const wired = deps({ authorize: realAuthorize(fetchFn) })+ const provider = new GrokSubscriptionProvider('grok-4.6', wired as never)++ const turn = provider.completeSingleTurn([{ role: 'user', content: 'hi' }])+ const caught = turn.then(+ () => undefined,+ (e: unknown) => e+ )+ await vi.advanceTimersByTimeAsync(60_000)+ const err = await caught++ expect(err).toBeInstanceOf(CodexAuthorizeError)+ expect(err).toMatchObject({ code: 'authorize_capacity_exceeded', retryAfterMs: 10_000 })+ const classified = provider.classifyError(err)+ expect(classified).toMatchObject({+ code: LlmErrorCode.ApiCallFailed,+ retryable: false,+ providerDispatched: false,+ })+ expect(classifyFailoverClass(classified.code, classified.retryable)).toBeNull()+ // Exactly two authorizes: the retry ran once and the third answer was never asked for.+ expect(fetchFn).toHaveBeenCalledTimes(2)+ expect(wired.proxy.stream).toHaveBeenCalledTimes(0)+ })++ it('M-A3: a capacity refusal without Retry-After is not retried', async () => {+ vi.useFakeTimers()+ const fetchFn = vi+ .fn()+ .mockResolvedValueOnce(capacityRefusal())+ .mockResolvedValueOnce(authorizedSecond())+ const wired = deps({ authorize: realAuthorize(fetchFn) })+ const provider = new GrokSubscriptionProvider('grok-4.6', wired as never)++ const turn = provider.completeSingleTurn([{ role: 'user', content: 'hi' }])+ const caught = turn.then(+ () => undefined,+ (e: unknown) => e+ )+ await vi.advanceTimersByTimeAsync(60_000)+ const err = await caught++ expect(err).toBeInstanceOf(CodexAuthorizeError)+ expect(err).toMatchObject({ code: 'authorize_capacity_exceeded' })+ expect((err as CodexAuthorizeError).retryAfterMs).toBeUndefined()+ // Witness: the first authorize ran; the negative is that no second one did.+ expect(fetchFn).toHaveBeenCalledTimes(1)+ expect(wired.proxy.stream).toHaveBeenCalledTimes(0)+ })++ it('M-A4: a capacity refusal on a caller-pinned attempt index is not retried', async () => {+ vi.useFakeTimers()+ const fetchFn = vi+ .fn()+ .mockResolvedValueOnce(capacityRefusal(10))+ .mockResolvedValueOnce(authorizedSecond())+ const wired = deps({ authorize: realAuthorize(fetchFn) })+ wired.attemptContext = vi.fn(() => ({+ policyRevision: 1,+ policyHash: 'b'.repeat(64),+ hostRef: 'chatllm',+ providerAttemptIndex: 3,+ }))+ const provider = new GrokSubscriptionProvider('grok-4.6', wired as never)++ const turn = provider.completeSingleTurn([{ role: 'user', content: 'hi' }])+ const caught = turn.then(+ () => undefined,+ (e: unknown) => e+ )+ await vi.advanceTimersByTimeAsync(60_000)+ const err = await caught++ expect(err).toMatchObject({ code: 'authorize_capacity_exceeded', retryAfterMs: 10_000 })+ // Witness: the pinned authorize ran; the negative is that no second one did.+ expect(fetchFn).toHaveBeenCalledTimes(1)+ expect(JSON.parse(fetchFn.mock.calls[0][1].body as string).providerAttemptIndex).toBe(3)+ expect(wired.proxy.stream).toHaveBeenCalledTimes(0)+ })++ it('M-A5: a proxy error named authorize_capacity_exceeded is not retried', async () => {+ vi.useFakeTimers()+ const err = new GrokProxyError('authorize_capacity_exceeded', 'proxy refused', {+ retryAfterMs: 1000,+ })+ const wired = deps({+ authorize: authorizeTwice(),+ stream: vi.fn().mockRejectedValueOnce(err).mockResolvedValueOnce(ok),+ })+ const provider = new GrokSubscriptionProvider('grok-4.6', wired as never)++ const turn = provider.completeSingleTurn([{ role: 'user', content: 'hi' }])+ const settled = expect(turn).rejects.toBe(err)+ await vi.advanceTimersByTimeAsync(60_000)+ await settled+ // Witness: the attempt was dispatched once; the negative is that no retry followed.+ expect(wired.authorizer.authorize).toHaveBeenCalledTimes(1)+ expect(wired.proxy.stream).toHaveBeenCalledTimes(1)+ })++ it('M-A6: an abort during the capacity wait rejects with the abort reason and sends nothing more', async () => {+ vi.useFakeTimers()+ const controller = new AbortController()+ const reason = new Error('caller gave up')+ const fetchFn = vi+ .fn()+ .mockResolvedValueOnce(capacityRefusal(10))+ .mockResolvedValueOnce(authorizedSecond())+ const wired = deps({ authorize: realAuthorize(fetchFn), stream: vi.fn().mockResolvedValue(ok) })+ const provider = new GrokSubscriptionProvider('grok-4.6', wired as never)++ const turn = provider.completeSingleTurn([{ role: 'user', content: 'hi' }], {+ signal: controller.signal,+ })+ const settled = expect(turn).rejects.toBe(reason)+ await vi.advanceTimersByTimeAsync(5000)+ // Witness: the first authorize ran and the wait is pending.+ expect(fetchFn).toHaveBeenCalledTimes(1)+ controller.abort(reason)+ await settled+ await vi.advanceTimersByTimeAsync(60_000)+ expect(fetchFn).toHaveBeenCalledTimes(1)+ expect(wired.proxy.stream).toHaveBeenCalledTimes(0)+ })+it('G1-12: an abort after the wait resolves and before the second authorize sends nothing more', async () => {vi.useFakeTimers()const controller = new AbortController()diff --git a/mcp-host/src/llm/__tests__/providerAttemptAuthorizer.test.ts b/mcp-host/src/llm/__tests__/providerAttemptAuthorizer.test.tsindex 053282149..4de97005f 100644--- a/mcp-host/src/llm/__tests__/providerAttemptAuthorizer.test.ts+++ b/mcp-host/src/llm/__tests__/providerAttemptAuthorizer.test.ts@@ -619,6 +619,53 @@ describe('ProviderAttemptAuthorizer', () => {})+ // M-A (review 5426789128): control-api's capacity refusal carries+ // Retry-After, and only that coded 503 has it read.+ it.each([+ [+ 'a capacity refusal with Retry-After',+ () =>+ Response.json(+ { error: 'authorize_capacity_exceeded' },+ { status: 503, headers: { 'retry-after': '10' } }+ ),+ 'authorize_capacity_exceeded',+ 10_000,+ ],+ [+ 'a capacity refusal without Retry-After',+ () => Response.json({ error: 'authorize_capacity_exceeded' }, { status: 503 }),+ 'authorize_capacity_exceeded',+ undefined,+ ],+ [+ 'an authorize_timeout with Retry-After',+ () =>+ Response.json(+ { error: 'authorize_timeout' },+ { status: 503, headers: { 'retry-after': '10' } }+ ),+ 'authorize_timeout',+ undefined,+ ],+ [+ 'an uncoded 503 with Retry-After',+ () =>+ new Response('<html>503 Service Temporarily Unavailable</html>', {+ status: 503,+ headers: { 'content-type': 'text/html', 'retry-after': '10' },+ }),+ 'provider_unavailable',+ undefined,+ ],+ ] as const)('M-A reads Retry-After for %s', async (_label, response, code, retryAfter) => {+ const { err, fetchFn } = await authorizeFailure(response())+ expect(fetchFn).toHaveBeenCalledTimes(1)+ expect(err).toBeInstanceOf(CodexAuthorizeError)+ expect(err).toMatchObject({ code })+ expect((err as CodexAuthorizeError).retryAfterMs).toBe(retryAfter)+ })+// G1-6 (#720): a limiter in front of control-api can answer 429 with no JSON// code. It is a rate limit, not a provider outage.it('G1-6d reads an HTML 429 from the gateway as rate_limited', async () => {diff --git a/mcp-host/src/llm/codexSubscription.ts b/mcp-host/src/llm/codexSubscription.tsindex 788c6611e..1f47f724b 100644--- a/mcp-host/src/llm/codexSubscription.ts+++ b/mcp-host/src/llm/codexSubscription.ts@@ -22,7 +22,7 @@ import { CodexLlmProxyClient, CodexProxyError } from './codexLlmProxyClient'import { classifyUnknown } from './errorClassification'import { IMAGE_SOURCE_INVALID, canonicalRefusalCode, projectMessage } from './imageSource'import { CodexAuthorizeError, ProviderAttemptAuthorizer } from './providerAttemptAuthorizer'-import { rateLimitRetryDelayMs, waitBeforeRetry } from './rateLimitRetry'+import { authorizeRetryDelayMs, rateLimitRetryDelayMs, waitBeforeRetry } from './rateLimitRetry'import { type LlmProvider, descriptorFor } from './registryCore'import type { ClassifiedError, SingleTurnProvider } from './types'@@ -528,12 +528,17 @@ export class CodexSubscriptionProvider implements SingleTurnProvider {// after that delay, under a new authorize: a redeemed ticket cannot be// reused. The 429 may come from the proxy or from control-api's// authorize limiter (G1-11). A caller that pins the attempt index owns- // its own retries.+ // its own retries. control-api's authorize_capacity_exceeded, refused+ // before any read, is retried once the same way (M-A, review+ // 5426789128); a proxy error of that name is not.const waitMs =- context.providerAttemptIndex === undefined &&- (err instanceof CodexProxyError || err instanceof CodexAuthorizeError)- ? rateLimitRetryDelayMs(err.code, err.retryAfterMs)- : undefined+ context.providerAttemptIndex !== undefined+ ? undefined+ : err instanceof CodexAuthorizeError+ ? authorizeRetryDelayMs(err.code, err.retryAfterMs)+ : err instanceof CodexProxyError+ ? rateLimitRetryDelayMs(err.code, err.retryAfterMs)+ : undefinedif (waitMs === undefined) throw errawait waitBeforeRetry(waitMs, options?.signal)// The wait stops watching the signal once its timer fires: an abort indiff --git a/mcp-host/src/llm/failover/__tests__/subscriptionAdmission.integration.test.ts b/mcp-host/src/llm/failover/__tests__/subscriptionAdmission.integration.test.tsindex ce8fd6183..01cb5adc7 100644--- a/mcp-host/src/llm/failover/__tests__/subscriptionAdmission.integration.test.ts+++ b/mcp-host/src/llm/failover/__tests__/subscriptionAdmission.integration.test.ts@@ -8,6 +8,11 @@ import { CodexLlmProxyClient } from '../../codexLlmProxyClient'import { CodexSubscriptionProvider } from '../../codexSubscription'import { GrokLlmProxyClient } from '../../grokLlmProxyClient'import { GrokSubscriptionProvider } from '../../grokSubscription'+import {+ CodexAuthorizeError,+ ProviderAttemptAuthorizer,+ resolveCodexAuthorizeUrl,+} from '../../providerAttemptAuthorizer'import { type ClassifiedLike, FailoverEngine } from '../engine'import type { FailoverTarget, ModelPair } from '../types'@@ -354,5 +359,48 @@ describe.each(['grok', 'codex'] as const)(expect(f.engine.servedBy()).toEqual({ ...f.primary, fallback: false })})++ // M-A (review 5426789128): control-api's capacity refusal with+ // Retry-After is retried once in place; the second refusal is terminal and+ // the turn never leaves the primary or installs a cooldown.+ it('a repeated authorize capacity refusal retries once on primary without failover', async () => {+ const f = await fixture(variant)+ const authorizeFetch = vi.fn(async () =>+ Response.json(+ { error: 'authorize_capacity_exceeded' },+ { status: 503, headers: { 'retry-after': '1' } }+ )+ )+ const authorizer = new ProviderAttemptAuthorizer({+ authorizeUrl: resolveCodexAuthorizeUrl('http://gateway:8092'),+ readPlatformJwt: () => 'platform-jwt',+ fetchFn: authorizeFetch as unknown as typeof fetch,+ })+ f.authorize.mockImplementation(((body: never, options: never) =>+ authorizer.authorize(body, options)) as never)++ const err = await f.run('bounded text').then(+ () => undefined,+ (e: unknown) => e+ )++ expect(err).toBeInstanceOf(CodexAuthorizeError)+ expect(err).toMatchObject({ code: 'authorize_capacity_exceeded', retryAfterMs: 1000 })+ expect(f.classified()).toMatchObject({+ code: LlmErrorCode.ApiCallFailed,+ retryable: false,+ providerCode: 'authorize_capacity_exceeded',+ providerDispatched: false,+ })+ // Witness: authorize ran exactly twice, the attempt and its one retry.+ expect(f.authorize).toHaveBeenCalledTimes(2)+ expect(authorizeFetch).toHaveBeenCalledTimes(2)+ expect(f.hostFetch).not.toHaveBeenCalled()+ expect(f.built.map(target => target.kind)).toEqual(['primary'])+ expect(f.fallback).not.toHaveBeenCalled()+ expect(f.onSwitch).not.toHaveBeenCalled()+ expect(f.metricInc).not.toHaveBeenCalled()+ expect(f.engine.planTargets(f.primary)[0]?.kind).toBe('primary')+ })})diff --git a/mcp-host/src/llm/grokSubscription.ts b/mcp-host/src/llm/grokSubscription.tsindex ce09bbe59..634da4788 100644--- a/mcp-host/src/llm/grokSubscription.ts+++ b/mcp-host/src/llm/grokSubscription.ts@@ -23,7 +23,7 @@ import { classifyUnknown } from './errorClassification'import { GrokLlmProxyClient, GrokProxyError } from './grokLlmProxyClient'import { IMAGE_SOURCE_INVALID, canonicalRefusalCode, projectMessage } from './imageSource'import { CodexAuthorizeError, ProviderAttemptAuthorizer } from './providerAttemptAuthorizer'-import { rateLimitRetryDelayMs, waitBeforeRetry } from './rateLimitRetry'+import { authorizeRetryDelayMs, rateLimitRetryDelayMs, waitBeforeRetry } from './rateLimitRetry'import { type LlmProvider, descriptorFor } from './registryCore'import type { ClassifiedError, SingleTurnProvider } from './types'@@ -523,12 +523,17 @@ export class GrokSubscriptionProvider implements SingleTurnProvider {// after that delay, under a new authorize: a redeemed ticket cannot be// reused. The 429 may come from the proxy or from control-api's// authorize limiter (G1-11). A caller that pins the attempt index owns- // its own retries.+ // its own retries. control-api's authorize_capacity_exceeded, refused+ // before any read, is retried once the same way (M-A, review+ // 5426789128); a proxy error of that name is not.const waitMs =- context.providerAttemptIndex === undefined &&- (err instanceof GrokProxyError || err instanceof CodexAuthorizeError)- ? rateLimitRetryDelayMs(err.code, err.retryAfterMs)- : undefined+ context.providerAttemptIndex !== undefined+ ? undefined+ : err instanceof CodexAuthorizeError+ ? authorizeRetryDelayMs(err.code, err.retryAfterMs)+ : err instanceof GrokProxyError+ ? rateLimitRetryDelayMs(err.code, err.retryAfterMs)+ : undefinedif (waitMs === undefined) throw errawait waitBeforeRetry(waitMs, options?.signal)// The wait stops watching the signal once its timer fires: an abort indiff --git a/mcp-host/src/llm/providerAttemptAuthorizer.ts b/mcp-host/src/llm/providerAttemptAuthorizer.tsindex 9cb58b799..bf5621fe3 100644--- a/mcp-host/src/llm/providerAttemptAuthorizer.ts+++ b/mcp-host/src/llm/providerAttemptAuthorizer.ts@@ -221,7 +221,15 @@ export class ProviderAttemptAuthorizer {: code === 'authorize_timeout'? 'Request authorization timed out. Wait for active requests to finish, then try again.': `authorize failed with ${response.status}`,- { retryAfterMs: response.status === 429 ? retryAfterMs(response) : undefined }+ // Retry-After is read on a 429 and on control-api's capacity refusal,+ // which the provider retries once after it (M-A).+ {+ retryAfterMs:+ response.status === 429 ||+ (response.status === 503 && code === 'authorize_capacity_exceeded')+ ? retryAfterMs(response)+ : undefined,+ })}for (const key of LEAK_KEYS) {diff --git a/mcp-host/src/llm/rateLimitRetry.ts b/mcp-host/src/llm/rateLimitRetry.tsindex a9e3228a8..2797c475b 100644--- a/mcp-host/src/llm/rateLimitRetry.ts+++ b/mcp-host/src/llm/rateLimitRetry.ts@@ -19,7 +19,27 @@ export function rateLimitRetryDelayMs(code: string,retryAfterMs: number | undefined): number | undefined {- if (code !== 'rate_limited' || retryAfterMs === undefined) return undefined+ return code === 'rate_limited' ? boundedRetryDelayMs(retryAfterMs) : undefined+}++/**+ * The wait before the single retry of an authorize error: a `rate_limited`+ * as in {@link rateLimitRetryDelayMs}, or control-api's+ * `authorize_capacity_exceeded` with its Retry-After, on the same bound.+ * Nothing was read or recorded for that refusal, so the retry stays with the+ * same provider; a second refusal is terminal and never fails over.+ */+export function authorizeRetryDelayMs(+ code: string,+ retryAfterMs: number | undefined+): number | undefined {+ return code === 'rate_limited' || code === 'authorize_capacity_exceeded'+ ? boundedRetryDelayMs(retryAfterMs)+ : undefined+}++function boundedRetryDelayMs(retryAfterMs: number | undefined): number | undefined {+ if (retryAfterMs === undefined) return undefinedreturn retryAfterMs > 0 && retryAfterMs <= RATE_LIMIT_RETRY_MAX_WAIT_MS ? retryAfterMs : undefined}