Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
fix(gateway): let control-api answer slow authorizes before nginx times out
Alfredo Lopeza888536a4c9c
diff --git a/control-api/package-lock.json b/control-api/package-lock.jsonindex 6094a5ab6..17ea1b018 100644--- a/control-api/package-lock.json+++ b/control-api/package-lock.json@@ -1,12 +1,12 @@{"name": "clerum-control-api",- "version": "0.1.719",+ "version": "0.1.720","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "clerum-control-api",- "version": "0.1.719",+ "version": "0.1.720","dependencies": {"@clerum/display-field": "file:../packages/display-field","@clerum/grok-provider-attempt-contract": "file:../packages/grok-provider-attempt-contract",diff --git a/control-api/package.json b/control-api/package.jsonindex c12af00a3..a3cbb2738 100644--- a/control-api/package.json+++ b/control-api/package.json@@ -1,6 +1,6 @@{"name": "clerum-control-api",- "version": "0.1.719",+ "version": "0.1.720","description": "Internal Clerum control plane REST API + UI for managing CRDs and secrets","main": "dist/main.js","engines": {diff --git a/control-api/test/deploy.controlPlaneUnavailableGateway.test.ts b/control-api/test/deploy.controlPlaneUnavailableGateway.test.tsindex 722873daf..0b46d9414 100644--- a/control-api/test/deploy.controlPlaneUnavailableGateway.test.ts+++ b/control-api/test/deploy.controlPlaneUnavailableGateway.test.ts@@ -1,5 +1,6 @@import { describe, expect, it } from 'vitest'import { readFileSync } from 'node:fs'+import { LLM_PROVIDER_ATTEMPT_ADMISSION_POLICY } from '../src/middleware/llmProviderAttemptAdmissionLimits.js'// G1-10 (#720): a 502 that a gateway generates itself means control-api// produced no valid response (connect refused, closed before a response, or an@@ -30,14 +31,19 @@ function proxyRedeemAbortMs(proxy: string): number {return Number(match![1].replace(/_/g, ''))}-/** The one `proxy_connect_timeout` in a block, in milliseconds (`Ns` or `Nms`). */-function connectTimeoutMs(block: string): number {- const matches = [...block.matchAll(/\bproxy_connect_timeout\s+([0-9]+)(ms|s)\s*;/g)]- expect(matches, 'exactly one proxy_connect_timeout').toHaveLength(1)+/** The one `<directive>` timeout in a block, in milliseconds (`Ns` or `Nms`). */+function timeoutMs(block: string, directive: string): number {+ const matches = [...block.matchAll(new RegExp(`\\b${directive}\\s+([0-9]+)(ms|s)\\s*;`, 'g'))]+ expect(matches, `exactly one ${directive}`).toHaveLength(1)const [, value, unit] = matches[0]return unit === 's' ? Number(value) * 1000 : Number(value)}+/** The one `proxy_connect_timeout` in a block, in milliseconds. */+function connectTimeoutMs(block: string): number {+ return timeoutMs(block, 'proxy_connect_timeout')+}+// Any spelling nginx accepts: `error_page 502 = @x;`, `error_page 502 =@x;`, …const ERROR_PAGE = /error_page\s+502\s*=\s*@control_plane_unavailable\s*;/g// Any error_page that covers a 502, whatever it points to.@@ -223,6 +229,25 @@ describe('gateway-generated connect-timeout 504 on user-facing control-plane hopexpect(block).not.toMatch(/\bproxy_connect_timeout\b/)})+ it.each(['proxy_read_timeout', 'proxy_send_timeout'])(+ 'G1-12f: authorize %s outlasts the longest retained authorize, so control-api answers it',+ directive => {+ const approval = directives(configMap(WORKFLOW_APPROVAL_GATEWAY))+ const block = locationBlock(approval, AUTHORIZE)+ const { queueWaitMs, readDeadlineMs, workDeadlineMs } = LLM_PROVIDER_ATTEMPT_ADMISSION_POLICY+ const longestAuthorizeMs = queueWaitMs + readDeadlineMs + workDeadlineMs+ // Witnesses: the authorize block and the admission policy were read, and+ // the server keeps 30 s for every other route, so the bound is local.+ expect(block).toContain(AUTHORIZE_PROXY_PASS)+ expect(longestAuthorizeMs).toBeGreaterThan(30_000)+ const outside = approval.replace(block, '')+ expect(timeoutMs(outside, directive)).toBe(30_000)+ // Otherwise a queued request gets nginx's bare 504 before control-api's+ // queue_wait or authorize_timeout.+ expect(timeoutMs(block, directive)).toBeGreaterThan(longestAuthorizeMs)+ }+ )+it.each([['Codex finalize', CODEX_FINALIZE],['Grok finalize', GROK_FINALIZE],diff --git a/deploy/base/control-plane/configmaps.yaml b/deploy/base/control-plane/configmaps.yamlindex 39febffec..de1227213 100644--- a/deploy/base/control-plane/configmaps.yaml+++ b/deploy/base/control-plane/configmaps.yaml@@ -890,6 +890,13 @@ data:}error_page 502 = @control_plane_unavailable;error_page 504 = @control_plane_connect_timeout;+ # control-api holds a retained authorize for up to its queue wait+ # (40 s), body read (10 s) and work deadline (30 s), and answers+ # queue_wait or authorize_timeout itself. The server's 30 s would+ # turn those answers into nginx's own 504. control-api pauses a+ # queued body, so the send timer stalls as well as the read timer.+ proxy_read_timeout 90s;+ proxy_send_timeout 90s;# This exact location matches the normalized URI, but a proxy_pass# without a URI part forwards the raw one. Aliases such as# %61uthorize or ./authorize would then reach control-api with thediff --git a/docs/architecture/codex-subscription-transport-contract.md b/docs/architecture/codex-subscription-transport-contract.mdindex 40468fb11..5648a55f8 100644--- a/docs/architecture/codex-subscription-transport-contract.md+++ b/docs/architecture/codex-subscription-transport-contract.md@@ -137,7 +137,11 @@ order, paused before any read. One principal (JWT `sub` plus its sorted`hostRefs`) holds at most two positions, running or queued. A waiter isrefused after 40 s, the holder's read deadline (10 s) plus its work deadline(30 s). These bounds limit latency and fairness, not memory; the count of one-is an uncertified ceiling (#813).+is an uncertified ceiling (#813). The gateway's authorize location sets its+read and send timeouts to 90 s, above the 80 s a retained authorize can take+(queue, read and work), so control-api's own answer reaches the Host. A bare+504 from the gateway (a read or send timeout on an accepted connection) is+read as `authorize_timeout`, never as a provider outage.The critical Codex test title migrated from "visual admission overflow answers503 provider_unavailable and logs visual_gate" to "visual admission overflowdiff --git a/docs/architecture/grok-subscription-transport-contract.md b/docs/architecture/grok-subscription-transport-contract.mdindex 86a80d9a4..3916cafe8 100644--- a/docs/architecture/grok-subscription-transport-contract.md+++ b/docs/architecture/grok-subscription-transport-contract.md@@ -261,7 +261,11 @@ Errors:`hostRefs`) holds at most two positions, running or queued. A waiter isrefused after 40 s, the holder's read deadline (10 s) plus its work deadline(30 s). These bounds limit latency and fairness, not memory; the count of one- is an uncertified ceiling (#813).+ is an uncertified ceiling (#813). The gateway's authorize location sets its+ read and send timeouts to 90 s, above the 80 s a retained authorize can take+ (queue, read and work), so control-api's own answer reaches the Host. A bare+ 504 from the gateway (a read or send timeout on an accepted connection) is+ read as `authorize_timeout`, never as a provider outage.Desktop. The composer budget for `grok-subscription` is 16 MiB per image and16 MiB decoded in total, with no dimension bound. That is the shared rpc-proxydiff --git a/mcp-host/package-lock.json b/mcp-host/package-lock.jsonindex e523568e4..1a3db01f1 100644--- a/mcp-host/package-lock.json+++ b/mcp-host/package-lock.json@@ -1,12 +1,12 @@{"name": "clerum-mcp-host",- "version": "0.1.489",+ "version": "0.1.490","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "clerum-mcp-host",- "version": "0.1.489",+ "version": "0.1.490","license": "SEE LICENSE IN LICENSE","dependencies": {"@anthropic-ai/sdk": "^0.32.0",diff --git a/mcp-host/package.json b/mcp-host/package.jsonindex 36bf0bf79..5f1ace456 100644--- a/mcp-host/package.json+++ b/mcp-host/package.json@@ -1,6 +1,6 @@{"name": "clerum-mcp-host",- "version": "0.1.489",+ "version": "0.1.490","description": "MCP Host - reads Host CRD and provides LLM access via OpenAI/Claude","main": "dist/main.js","engines": {diff --git a/mcp-host/src/llm/__tests__/providerAttemptAuthorizer.test.ts b/mcp-host/src/llm/__tests__/providerAttemptAuthorizer.test.tsindex d0a267b1d..053282149 100644--- a/mcp-host/src/llm/__tests__/providerAttemptAuthorizer.test.ts+++ b/mcp-host/src/llm/__tests__/providerAttemptAuthorizer.test.ts@@ -453,6 +453,43 @@ describe('ProviderAttemptAuthorizer', () => {})})+ // M-B (review 5426789128): the gateway's read or send timeout on authorize+ // is a bare 504. control-api accepted the connection and authorize outlived+ // the gateway, so it is a terminal authorize_timeout, not a provider outage+ // that fails over and installs a cooldown.+ it('T-R7-1c reads an HTML 504 from the gateway as a terminal authorize_timeout', async () => {+ const { err, fetchFn } = await authorizeFailure(nginx(504, 'Gateway Time-out'))+ // Liveness witness: the authorize hop really ran and got the 504.+ expect(fetchFn).toHaveBeenCalledTimes(1)+ expect(err).toBeInstanceOf(CodexAuthorizeError)+ const message =+ 'Request authorization timed out. Wait for active requests to finish, then try again.'+ expect(err).toMatchObject({ code: 'authorize_timeout', message })+ expect((err as CodexAuthorizeError).retryAfterMs).toBeUndefined()+ for (const provider of [+ new CodexSubscriptionProvider('gpt-5.3-codex', {} as never),+ new GrokSubscriptionProvider('grok-4.6', {} as never),+ ]) {+ const classified = provider.classifyError(err)+ expect(classified).toMatchObject({+ code: LlmErrorCode.ApiCallFailed,+ retryable: false,+ message,+ providerCode: 'authorize_timeout',+ providerDispatched: false,+ })+ expect(classifyFailoverClass(classified.code, classified.retryable)).toBeNull()+ }+ })++ it('T-R7-1c keeps a coded 504 from control-api as its own code', async () => {+ const { err, fetchFn } = await authorizeFailure(+ Response.json({ error: 'control_plane_unavailable' }, { status: 504 })+ )+ expect(fetchFn).toHaveBeenCalledTimes(1)+ expect(err).toMatchObject({ code: 'control_plane_unavailable' })+ })+async function nativeAuthorizeFailure(response: Response) {const responseBody = await response.text()const responseHeaders: Record<string, string> = {}diff --git a/mcp-host/src/llm/providerAttemptAuthorizer.ts b/mcp-host/src/llm/providerAttemptAuthorizer.tsindex d7b12317f..9cb58b799 100644--- a/mcp-host/src/llm/providerAttemptAuthorizer.ts+++ b/mcp-host/src/llm/providerAttemptAuthorizer.ts@@ -197,6 +197,11 @@ export class ProviderAttemptAuthorizer {// `Too Many Requests`, so only a machine code a 429 carries (such as// `budget_denied`) replaces `rate_limited`; a 429 with no JSON code is// `rate_limited` too.+ // A 504 with no JSON code is the gateway's read or send timeout on a+ // connection control-api accepted (a connect timeout is answered as+ // `control_plane_unavailable`), so authorize outlived the gateway, as+ // with control-api's own `authorize_timeout`. It is not a provider+ // outage and must not fail over.const code =response.status === 413? 'payload_too_large'@@ -204,7 +209,9 @@ export class ProviderAttemptAuthorizer {? rateLimitedCode(payload.error): typeof payload.error === 'string'? payload.error- : 'provider_unavailable'+ : response.status === 504+ ? 'authorize_timeout'+ : 'provider_unavailable'throw new CodexAuthorizeError(code,code === 'payload_too_large'