Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

fix(gateway): let control-api answer slow authorizes before nginx times out

Alfredo Lopeza888536a4c9c
diff --git a/control-api/package-lock.json b/control-api/package-lock.json
index 6094a5ab6..17ea1b018 100644
--- a/control-api/package-lock.json
+++ b/control-api/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "clerum-control-api",
- "version": "0.1.719",
+ "version": "0.1.720",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "clerum-control-api",
- "version": "0.1.719",
+ "version": "0.1.720",
"dependencies": {
"@clerum/display-field": "file:../packages/display-field",
"@clerum/grok-provider-attempt-contract": "file:../packages/grok-provider-attempt-contract",
diff --git a/control-api/package.json b/control-api/package.json
index c12af00a3..a3cbb2738 100644
--- a/control-api/package.json
+++ b/control-api/package.json
@@ -1,6 +1,6 @@
{
"name": "clerum-control-api",
- "version": "0.1.719",
+ "version": "0.1.720",
"description": "Internal Clerum control plane REST API + UI for managing CRDs and secrets",
"main": "dist/main.js",
"engines": {
diff --git a/control-api/test/deploy.controlPlaneUnavailableGateway.test.ts b/control-api/test/deploy.controlPlaneUnavailableGateway.test.ts
index 722873daf..0b46d9414 100644
--- a/control-api/test/deploy.controlPlaneUnavailableGateway.test.ts
+++ b/control-api/test/deploy.controlPlaneUnavailableGateway.test.ts
@@ -1,5 +1,6 @@
import { describe, expect, it } from 'vitest'
import { readFileSync } from 'node:fs'
+import { LLM_PROVIDER_ATTEMPT_ADMISSION_POLICY } from '../src/middleware/llmProviderAttemptAdmissionLimits.js'
// G1-10 (#720): a 502 that a gateway generates itself means control-api
// produced no valid response (connect refused, closed before a response, or an
@@ -30,14 +31,19 @@ function proxyRedeemAbortMs(proxy: string): number {
return Number(match![1].replace(/_/g, ''))
}
-/** The one `proxy_connect_timeout` in a block, in milliseconds (`Ns` or `Nms`). */
-function connectTimeoutMs(block: string): number {
- const matches = [...block.matchAll(/\bproxy_connect_timeout\s+([0-9]+)(ms|s)\s*;/g)]
- expect(matches, 'exactly one proxy_connect_timeout').toHaveLength(1)
+/** The one `<directive>` timeout in a block, in milliseconds (`Ns` or `Nms`). */
+function timeoutMs(block: string, directive: string): number {
+ const matches = [...block.matchAll(new RegExp(`\\b${directive}\\s+([0-9]+)(ms|s)\\s*;`, 'g'))]
+ expect(matches, `exactly one ${directive}`).toHaveLength(1)
const [, value, unit] = matches[0]
return unit === 's' ? Number(value) * 1000 : Number(value)
}
+/** The one `proxy_connect_timeout` in a block, in milliseconds. */
+function connectTimeoutMs(block: string): number {
+ return timeoutMs(block, 'proxy_connect_timeout')
+}
+
// Any spelling nginx accepts: `error_page 502 = @x;`, `error_page 502 =@x;`, …
const ERROR_PAGE = /error_page\s+502\s*=\s*@control_plane_unavailable\s*;/g
// Any error_page that covers a 502, whatever it points to.
@@ -223,6 +229,25 @@ describe('gateway-generated connect-timeout 504 on user-facing control-plane hop
expect(block).not.toMatch(/\bproxy_connect_timeout\b/)
})
+ it.each(['proxy_read_timeout', 'proxy_send_timeout'])(
+ 'G1-12f: authorize %s outlasts the longest retained authorize, so control-api answers it',
+ directive => {
+ const approval = directives(configMap(WORKFLOW_APPROVAL_GATEWAY))
+ const block = locationBlock(approval, AUTHORIZE)
+ const { queueWaitMs, readDeadlineMs, workDeadlineMs } = LLM_PROVIDER_ATTEMPT_ADMISSION_POLICY
+ const longestAuthorizeMs = queueWaitMs + readDeadlineMs + workDeadlineMs
+ // Witnesses: the authorize block and the admission policy were read, and
+ // the server keeps 30 s for every other route, so the bound is local.
+ expect(block).toContain(AUTHORIZE_PROXY_PASS)
+ expect(longestAuthorizeMs).toBeGreaterThan(30_000)
+ const outside = approval.replace(block, '')
+ expect(timeoutMs(outside, directive)).toBe(30_000)
+ // Otherwise a queued request gets nginx's bare 504 before control-api's
+ // queue_wait or authorize_timeout.
+ expect(timeoutMs(block, directive)).toBeGreaterThan(longestAuthorizeMs)
+ }
+ )
+
it.each([
['Codex finalize', CODEX_FINALIZE],
['Grok finalize', GROK_FINALIZE],
diff --git a/deploy/base/control-plane/configmaps.yaml b/deploy/base/control-plane/configmaps.yaml
index 39febffec..de1227213 100644
--- a/deploy/base/control-plane/configmaps.yaml
+++ b/deploy/base/control-plane/configmaps.yaml
@@ -890,6 +890,13 @@ data:
}
error_page 502 = @control_plane_unavailable;
error_page 504 = @control_plane_connect_timeout;
+ # control-api holds a retained authorize for up to its queue wait
+ # (40 s), body read (10 s) and work deadline (30 s), and answers
+ # queue_wait or authorize_timeout itself. The server's 30 s would
+ # turn those answers into nginx's own 504. control-api pauses a
+ # queued body, so the send timer stalls as well as the read timer.
+ proxy_read_timeout 90s;
+ proxy_send_timeout 90s;
# This exact location matches the normalized URI, but a proxy_pass
# without a URI part forwards the raw one. Aliases such as
# %61uthorize or ./authorize would then reach control-api with the
diff --git a/docs/architecture/codex-subscription-transport-contract.md b/docs/architecture/codex-subscription-transport-contract.md
index 40468fb11..5648a55f8 100644
--- a/docs/architecture/codex-subscription-transport-contract.md
+++ b/docs/architecture/codex-subscription-transport-contract.md
@@ -137,7 +137,11 @@ order, paused before any read. One principal (JWT `sub` plus its sorted
`hostRefs`) holds at most two positions, running or queued. A waiter is
refused after 40 s, the holder's read deadline (10 s) plus its work deadline
(30 s). These bounds limit latency and fairness, not memory; the count of one
-is an uncertified ceiling (#813).
+is an uncertified ceiling (#813). The gateway's authorize location sets its
+read and send timeouts to 90 s, above the 80 s a retained authorize can take
+(queue, read and work), so control-api's own answer reaches the Host. A bare
+504 from the gateway (a read or send timeout on an accepted connection) is
+read as `authorize_timeout`, never as a provider outage.
The critical Codex test title migrated from "visual admission overflow answers
503 provider_unavailable and logs visual_gate" to "visual admission overflow
diff --git a/docs/architecture/grok-subscription-transport-contract.md b/docs/architecture/grok-subscription-transport-contract.md
index 86a80d9a4..3916cafe8 100644
--- a/docs/architecture/grok-subscription-transport-contract.md
+++ b/docs/architecture/grok-subscription-transport-contract.md
@@ -261,7 +261,11 @@ Errors:
`hostRefs`) holds at most two positions, running or queued. A waiter is
refused after 40 s, the holder's read deadline (10 s) plus its work deadline
(30 s). These bounds limit latency and fairness, not memory; the count of one
- is an uncertified ceiling (#813).
+ is an uncertified ceiling (#813). The gateway's authorize location sets its
+ read and send timeouts to 90 s, above the 80 s a retained authorize can take
+ (queue, read and work), so control-api's own answer reaches the Host. A bare
+ 504 from the gateway (a read or send timeout on an accepted connection) is
+ read as `authorize_timeout`, never as a provider outage.
Desktop. The composer budget for `grok-subscription` is 16 MiB per image and
16 MiB decoded in total, with no dimension bound. That is the shared rpc-proxy
diff --git a/mcp-host/package-lock.json b/mcp-host/package-lock.json
index e523568e4..1a3db01f1 100644
--- a/mcp-host/package-lock.json
+++ b/mcp-host/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "clerum-mcp-host",
- "version": "0.1.489",
+ "version": "0.1.490",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "clerum-mcp-host",
- "version": "0.1.489",
+ "version": "0.1.490",
"license": "SEE LICENSE IN LICENSE",
"dependencies": {
"@anthropic-ai/sdk": "^0.32.0",
diff --git a/mcp-host/package.json b/mcp-host/package.json
index 36bf0bf79..5f1ace456 100644
--- a/mcp-host/package.json
+++ b/mcp-host/package.json
@@ -1,6 +1,6 @@
{
"name": "clerum-mcp-host",
- "version": "0.1.489",
+ "version": "0.1.490",
"description": "MCP Host - reads Host CRD and provides LLM access via OpenAI/Claude",
"main": "dist/main.js",
"engines": {
diff --git a/mcp-host/src/llm/__tests__/providerAttemptAuthorizer.test.ts b/mcp-host/src/llm/__tests__/providerAttemptAuthorizer.test.ts
index d0a267b1d..053282149 100644
--- a/mcp-host/src/llm/__tests__/providerAttemptAuthorizer.test.ts
+++ b/mcp-host/src/llm/__tests__/providerAttemptAuthorizer.test.ts
@@ -453,6 +453,43 @@ describe('ProviderAttemptAuthorizer', () => {
})
})
+ // M-B (review 5426789128): the gateway's read or send timeout on authorize
+ // is a bare 504. control-api accepted the connection and authorize outlived
+ // the gateway, so it is a terminal authorize_timeout, not a provider outage
+ // that fails over and installs a cooldown.
+ it('T-R7-1c reads an HTML 504 from the gateway as a terminal authorize_timeout', async () => {
+ const { err, fetchFn } = await authorizeFailure(nginx(504, 'Gateway Time-out'))
+ // Liveness witness: the authorize hop really ran and got the 504.
+ expect(fetchFn).toHaveBeenCalledTimes(1)
+ expect(err).toBeInstanceOf(CodexAuthorizeError)
+ const message =
+ 'Request authorization timed out. Wait for active requests to finish, then try again.'
+ expect(err).toMatchObject({ code: 'authorize_timeout', message })
+ expect((err as CodexAuthorizeError).retryAfterMs).toBeUndefined()
+ for (const provider of [
+ new CodexSubscriptionProvider('gpt-5.3-codex', {} as never),
+ new GrokSubscriptionProvider('grok-4.6', {} as never),
+ ]) {
+ const classified = provider.classifyError(err)
+ expect(classified).toMatchObject({
+ code: LlmErrorCode.ApiCallFailed,
+ retryable: false,
+ message,
+ providerCode: 'authorize_timeout',
+ providerDispatched: false,
+ })
+ expect(classifyFailoverClass(classified.code, classified.retryable)).toBeNull()
+ }
+ })
+
+ it('T-R7-1c keeps a coded 504 from control-api as its own code', async () => {
+ const { err, fetchFn } = await authorizeFailure(
+ Response.json({ error: 'control_plane_unavailable' }, { status: 504 })
+ )
+ expect(fetchFn).toHaveBeenCalledTimes(1)
+ expect(err).toMatchObject({ code: 'control_plane_unavailable' })
+ })
+
async function nativeAuthorizeFailure(response: Response) {
const responseBody = await response.text()
const responseHeaders: Record<string, string> = {}
diff --git a/mcp-host/src/llm/providerAttemptAuthorizer.ts b/mcp-host/src/llm/providerAttemptAuthorizer.ts
index d7b12317f..9cb58b799 100644
--- a/mcp-host/src/llm/providerAttemptAuthorizer.ts
+++ b/mcp-host/src/llm/providerAttemptAuthorizer.ts
@@ -197,6 +197,11 @@ export class ProviderAttemptAuthorizer {
// `Too Many Requests`, so only a machine code a 429 carries (such as
// `budget_denied`) replaces `rate_limited`; a 429 with no JSON code is
// `rate_limited` too.
+ // A 504 with no JSON code is the gateway's read or send timeout on a
+ // connection control-api accepted (a connect timeout is answered as
+ // `control_plane_unavailable`), so authorize outlived the gateway, as
+ // with control-api's own `authorize_timeout`. It is not a provider
+ // outage and must not fail over.
const code =
response.status === 413
? 'payload_too_large'
@@ -204,7 +209,9 @@ export class ProviderAttemptAuthorizer {
? rateLimitedCode(payload.error)
: typeof payload.error === 'string'
? payload.error
- : 'provider_unavailable'
+ : response.status === 504
+ ? 'authorize_timeout'
+ : 'provider_unavailable'
throw new CodexAuthorizeError(
code,
code === 'payload_too_large'