Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

test(control-api): map a gateway connect-timeout 504 as control_plane_unavailable (#820)

Alfredo Lopez199f18bba67c
diff --git a/control-api/package-lock.json b/control-api/package-lock.json
index c0d3f00d5..efe682948 100644
--- a/control-api/package-lock.json
+++ b/control-api/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "clerum-control-api",
- "version": "0.1.636",
+ "version": "0.1.637",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "clerum-control-api",
- "version": "0.1.636",
+ "version": "0.1.637",
"dependencies": {
"@clerum/display-field": "file:../packages/display-field",
"@clerum/grok-provider-attempt-contract": "file:../packages/grok-provider-attempt-contract",
diff --git a/control-api/package.json b/control-api/package.json
index 7bab7053e..52a7ddeb8 100644
--- a/control-api/package.json
+++ b/control-api/package.json
@@ -1,6 +1,6 @@
{
"name": "clerum-control-api",
- "version": "0.1.636",
+ "version": "0.1.637",
"description": "Internal Clerum control plane REST API + UI for managing CRDs and secrets",
"main": "dist/main.js",
"engines": {
diff --git a/control-api/test/deploy.controlPlaneUnavailableGateway.test.ts b/control-api/test/deploy.controlPlaneUnavailableGateway.test.ts
index ecb27922e..1dd1bc8c3 100644
--- a/control-api/test/deploy.controlPlaneUnavailableGateway.test.ts
+++ b/control-api/test/deploy.controlPlaneUnavailableGateway.test.ts
@@ -6,14 +6,38 @@ import { readFileSync } from 'node:fs'
// invalid header; `proxy_intercept_errors` is off, so a 502 control-api returns
// passes through). In the last two cases control-api may already have processed
// the request. On the hops whose failure reaches the user, that 502 is answered
-// as JSON `control_plane_unavailable`. A 504 is not mapped: control-api may be
-// alive and slow.
+// as JSON `control_plane_unavailable`.
+//
+// G1-12 (#820): nginx answers 504 both for a connect timeout and for a read or
+// send timeout. Only a connect timeout leaves `$upstream_connect_time` at "-"
+// (nginx assigns it in `ngx_http_upstream_send_request`, which a connect
+// timeout never reaches), so on the same hops only that 504 is mapped. A
+// read-timeout 504 keeps nginx's own answer: control-api may be alive and slow.
const CONFIG = readFileSync(
new URL('../../deploy/base/control-plane/configmaps.yaml', import.meta.url),
'utf-8'
)
+/** `CONTROL_API_REQUEST_TIMEOUT_MS` of one proxy: how long it waits for redeem. */
+function proxyRedeemAbortMs(proxy: string): number {
+ const source = readFileSync(
+ new URL(`../../${proxy}/src/controlApiClient.ts`, import.meta.url),
+ 'utf-8'
+ )
+ const match = source.match(/export const CONTROL_API_REQUEST_TIMEOUT_MS = ([0-9_]+)\n/)
+ expect(match, `${proxy} must export CONTROL_API_REQUEST_TIMEOUT_MS as a literal`).not.toBeNull()
+ return Number(match![1].replace(/_/g, ''))
+}
+
+/** The one `proxy_connect_timeout` in a block, in milliseconds (`Ns` or `Nms`). */
+function connectTimeoutMs(block: string): number {
+ const matches = [...block.matchAll(/\bproxy_connect_timeout\s+([0-9]+)(ms|s)\s*;/g)]
+ expect(matches, 'exactly one proxy_connect_timeout').toHaveLength(1)
+ const [, value, unit] = matches[0]
+ return unit === 's' ? Number(value) * 1000 : Number(value)
+}
+
// Any spelling nginx accepts: `error_page 502 = @x;`, `error_page 502 =@x;`, …
const ERROR_PAGE = /error_page\s+502\s*=\s*@control_plane_unavailable\s*;/g
// Any error_page that covers a 502, whatever it points to.
@@ -93,7 +117,7 @@ describe('gateway-generated 502 on user-facing control-plane hops (G1-10, #720)'
expect(block).not.toContain('error_page')
})
- it('G1-10d: the mapping stays on the three user-facing hops and never covers a 504', () => {
+ it('G1-10d: the 502 mapping stays on the three user-facing hops', () => {
const approval = directives(configMap(WORKFLOW_APPROVAL_GATEWAY))
const rpc = directives(configMap(RPC_GATEWAY))
// Every error_page that covers a 502, at location or server level and in
@@ -104,7 +128,6 @@ describe('gateway-generated 502 on user-facing control-plane hops (G1-10, #720)'
expect(count(rpc, ANY_502_ERROR_PAGE)).toBe(2)
for (const gateway of [approval, rpc]) {
expect(gateway).toContain(NAMED_LOCATION)
- expect(gateway).not.toMatch(/error_page[^;]*\b504\b/)
}
})
@@ -120,3 +143,92 @@ describe('gateway-generated 502 on user-facing control-plane hops (G1-10, #720)'
}
)
})
+
+// Any spelling nginx accepts, as for the 502.
+const CONNECT_TIMEOUT_ERROR_PAGE = /error_page\s+504\s*=\s*@control_plane_connect_timeout\s*;/g
+// Any error_page that covers a 504, whatever it points to.
+const ANY_504_ERROR_PAGE = /error_page[^;]*\b504\b[^;]*;/g
+const CONNECT_TIMEOUT_LOCATION = 'location @control_plane_connect_timeout {'
+const CONNECT_FAILED = 'if ($upstream_connect_time = "-") {'
+
+describe('gateway-generated connect-timeout 504 on user-facing control-plane hops (G1-12, #820)', () => {
+ it.each([
+ ['H2 authorize', WORKFLOW_APPROVAL_GATEWAY, AUTHORIZE],
+ ['H3 Codex redeem', RPC_GATEWAY, CODEX_REDEEM],
+ ['H3 Grok redeem', RPC_GATEWAY, GROK_REDEEM],
+ ])('G1-12a: %s sends its own 504 to the connect-timeout check', (_hop, gateway, opening) => {
+ const block = locationBlock(directives(configMap(gateway)), opening)
+ // Witness: this is the block that proxies to control-api.
+ expect(block).toContain('proxy_pass http://control_api_upstream;')
+ expect(count(block, CONNECT_TIMEOUT_ERROR_PAGE)).toBe(1)
+ })
+
+ it.each([WORKFLOW_APPROVAL_GATEWAY, RPC_GATEWAY])(
+ 'G1-12b: %s answers control_plane_unavailable only when no connection was made, else its own 504',
+ gateway => {
+ const block = locationBlock(directives(configMap(gateway)), CONNECT_TIMEOUT_LOCATION)
+ expect(block).toContain('default_type application/json;')
+ const branchStart = block.indexOf(CONNECT_FAILED)
+ expect(branchStart, 'the named location must test $upstream_connect_time').toBeGreaterThan(0)
+ const branchEnd = block.indexOf('\n }', branchStart)
+ expect(branchEnd).toBeGreaterThan(branchStart)
+ const connectFailed = block.slice(branchStart, branchEnd)
+ expect(connectFailed).toContain(`return 503 '{"error":"control_plane_unavailable"}';`)
+ expect(connectFailed).toContain('add_header Cache-Control "no-store, private" always;')
+ expect(connectFailed).toContain('add_header Pragma "no-cache" always;')
+ expect(connectFailed).toContain('add_header X-Content-Type-Options "nosniff" always;')
+ // A read or send timeout (connection made) keeps nginx's own 504.
+ expect(block.slice(branchEnd)).toMatch(/\n\s*return 504;/)
+ expect(count(block, /\breturn\b/g)).toBe(2)
+ }
+ )
+
+ it.each([
+ ['Codex redeem', CODEX_REDEEM, 'codex-llm-proxy'],
+ ['Grok redeem', GROK_REDEEM, 'grok-llm-proxy'],
+ ])(
+ 'G1-12c: %s times out its connect before the proxy aborts the request',
+ (_route, opening, proxy) => {
+ const block = locationBlock(directives(configMap(RPC_GATEWAY)), opening)
+ const abortMs = proxyRedeemAbortMs(proxy)
+ // Witness: the proxy's own abort was read.
+ expect(abortMs).toBeGreaterThan(0)
+ // nginx's default (60 s) is longer, so the proxy gave up first and never
+ // saw the gateway's answer.
+ expect(connectTimeoutMs(block)).toBeLessThan(abortMs)
+ }
+ )
+
+ it('G1-12c: authorize inherits the approval gateway connect timeout', () => {
+ const approval = directives(configMap(WORKFLOW_APPROVAL_GATEWAY))
+ expect(connectTimeoutMs(approval)).toBe(5000)
+ const block = locationBlock(approval, AUTHORIZE)
+ // Witness: the authorize block was read.
+ expect(block).toContain('proxy_pass http://control_api_upstream;')
+ expect(block).not.toMatch(/\bproxy_connect_timeout\b/)
+ })
+
+ it.each([
+ ['Codex finalize', CODEX_FINALIZE],
+ ['Grok finalize', GROK_FINALIZE],
+ ])('G1-12d: %s keeps nginx default timeouts', (_route, opening) => {
+ const block = locationBlock(directives(configMap(RPC_GATEWAY)), opening)
+ // Witness: the finalize block exists and proxies to control-api.
+ expect(block).toContain('proxy_pass http://control_api_upstream;')
+ expect(block).not.toMatch(/\bproxy_connect_timeout\b/)
+ })
+
+ it('G1-12e: the 504 mapping stays on the three user-facing hops', () => {
+ const approval = directives(configMap(WORKFLOW_APPROVAL_GATEWAY))
+ const rpc = directives(configMap(RPC_GATEWAY))
+ // Every error_page that covers a 504, at location or server level and in
+ // any spelling, is one of the mapped hops.
+ expect(count(approval, CONNECT_TIMEOUT_ERROR_PAGE)).toBe(1)
+ expect(count(approval, ANY_504_ERROR_PAGE)).toBe(1)
+ expect(count(rpc, CONNECT_TIMEOUT_ERROR_PAGE)).toBe(2)
+ expect(count(rpc, ANY_504_ERROR_PAGE)).toBe(2)
+ for (const gateway of [approval, rpc]) {
+ expect(gateway).toContain(CONNECT_TIMEOUT_LOCATION)
+ }
+ })
+})