Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
test(control-api): map a gateway connect-timeout 504 as control_plane_unavailable (#820)
Alfredo Lopez199f18bba67c
diff --git a/control-api/package-lock.json b/control-api/package-lock.jsonindex c0d3f00d5..efe682948 100644--- a/control-api/package-lock.json+++ b/control-api/package-lock.json@@ -1,12 +1,12 @@{"name": "clerum-control-api",- "version": "0.1.636",+ "version": "0.1.637","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "clerum-control-api",- "version": "0.1.636",+ "version": "0.1.637","dependencies": {"@clerum/display-field": "file:../packages/display-field","@clerum/grok-provider-attempt-contract": "file:../packages/grok-provider-attempt-contract",diff --git a/control-api/package.json b/control-api/package.jsonindex 7bab7053e..52a7ddeb8 100644--- a/control-api/package.json+++ b/control-api/package.json@@ -1,6 +1,6 @@{"name": "clerum-control-api",- "version": "0.1.636",+ "version": "0.1.637","description": "Internal Clerum control plane REST API + UI for managing CRDs and secrets","main": "dist/main.js","engines": {diff --git a/control-api/test/deploy.controlPlaneUnavailableGateway.test.ts b/control-api/test/deploy.controlPlaneUnavailableGateway.test.tsindex ecb27922e..1dd1bc8c3 100644--- a/control-api/test/deploy.controlPlaneUnavailableGateway.test.ts+++ b/control-api/test/deploy.controlPlaneUnavailableGateway.test.ts@@ -6,14 +6,38 @@ import { readFileSync } from 'node:fs'// invalid header; `proxy_intercept_errors` is off, so a 502 control-api returns// passes through). In the last two cases control-api may already have processed// the request. On the hops whose failure reaches the user, that 502 is answered-// as JSON `control_plane_unavailable`. A 504 is not mapped: control-api may be-// alive and slow.+// as JSON `control_plane_unavailable`.+//+// G1-12 (#820): nginx answers 504 both for a connect timeout and for a read or+// send timeout. Only a connect timeout leaves `$upstream_connect_time` at "-"+// (nginx assigns it in `ngx_http_upstream_send_request`, which a connect+// timeout never reaches), so on the same hops only that 504 is mapped. A+// read-timeout 504 keeps nginx's own answer: control-api may be alive and slow.const CONFIG = readFileSync(new URL('../../deploy/base/control-plane/configmaps.yaml', import.meta.url),'utf-8')+/** `CONTROL_API_REQUEST_TIMEOUT_MS` of one proxy: how long it waits for redeem. */+function proxyRedeemAbortMs(proxy: string): number {+ const source = readFileSync(+ new URL(`../../${proxy}/src/controlApiClient.ts`, import.meta.url),+ 'utf-8'+ )+ const match = source.match(/export const CONTROL_API_REQUEST_TIMEOUT_MS = ([0-9_]+)\n/)+ expect(match, `${proxy} must export CONTROL_API_REQUEST_TIMEOUT_MS as a literal`).not.toBeNull()+ return Number(match![1].replace(/_/g, ''))+}++/** The one `proxy_connect_timeout` in a block, in milliseconds (`Ns` or `Nms`). */+function connectTimeoutMs(block: string): number {+ const matches = [...block.matchAll(/\bproxy_connect_timeout\s+([0-9]+)(ms|s)\s*;/g)]+ expect(matches, 'exactly one proxy_connect_timeout').toHaveLength(1)+ const [, value, unit] = matches[0]+ return unit === 's' ? Number(value) * 1000 : Number(value)+}+// Any spelling nginx accepts: `error_page 502 = @x;`, `error_page 502 =@x;`, …const ERROR_PAGE = /error_page\s+502\s*=\s*@control_plane_unavailable\s*;/g// Any error_page that covers a 502, whatever it points to.@@ -93,7 +117,7 @@ describe('gateway-generated 502 on user-facing control-plane hops (G1-10, #720)'expect(block).not.toContain('error_page')})- it('G1-10d: the mapping stays on the three user-facing hops and never covers a 504', () => {+ it('G1-10d: the 502 mapping stays on the three user-facing hops', () => {const approval = directives(configMap(WORKFLOW_APPROVAL_GATEWAY))const rpc = directives(configMap(RPC_GATEWAY))// Every error_page that covers a 502, at location or server level and in@@ -104,7 +128,6 @@ describe('gateway-generated 502 on user-facing control-plane hops (G1-10, #720)'expect(count(rpc, ANY_502_ERROR_PAGE)).toBe(2)for (const gateway of [approval, rpc]) {expect(gateway).toContain(NAMED_LOCATION)- expect(gateway).not.toMatch(/error_page[^;]*\b504\b/)}})@@ -120,3 +143,92 @@ describe('gateway-generated 502 on user-facing control-plane hops (G1-10, #720)'})})++// Any spelling nginx accepts, as for the 502.+const CONNECT_TIMEOUT_ERROR_PAGE = /error_page\s+504\s*=\s*@control_plane_connect_timeout\s*;/g+// Any error_page that covers a 504, whatever it points to.+const ANY_504_ERROR_PAGE = /error_page[^;]*\b504\b[^;]*;/g+const CONNECT_TIMEOUT_LOCATION = 'location @control_plane_connect_timeout {'+const CONNECT_FAILED = 'if ($upstream_connect_time = "-") {'++describe('gateway-generated connect-timeout 504 on user-facing control-plane hops (G1-12, #820)', () => {+ it.each([+ ['H2 authorize', WORKFLOW_APPROVAL_GATEWAY, AUTHORIZE],+ ['H3 Codex redeem', RPC_GATEWAY, CODEX_REDEEM],+ ['H3 Grok redeem', RPC_GATEWAY, GROK_REDEEM],+ ])('G1-12a: %s sends its own 504 to the connect-timeout check', (_hop, gateway, opening) => {+ const block = locationBlock(directives(configMap(gateway)), opening)+ // Witness: this is the block that proxies to control-api.+ expect(block).toContain('proxy_pass http://control_api_upstream;')+ expect(count(block, CONNECT_TIMEOUT_ERROR_PAGE)).toBe(1)+ })++ it.each([WORKFLOW_APPROVAL_GATEWAY, RPC_GATEWAY])(+ 'G1-12b: %s answers control_plane_unavailable only when no connection was made, else its own 504',+ gateway => {+ const block = locationBlock(directives(configMap(gateway)), CONNECT_TIMEOUT_LOCATION)+ expect(block).toContain('default_type application/json;')+ const branchStart = block.indexOf(CONNECT_FAILED)+ expect(branchStart, 'the named location must test $upstream_connect_time').toBeGreaterThan(0)+ const branchEnd = block.indexOf('\n }', branchStart)+ expect(branchEnd).toBeGreaterThan(branchStart)+ const connectFailed = block.slice(branchStart, branchEnd)+ expect(connectFailed).toContain(`return 503 '{"error":"control_plane_unavailable"}';`)+ expect(connectFailed).toContain('add_header Cache-Control "no-store, private" always;')+ expect(connectFailed).toContain('add_header Pragma "no-cache" always;')+ expect(connectFailed).toContain('add_header X-Content-Type-Options "nosniff" always;')+ // A read or send timeout (connection made) keeps nginx's own 504.+ expect(block.slice(branchEnd)).toMatch(/\n\s*return 504;/)+ expect(count(block, /\breturn\b/g)).toBe(2)+ }+ )++ it.each([+ ['Codex redeem', CODEX_REDEEM, 'codex-llm-proxy'],+ ['Grok redeem', GROK_REDEEM, 'grok-llm-proxy'],+ ])(+ 'G1-12c: %s times out its connect before the proxy aborts the request',+ (_route, opening, proxy) => {+ const block = locationBlock(directives(configMap(RPC_GATEWAY)), opening)+ const abortMs = proxyRedeemAbortMs(proxy)+ // Witness: the proxy's own abort was read.+ expect(abortMs).toBeGreaterThan(0)+ // nginx's default (60 s) is longer, so the proxy gave up first and never+ // saw the gateway's answer.+ expect(connectTimeoutMs(block)).toBeLessThan(abortMs)+ }+ )++ it('G1-12c: authorize inherits the approval gateway connect timeout', () => {+ const approval = directives(configMap(WORKFLOW_APPROVAL_GATEWAY))+ expect(connectTimeoutMs(approval)).toBe(5000)+ const block = locationBlock(approval, AUTHORIZE)+ // Witness: the authorize block was read.+ expect(block).toContain('proxy_pass http://control_api_upstream;')+ expect(block).not.toMatch(/\bproxy_connect_timeout\b/)+ })++ it.each([+ ['Codex finalize', CODEX_FINALIZE],+ ['Grok finalize', GROK_FINALIZE],+ ])('G1-12d: %s keeps nginx default timeouts', (_route, opening) => {+ const block = locationBlock(directives(configMap(RPC_GATEWAY)), opening)+ // Witness: the finalize block exists and proxies to control-api.+ expect(block).toContain('proxy_pass http://control_api_upstream;')+ expect(block).not.toMatch(/\bproxy_connect_timeout\b/)+ })++ it('G1-12e: the 504 mapping stays on the three user-facing hops', () => {+ const approval = directives(configMap(WORKFLOW_APPROVAL_GATEWAY))+ const rpc = directives(configMap(RPC_GATEWAY))+ // Every error_page that covers a 504, at location or server level and in+ // any spelling, is one of the mapped hops.+ expect(count(approval, CONNECT_TIMEOUT_ERROR_PAGE)).toBe(1)+ expect(count(approval, ANY_504_ERROR_PAGE)).toBe(1)+ expect(count(rpc, CONNECT_TIMEOUT_ERROR_PAGE)).toBe(2)+ expect(count(rpc, ANY_504_ERROR_PAGE)).toBe(2)+ for (const gateway of [approval, rpc]) {+ expect(gateway).toContain(CONNECT_TIMEOUT_LOCATION)+ }+ })+})