Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
fix(llm-proxy): report a redeem that cannot connect as control_plane_unavailable
Alfredo Lopez8331cd73880f
diff --git a/codex-llm-proxy/src/controlApiClient.ts b/codex-llm-proxy/src/controlApiClient.tsindex a6150ada2..394a51e92 100644--- a/codex-llm-proxy/src/controlApiClient.ts+++ b/codex-llm-proxy/src/controlApiClient.ts@@ -42,13 +42,45 @@ export type ControlApiClientConfig = {export class ControlApiClientError extends Error {constructor(readonly code: string,- message: string+ message: string,+ // The fetch failure's cause code, for the attempt log (G1-4).+ readonly causeCode?: string) {super(message)this.name = 'ControlApiClientError'}}+// G1-4 (#720): fetch failure codes that prove no control-plane process got+// the request: nothing accepted the connection, the name did not resolve, no+// route existed, or the TCP connect timed out. A reset, or anything after the+// request timeout fired, may have reached a live process and stays ambiguous.+// The Grok proxy carries the same set; the proxies share no module (#799).+const CONNECT_PHASE_CODES: ReadonlySet<string> = new Set([+ 'ECONNREFUSED',+ 'ENOTFOUND',+ 'EAI_AGAIN',+ 'EHOSTUNREACH',+ 'ENETUNREACH',+ 'UND_ERR_CONNECT_TIMEOUT',+])++// The code undici puts on a failed fetch's cause. Only a code-shaped string is+// returned, so nothing else from the error can reach a log.+export function fetchCauseCode(err: unknown): string | undefined {+ if (!(err instanceof Error)) return undefined+ const cause: unknown = (err as { cause?: unknown }).cause+ if (typeof cause !== 'object' || cause === null) return undefined+ const code: unknown = (cause as { code?: unknown }).code+ return typeof code === 'string' && /^[A-Z][A-Z0-9_]{0,63}$/.test(code) ? code : undefined+}++function isConnectPhaseFailure(err: unknown, signal: AbortSignal): boolean {+ if (signal.aborted || !(err instanceof TypeError)) return false+ const code = fetchCauseCode(err)+ return code !== undefined && CONNECT_PHASE_CODES.has(code)+}+export class ControlApiClient {constructor(private readonly config: ControlApiClientConfig) {}@@ -96,16 +128,32 @@ export class ControlApiClient {private async post(path: string, payload: Record<string, unknown>): Promise<unknown> {const base = this.config.baseUrl.replace(/\/+$/, '')const fetchFn = this.config.fetchFn ?? fetch- const response = await fetchFn(`${base}${path.startsWith('/') ? path : `/${path}`}`, {- method: 'POST',- headers: {- 'content-type': 'application/json',- authorization: `Bearer ${this.config.serviceToken}`,- 'x-service-token': this.config.serviceName,- },- body: JSON.stringify(payload),- signal: AbortSignal.timeout(CONTROL_API_REQUEST_TIMEOUT_MS),- })+ const signal = AbortSignal.timeout(CONTROL_API_REQUEST_TIMEOUT_MS)+ let response: Response+ try {+ response = await fetchFn(`${base}${path.startsWith('/') ? path : `/${path}`}`, {+ method: 'POST',+ headers: {+ 'content-type': 'application/json',+ authorization: `Bearer ${this.config.serviceToken}`,+ 'x-service-token': this.config.serviceName,+ },+ body: JSON.stringify(payload),+ signal,+ })+ } catch (err) {+ if (!isConnectPhaseFailure(err, signal)) throw err+ const causeCode = fetchCauseCode(err)+ logger.warn(+ { event: 'codex_proxy_control_api_unreachable', causeCode, path },+ 'control API is unreachable'+ )+ throw new ControlApiClientError(+ 'control_plane_unavailable',+ 'control API is unreachable',+ causeCode+ )+ }const raw = await response.text()let parsed: unknown = nullif (raw) {diff --git a/codex-llm-proxy/src/server.ts b/codex-llm-proxy/src/server.tsindex 3829af588..efe47ceb0 100644--- a/codex-llm-proxy/src/server.ts+++ b/codex-llm-proxy/src/server.ts@@ -23,7 +23,7 @@ import {UpstreamTimeoutError,} from './codexTransport.js'import type { CodexLlmProxyConfig } from './config.js'-import { ControlApiClient, ControlApiClientError } from './controlApiClient.js'+import { ControlApiClient, ControlApiClientError, fetchCauseCode } from './controlApiClient.js'import { logger } from './logger.js'import { createProxyMetrics } from './metrics.js'import {@@ -597,6 +597,8 @@ export function createProxyApps()if (err instanceof UpstreamTimeoutError) metrics.observeUpstreamTimeout(err.kind)const deliveredAs = res.headersSent ? 'sse_error' : 'http_status'+ const causeCode =+ err instanceof ControlApiClientError ? err.causeCode : fetchCauseCode(err)logger.warn({event: 'codex_proxy_attempt_finished',@@ -610,6 +612,8 @@ export function createProxyApps(: {}),// RequestLimitError messages are fixed strings with no request data....(err instanceof RequestLimitError ? { reason: err.message } : {}),+ // G1-4: a failed fetch's cause code only, never its message or URL.+ ...(causeCode ? { causeCode } : {}),deliveredAs,...(deliveredAs === 'http_status' ? { httpStatus: mapped.status } : {}),toolCalls,@@ -794,6 +798,8 @@ const ATTEMPT_ERROR_STATUS: Record<string, number> = {tool_call_limit_exceeded: 422,// An upstream 4xx the same request would get again (G1-3, #720).upstream_rejected: 502,+ // A control-api call no live process received (G1-4, #720).+ control_plane_unavailable: 503,invalid_tool_arguments: 422,// The attempt ran for its whole stream budget. Retrying the same request// would spend the same budget again, so it is a gateway timeout, not 503.diff --git a/grok-llm-proxy/src/controlApiClient.ts b/grok-llm-proxy/src/controlApiClient.tsindex ae87b0b4b..97e40220b 100644--- a/grok-llm-proxy/src/controlApiClient.ts+++ b/grok-llm-proxy/src/controlApiClient.ts@@ -42,13 +42,45 @@ export type ControlApiClientConfig = {export class ControlApiClientError extends Error {constructor(readonly code: string,- message: string+ message: string,+ // The fetch failure's cause code, for the attempt log (G1-4).+ readonly causeCode?: string) {super(message)this.name = 'ControlApiClientError'}}+// G1-4 (#720): fetch failure codes that prove no control-plane process got+// the request: nothing accepted the connection, the name did not resolve, no+// route existed, or the TCP connect timed out. A reset, or anything after the+// request timeout fired, may have reached a live process and stays ambiguous.+// The Codex proxy carries the same set; the proxies share no module (#799).+const CONNECT_PHASE_CODES: ReadonlySet<string> = new Set([+ 'ECONNREFUSED',+ 'ENOTFOUND',+ 'EAI_AGAIN',+ 'EHOSTUNREACH',+ 'ENETUNREACH',+ 'UND_ERR_CONNECT_TIMEOUT',+])++// The code undici puts on a failed fetch's cause. Only a code-shaped string is+// returned, so nothing else from the error can reach a log.+export function fetchCauseCode(err: unknown): string | undefined {+ if (!(err instanceof Error)) return undefined+ const cause: unknown = (err as { cause?: unknown }).cause+ if (typeof cause !== 'object' || cause === null) return undefined+ const code: unknown = (cause as { code?: unknown }).code+ return typeof code === 'string' && /^[A-Z][A-Z0-9_]{0,63}$/.test(code) ? code : undefined+}++function isConnectPhaseFailure(err: unknown, signal: AbortSignal): boolean {+ if (signal.aborted || !(err instanceof TypeError)) return false+ const code = fetchCauseCode(err)+ return code !== undefined && CONNECT_PHASE_CODES.has(code)+}+export class ControlApiClient {constructor(private readonly config: ControlApiClientConfig) {}@@ -100,16 +132,32 @@ export class ControlApiClient {private async post(path: string, payload: Record<string, unknown>): Promise<unknown> {const base = this.config.baseUrl.replace(/\/+$/, '')const fetchFn = this.config.fetchFn ?? fetch- const response = await fetchFn(`${base}${path.startsWith('/') ? path : `/${path}`}`, {- method: 'POST',- headers: {- 'content-type': 'application/json',- authorization: `Bearer ${this.config.serviceToken}`,- 'x-service-token': this.config.serviceName,- },- body: JSON.stringify(payload),- signal: AbortSignal.timeout(CONTROL_API_REQUEST_TIMEOUT_MS),- })+ const signal = AbortSignal.timeout(CONTROL_API_REQUEST_TIMEOUT_MS)+ let response: Response+ try {+ response = await fetchFn(`${base}${path.startsWith('/') ? path : `/${path}`}`, {+ method: 'POST',+ headers: {+ 'content-type': 'application/json',+ authorization: `Bearer ${this.config.serviceToken}`,+ 'x-service-token': this.config.serviceName,+ },+ body: JSON.stringify(payload),+ signal,+ })+ } catch (err) {+ if (!isConnectPhaseFailure(err, signal)) throw err+ const causeCode = fetchCauseCode(err)+ logger.warn(+ { event: 'grok_proxy_control_api_unreachable', causeCode, path },+ 'control API is unreachable'+ )+ throw new ControlApiClientError(+ 'control_plane_unavailable',+ 'control API is unreachable',+ causeCode+ )+ }const raw = await response.text()let parsed: unknown = nullif (raw) {diff --git a/grok-llm-proxy/src/server.ts b/grok-llm-proxy/src/server.tsindex 9c881d28e..86e7ccad4 100644--- a/grok-llm-proxy/src/server.ts+++ b/grok-llm-proxy/src/server.ts@@ -16,7 +16,7 @@ import {} from './auth/executionTicketVerifier.js'import { type PlatformJwtClaims, verifyPlatformJwt } from './auth/platformJwtVerifier.js'import type { GrokLlmProxyConfig } from './config.js'-import { ControlApiClient, ControlApiClientError } from './controlApiClient.js'+import { ControlApiClient, ControlApiClientError, fetchCauseCode } from './controlApiClient.js'import {GrokTransportError,listGrokModels,@@ -470,6 +470,8 @@ export function createProxyApps()if (err instanceof UpstreamTimeoutError) metrics.observeUpstreamTimeout(err.kind)const deliveredAs = res.headersSent ? 'sse_error' : 'http_status'+ const causeCode =+ err instanceof ControlApiClientError ? err.causeCode : fetchCauseCode(err)logger.warn({event: 'grok_proxy_attempt_finished',@@ -483,6 +485,8 @@ export function createProxyApps(: {}),// RequestLimitError messages are fixed strings with no request data....(err instanceof RequestLimitError ? { reason: err.message } : {}),+ // G1-4: a failed fetch's cause code only, never its message or URL.+ ...(causeCode ? { causeCode } : {}),deliveredAs,...(deliveredAs === 'http_status' ? { httpStatus: mapped.status } : {}),toolCalls,@@ -667,6 +671,8 @@ const ATTEMPT_ERROR_STATUS: Record<string, number> = {tool_call_limit_exceeded: 422,// An upstream 4xx the same request would get again (G1-3, #720).upstream_rejected: 502,+ // A control-api call no live process received (G1-4, #720).+ control_plane_unavailable: 503,tool_call_arguments_exceeded: 422,invalid_tool_arguments: 422,client_upgrade_required: 426,diff --git a/tests/e2e/fixtures/codex-subscription/sanitized-upstream-contract.json b/tests/e2e/fixtures/codex-subscription/sanitized-upstream-contract.jsonindex d30f1f1d5..1bea6b6e4 100644--- a/tests/e2e/fixtures/codex-subscription/sanitized-upstream-contract.json+++ b/tests/e2e/fixtures/codex-subscription/sanitized-upstream-contract.json@@ -63,6 +63,7 @@"provider_unavailable","rate_limited","upstream_rejected",+ "control_plane_unavailable","origin_denied","ticket_invalid","ticket_replayed",diff --git a/tests/e2e/fixtures/grok-subscription/sanitized-upstream-contract.json b/tests/e2e/fixtures/grok-subscription/sanitized-upstream-contract.jsonindex 29dfdb144..bf373a3c7 100644--- a/tests/e2e/fixtures/grok-subscription/sanitized-upstream-contract.json+++ b/tests/e2e/fixtures/grok-subscription/sanitized-upstream-contract.json@@ -77,6 +77,7 @@"provider_unavailable","rate_limited","upstream_rejected",+ "control_plane_unavailable","origin_denied","ticket_invalid","ticket_replayed",diff --git a/tests/e2e/fixtures/package-lock.json b/tests/e2e/fixtures/package-lock.jsonindex 173cd6dc8..8c88caac1 100644--- a/tests/e2e/fixtures/package-lock.json+++ b/tests/e2e/fixtures/package-lock.json@@ -1,12 +1,12 @@{"name": "@clerum/e2e-test-fixtures",- "version": "1.0.99",+ "version": "1.0.100","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "@clerum/e2e-test-fixtures",- "version": "1.0.99",+ "version": "1.0.100","dependencies": {"@kubernetes/client-node": "^1.0.0"},diff --git a/tests/e2e/fixtures/package.json b/tests/e2e/fixtures/package.jsonindex 532ff75e9..b80adc934 100644--- a/tests/e2e/fixtures/package.json+++ b/tests/e2e/fixtures/package.json@@ -1,6 +1,6 @@{"name": "@clerum/e2e-test-fixtures",- "version": "1.0.99",+ "version": "1.0.100","description": "E2E test fixtures and utilities for Clerum testing","type": "module","main": "index.ts",