Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

fix(llm-proxy): report a redeem that cannot connect as control_plane_unavailable

Alfredo Lopez8331cd73880f
diff --git a/codex-llm-proxy/src/controlApiClient.ts b/codex-llm-proxy/src/controlApiClient.ts
index a6150ada2..394a51e92 100644
--- a/codex-llm-proxy/src/controlApiClient.ts
+++ b/codex-llm-proxy/src/controlApiClient.ts
@@ -42,13 +42,45 @@ export type ControlApiClientConfig = {
export class ControlApiClientError extends Error {
constructor(
readonly code: string,
- message: string
+ message: string,
+ // The fetch failure's cause code, for the attempt log (G1-4).
+ readonly causeCode?: string
) {
super(message)
this.name = 'ControlApiClientError'
}
}
+// G1-4 (#720): fetch failure codes that prove no control-plane process got
+// the request: nothing accepted the connection, the name did not resolve, no
+// route existed, or the TCP connect timed out. A reset, or anything after the
+// request timeout fired, may have reached a live process and stays ambiguous.
+// The Grok proxy carries the same set; the proxies share no module (#799).
+const CONNECT_PHASE_CODES: ReadonlySet<string> = new Set([
+ 'ECONNREFUSED',
+ 'ENOTFOUND',
+ 'EAI_AGAIN',
+ 'EHOSTUNREACH',
+ 'ENETUNREACH',
+ 'UND_ERR_CONNECT_TIMEOUT',
+])
+
+// The code undici puts on a failed fetch's cause. Only a code-shaped string is
+// returned, so nothing else from the error can reach a log.
+export function fetchCauseCode(err: unknown): string | undefined {
+ if (!(err instanceof Error)) return undefined
+ const cause: unknown = (err as { cause?: unknown }).cause
+ if (typeof cause !== 'object' || cause === null) return undefined
+ const code: unknown = (cause as { code?: unknown }).code
+ return typeof code === 'string' && /^[A-Z][A-Z0-9_]{0,63}$/.test(code) ? code : undefined
+}
+
+function isConnectPhaseFailure(err: unknown, signal: AbortSignal): boolean {
+ if (signal.aborted || !(err instanceof TypeError)) return false
+ const code = fetchCauseCode(err)
+ return code !== undefined && CONNECT_PHASE_CODES.has(code)
+}
+
export class ControlApiClient {
constructor(private readonly config: ControlApiClientConfig) {}
@@ -96,16 +128,32 @@ export class ControlApiClient {
private async post(path: string, payload: Record<string, unknown>): Promise<unknown> {
const base = this.config.baseUrl.replace(/\/+$/, '')
const fetchFn = this.config.fetchFn ?? fetch
- const response = await fetchFn(`${base}${path.startsWith('/') ? path : `/${path}`}`, {
- method: 'POST',
- headers: {
- 'content-type': 'application/json',
- authorization: `Bearer ${this.config.serviceToken}`,
- 'x-service-token': this.config.serviceName,
- },
- body: JSON.stringify(payload),
- signal: AbortSignal.timeout(CONTROL_API_REQUEST_TIMEOUT_MS),
- })
+ const signal = AbortSignal.timeout(CONTROL_API_REQUEST_TIMEOUT_MS)
+ let response: Response
+ try {
+ response = await fetchFn(`${base}${path.startsWith('/') ? path : `/${path}`}`, {
+ method: 'POST',
+ headers: {
+ 'content-type': 'application/json',
+ authorization: `Bearer ${this.config.serviceToken}`,
+ 'x-service-token': this.config.serviceName,
+ },
+ body: JSON.stringify(payload),
+ signal,
+ })
+ } catch (err) {
+ if (!isConnectPhaseFailure(err, signal)) throw err
+ const causeCode = fetchCauseCode(err)
+ logger.warn(
+ { event: 'codex_proxy_control_api_unreachable', causeCode, path },
+ 'control API is unreachable'
+ )
+ throw new ControlApiClientError(
+ 'control_plane_unavailable',
+ 'control API is unreachable',
+ causeCode
+ )
+ }
const raw = await response.text()
let parsed: unknown = null
if (raw) {
diff --git a/codex-llm-proxy/src/server.ts b/codex-llm-proxy/src/server.ts
index 3829af588..efe47ceb0 100644
--- a/codex-llm-proxy/src/server.ts
+++ b/codex-llm-proxy/src/server.ts
@@ -23,7 +23,7 @@ import {
UpstreamTimeoutError,
} from './codexTransport.js'
import type { CodexLlmProxyConfig } from './config.js'
-import { ControlApiClient, ControlApiClientError } from './controlApiClient.js'
+import { ControlApiClient, ControlApiClientError, fetchCauseCode } from './controlApiClient.js'
import { logger } from './logger.js'
import { createProxyMetrics } from './metrics.js'
import {
@@ -597,6 +597,8 @@ export function createProxyApps(
)
if (err instanceof UpstreamTimeoutError) metrics.observeUpstreamTimeout(err.kind)
const deliveredAs = res.headersSent ? 'sse_error' : 'http_status'
+ const causeCode =
+ err instanceof ControlApiClientError ? err.causeCode : fetchCauseCode(err)
logger.warn(
{
event: 'codex_proxy_attempt_finished',
@@ -610,6 +612,8 @@ export function createProxyApps(
: {}),
// RequestLimitError messages are fixed strings with no request data.
...(err instanceof RequestLimitError ? { reason: err.message } : {}),
+ // G1-4: a failed fetch's cause code only, never its message or URL.
+ ...(causeCode ? { causeCode } : {}),
deliveredAs,
...(deliveredAs === 'http_status' ? { httpStatus: mapped.status } : {}),
toolCalls,
@@ -794,6 +798,8 @@ const ATTEMPT_ERROR_STATUS: Record<string, number> = {
tool_call_limit_exceeded: 422,
// An upstream 4xx the same request would get again (G1-3, #720).
upstream_rejected: 502,
+ // A control-api call no live process received (G1-4, #720).
+ control_plane_unavailable: 503,
invalid_tool_arguments: 422,
// The attempt ran for its whole stream budget. Retrying the same request
// would spend the same budget again, so it is a gateway timeout, not 503.
diff --git a/grok-llm-proxy/src/controlApiClient.ts b/grok-llm-proxy/src/controlApiClient.ts
index ae87b0b4b..97e40220b 100644
--- a/grok-llm-proxy/src/controlApiClient.ts
+++ b/grok-llm-proxy/src/controlApiClient.ts
@@ -42,13 +42,45 @@ export type ControlApiClientConfig = {
export class ControlApiClientError extends Error {
constructor(
readonly code: string,
- message: string
+ message: string,
+ // The fetch failure's cause code, for the attempt log (G1-4).
+ readonly causeCode?: string
) {
super(message)
this.name = 'ControlApiClientError'
}
}
+// G1-4 (#720): fetch failure codes that prove no control-plane process got
+// the request: nothing accepted the connection, the name did not resolve, no
+// route existed, or the TCP connect timed out. A reset, or anything after the
+// request timeout fired, may have reached a live process and stays ambiguous.
+// The Codex proxy carries the same set; the proxies share no module (#799).
+const CONNECT_PHASE_CODES: ReadonlySet<string> = new Set([
+ 'ECONNREFUSED',
+ 'ENOTFOUND',
+ 'EAI_AGAIN',
+ 'EHOSTUNREACH',
+ 'ENETUNREACH',
+ 'UND_ERR_CONNECT_TIMEOUT',
+])
+
+// The code undici puts on a failed fetch's cause. Only a code-shaped string is
+// returned, so nothing else from the error can reach a log.
+export function fetchCauseCode(err: unknown): string | undefined {
+ if (!(err instanceof Error)) return undefined
+ const cause: unknown = (err as { cause?: unknown }).cause
+ if (typeof cause !== 'object' || cause === null) return undefined
+ const code: unknown = (cause as { code?: unknown }).code
+ return typeof code === 'string' && /^[A-Z][A-Z0-9_]{0,63}$/.test(code) ? code : undefined
+}
+
+function isConnectPhaseFailure(err: unknown, signal: AbortSignal): boolean {
+ if (signal.aborted || !(err instanceof TypeError)) return false
+ const code = fetchCauseCode(err)
+ return code !== undefined && CONNECT_PHASE_CODES.has(code)
+}
+
export class ControlApiClient {
constructor(private readonly config: ControlApiClientConfig) {}
@@ -100,16 +132,32 @@ export class ControlApiClient {
private async post(path: string, payload: Record<string, unknown>): Promise<unknown> {
const base = this.config.baseUrl.replace(/\/+$/, '')
const fetchFn = this.config.fetchFn ?? fetch
- const response = await fetchFn(`${base}${path.startsWith('/') ? path : `/${path}`}`, {
- method: 'POST',
- headers: {
- 'content-type': 'application/json',
- authorization: `Bearer ${this.config.serviceToken}`,
- 'x-service-token': this.config.serviceName,
- },
- body: JSON.stringify(payload),
- signal: AbortSignal.timeout(CONTROL_API_REQUEST_TIMEOUT_MS),
- })
+ const signal = AbortSignal.timeout(CONTROL_API_REQUEST_TIMEOUT_MS)
+ let response: Response
+ try {
+ response = await fetchFn(`${base}${path.startsWith('/') ? path : `/${path}`}`, {
+ method: 'POST',
+ headers: {
+ 'content-type': 'application/json',
+ authorization: `Bearer ${this.config.serviceToken}`,
+ 'x-service-token': this.config.serviceName,
+ },
+ body: JSON.stringify(payload),
+ signal,
+ })
+ } catch (err) {
+ if (!isConnectPhaseFailure(err, signal)) throw err
+ const causeCode = fetchCauseCode(err)
+ logger.warn(
+ { event: 'grok_proxy_control_api_unreachable', causeCode, path },
+ 'control API is unreachable'
+ )
+ throw new ControlApiClientError(
+ 'control_plane_unavailable',
+ 'control API is unreachable',
+ causeCode
+ )
+ }
const raw = await response.text()
let parsed: unknown = null
if (raw) {
diff --git a/grok-llm-proxy/src/server.ts b/grok-llm-proxy/src/server.ts
index 9c881d28e..86e7ccad4 100644
--- a/grok-llm-proxy/src/server.ts
+++ b/grok-llm-proxy/src/server.ts
@@ -16,7 +16,7 @@ import {
} from './auth/executionTicketVerifier.js'
import { type PlatformJwtClaims, verifyPlatformJwt } from './auth/platformJwtVerifier.js'
import type { GrokLlmProxyConfig } from './config.js'
-import { ControlApiClient, ControlApiClientError } from './controlApiClient.js'
+import { ControlApiClient, ControlApiClientError, fetchCauseCode } from './controlApiClient.js'
import {
GrokTransportError,
listGrokModels,
@@ -470,6 +470,8 @@ export function createProxyApps(
)
if (err instanceof UpstreamTimeoutError) metrics.observeUpstreamTimeout(err.kind)
const deliveredAs = res.headersSent ? 'sse_error' : 'http_status'
+ const causeCode =
+ err instanceof ControlApiClientError ? err.causeCode : fetchCauseCode(err)
logger.warn(
{
event: 'grok_proxy_attempt_finished',
@@ -483,6 +485,8 @@ export function createProxyApps(
: {}),
// RequestLimitError messages are fixed strings with no request data.
...(err instanceof RequestLimitError ? { reason: err.message } : {}),
+ // G1-4: a failed fetch's cause code only, never its message or URL.
+ ...(causeCode ? { causeCode } : {}),
deliveredAs,
...(deliveredAs === 'http_status' ? { httpStatus: mapped.status } : {}),
toolCalls,
@@ -667,6 +671,8 @@ const ATTEMPT_ERROR_STATUS: Record<string, number> = {
tool_call_limit_exceeded: 422,
// An upstream 4xx the same request would get again (G1-3, #720).
upstream_rejected: 502,
+ // A control-api call no live process received (G1-4, #720).
+ control_plane_unavailable: 503,
tool_call_arguments_exceeded: 422,
invalid_tool_arguments: 422,
client_upgrade_required: 426,
diff --git a/tests/e2e/fixtures/codex-subscription/sanitized-upstream-contract.json b/tests/e2e/fixtures/codex-subscription/sanitized-upstream-contract.json
index d30f1f1d5..1bea6b6e4 100644
--- a/tests/e2e/fixtures/codex-subscription/sanitized-upstream-contract.json
+++ b/tests/e2e/fixtures/codex-subscription/sanitized-upstream-contract.json
@@ -63,6 +63,7 @@
"provider_unavailable",
"rate_limited",
"upstream_rejected",
+ "control_plane_unavailable",
"origin_denied",
"ticket_invalid",
"ticket_replayed",
diff --git a/tests/e2e/fixtures/grok-subscription/sanitized-upstream-contract.json b/tests/e2e/fixtures/grok-subscription/sanitized-upstream-contract.json
index 29dfdb144..bf373a3c7 100644
--- a/tests/e2e/fixtures/grok-subscription/sanitized-upstream-contract.json
+++ b/tests/e2e/fixtures/grok-subscription/sanitized-upstream-contract.json
@@ -77,6 +77,7 @@
"provider_unavailable",
"rate_limited",
"upstream_rejected",
+ "control_plane_unavailable",
"origin_denied",
"ticket_invalid",
"ticket_replayed",
diff --git a/tests/e2e/fixtures/package-lock.json b/tests/e2e/fixtures/package-lock.json
index 173cd6dc8..8c88caac1 100644
--- a/tests/e2e/fixtures/package-lock.json
+++ b/tests/e2e/fixtures/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "@clerum/e2e-test-fixtures",
- "version": "1.0.99",
+ "version": "1.0.100",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "@clerum/e2e-test-fixtures",
- "version": "1.0.99",
+ "version": "1.0.100",
"dependencies": {
"@kubernetes/client-node": "^1.0.0"
},
diff --git a/tests/e2e/fixtures/package.json b/tests/e2e/fixtures/package.json
index 532ff75e9..b80adc934 100644
--- a/tests/e2e/fixtures/package.json
+++ b/tests/e2e/fixtures/package.json
@@ -1,6 +1,6 @@
{
"name": "@clerum/e2e-test-fixtures",
- "version": "1.0.99",
+ "version": "1.0.100",
"description": "E2E test fixtures and utilities for Clerum testing",
"type": "module",
"main": "index.ts",