Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

test(llm-proxy): require a redeem that cannot connect to answer control_plane_unavailable

Alfredo Lopezcdefcaaf995c
diff --git a/codex-llm-proxy/test/controlApiClient.test.ts b/codex-llm-proxy/test/controlApiClient.test.ts
index 30f81abc9..461d64173 100644
--- a/codex-llm-proxy/test/controlApiClient.test.ts
+++ b/codex-llm-proxy/test/controlApiClient.test.ts
@@ -1,6 +1,10 @@
import { createServer, type IncomingMessage, type Server } from 'node:http'
-import { describe, expect, it } from 'vitest'
-import { ControlApiClient, ControlApiClientError } from '../src/controlApiClient.js'
+import { describe, expect, it, vi } from 'vitest'
+import {
+ CONTROL_API_REQUEST_TIMEOUT_MS,
+ ControlApiClient,
+ ControlApiClientError,
+} from '../src/controlApiClient.js'
import { CODEX_CATALOG_ORIGIN, CODEX_COMPLETIONS_ORIGIN } from '../src/originPolicy.js'
const LOOPBACK_V4 = ['127', '0', '0', '1'].join('.')
@@ -259,3 +263,126 @@ describe('ControlApiClient response hardening', () => {
}
})
})
+
+// G1-4 (#720): a redeem that never reached a live control-plane process is
+// control_plane_unavailable; a failure that may have reached one is not.
+describe('ControlApiClient control-plane reachability', () => {
+ async function closedPortUrl(): Promise<string> {
+ const server = createServer()
+ await new Promise<void>(resolve => server.listen(0, '127.0.0.1', () => resolve()))
+ const addr = server.address()
+ const port = typeof addr === 'object' && addr ? addr.port : 0
+ await new Promise<void>(resolve => server.close(() => resolve()))
+ return `http://${LOOPBACK_V4}:${port}`
+ }
+
+ function clientAt(baseUrl: string, fetchFn?: typeof fetch): ControlApiClient {
+ return new ControlApiClient({
+ baseUrl: `${baseUrl}/api/v1`,
+ serviceName: 'codex-llm-proxy',
+ serviceToken: 'dev-codex-llm-proxy-token',
+ ...(fetchFn ? { fetchFn } : {}),
+ })
+ }
+
+ const redeemInput = {
+ executionTicket: 'ticket',
+ requestHash: 'e'.repeat(64),
+ operation: 'completion_stream' as const,
+ }
+
+ // The shape undici gives a fetch that failed before any response: a
+ // TypeError whose cause carries the system or undici error code.
+ function fetchFailure(code: string): TypeError {
+ return Object.assign(new TypeError('fetch failed'), {
+ cause: Object.assign(new Error(code), { code }),
+ })
+ }
+
+ it('G1-4a maps a refused connection to control_plane_unavailable', async () => {
+ const url = await closedPortUrl()
+ // Witness: the platform fetch fails this way against the closed port, so
+ // the mocked shapes below match what undici really produces.
+ const raw = await fetch(`${url}/api/v1/probe`).catch((caught: unknown) => caught)
+ expect(raw).toBeInstanceOf(TypeError)
+ expect((raw as { cause?: { code?: unknown } }).cause?.code).toBe('ECONNREFUSED')
+ await expect(clientAt(url).redeem(redeemInput)).rejects.toMatchObject({
+ name: 'ControlApiClientError',
+ code: 'control_plane_unavailable',
+ causeCode: 'ECONNREFUSED',
+ })
+ })
+
+ it.each(['ENOTFOUND', 'EAI_AGAIN', 'EHOSTUNREACH', 'ENETUNREACH', 'UND_ERR_CONNECT_TIMEOUT'])(
+ 'G1-4b maps a %s fetch failure to control_plane_unavailable',
+ async code => {
+ const fetchFn = vi.fn(async () => {
+ throw fetchFailure(code)
+ })
+ await expect(
+ clientAt('http://control-api.invalid', fetchFn as unknown as typeof fetch).redeem(
+ redeemInput
+ )
+ ).rejects.toMatchObject({ code: 'control_plane_unavailable', causeCode: code })
+ expect(fetchFn).toHaveBeenCalledTimes(1)
+ }
+ )
+
+ it('G1-4c rethrows a reset connection unchanged', async () => {
+ const failure = fetchFailure('ECONNRESET')
+ const fetchFn = vi.fn(async () => {
+ throw failure
+ })
+ await expect(
+ clientAt('http://control-api.invalid', fetchFn as unknown as typeof fetch).redeem(redeemInput)
+ ).rejects.toBe(failure)
+ expect(fetchFn).toHaveBeenCalledTimes(1)
+ })
+
+ it.each([
+ ['a connect-phase code', fetchFailure('UND_ERR_CONNECT_TIMEOUT')],
+ ['the timeout reason', new DOMException('The operation was aborted due to timeout', 'TimeoutError')],
+ ])('G1-4d rethrows %s unchanged once the request timeout fired', async (_label, failure) => {
+ const timeout = vi
+ .spyOn(AbortSignal, 'timeout')
+ .mockReturnValue(
+ AbortSignal.abort(new DOMException('The operation was aborted due to timeout', 'TimeoutError'))
+ )
+ try {
+ const fetchFn = vi.fn(async () => {
+ throw failure
+ })
+ await expect(
+ clientAt('http://control-api.invalid', fetchFn as unknown as typeof fetch).redeem(
+ redeemInput
+ )
+ ).rejects.toBe(failure)
+ expect(fetchFn).toHaveBeenCalledTimes(1)
+ expect(timeout).toHaveBeenCalledWith(CONTROL_API_REQUEST_TIMEOUT_MS)
+ } finally {
+ timeout.mockRestore()
+ }
+ })
+
+ it('G1-4e keeps a non-JSON 502 as provider_unavailable', async () => {
+ const server = await listen((_req, _body, res) => {
+ const raw = res as unknown as {
+ statusCode: number
+ setHeader: (name: string, value: string) => void
+ end: (chunk: string) => void
+ }
+ raw.statusCode = 502
+ raw.setHeader('content-type', 'text/html')
+ raw.end('<html>502 Bad Gateway</html>')
+ })
+ try {
+ await expect(clientAt(server.url).redeem(redeemInput)).rejects.toMatchObject({
+ code: 'provider_unavailable',
+ })
+ // Witness: the request reached the server.
+ expect(server.requests).toHaveLength(1)
+ } finally {
+ await server.close()
+ }
+ })
+})
diff --git a/codex-llm-proxy/test/server.security.test.ts b/codex-llm-proxy/test/server.security.test.ts
index c6abd77da..bf8fb88cb 100644
--- a/codex-llm-proxy/test/server.security.test.ts
+++ b/codex-llm-proxy/test/server.security.test.ts
@@ -1,5 +1,5 @@
import { generateKeyPairSync } from 'node:crypto'
-import { request as httpRequest } from 'node:http'
+import { createServer, request as httpRequest } from 'node:http'
import type { AddressInfo } from 'node:net'
import jwt from 'jsonwebtoken'
import request from 'supertest'
@@ -1250,6 +1250,48 @@ describe('codex-llm-proxy attempt telemetry', () => {
expect(failureCount(metricsText, 'other')).toBe(0)
})
+ // G1-4 (#720): a redeem whose connection nothing accepted (control-api or
+ // its gateway restarting) is reported as the control plane being down.
+ it('(g1-4a) answers 503 control_plane_unavailable when redeem cannot connect', async () => {
+ const closed = createServer()
+ await new Promise<void>(resolve => closed.listen(0, '127.0.0.1', () => resolve()))
+ const { port } = closed.address() as AddressInfo
+ await new Promise<void>(resolve => closed.close(() => resolve()))
+ const upstreamFetch = vi.fn(upstream(1, 0))
+ const { res, lines, metricsText } = await run(
+ 'att-control-down',
+ 0,
+ 0,
+ undefined,
+ undefined,
+ {
+ fetchFn: upstreamFetch as unknown as typeof fetch,
+ controlApiClient: new ControlApiClient({
+ baseUrl: `http://127.0.0.1:${port}/api/v1`,
+ serviceName: 'codex-llm-proxy',
+ serviceToken: 'dev-codex-llm-proxy-token',
+ }),
+ }
+ )
+ expect(res.status).toBe(503)
+ expect(res.headers['content-type']).toMatch(/^application\/json/)
+ expect(res.body).toEqual({ error: 'control_plane_unavailable' })
+ // Redeem failed, so the upstream was never called.
+ expect(upstreamFetch).not.toHaveBeenCalled()
+ expect(lines).toHaveLength(1)
+ expect(lines[0]).toMatchObject({
+ providerAttemptId: 'att-control-down',
+ outcome: 'failed',
+ code: 'control_plane_unavailable',
+ causeCode: 'ECONNREFUSED',
+ deliveredAs: 'http_status',
+ httpStatus: 503,
+ })
+ expectNoForbiddenKeys(lines[0]!)
+ expect(failureCount(metricsText, 'control_plane_unavailable')).toBe(1)
+ expect(failureCount(metricsText, 'other')).toBe(0)
+ })
+
// #731 — the recorded upstream context-window refusal reaches the Host as a
// 400 `context_length_exceeded` (the upstream's own status class), not as a
// retryable 503 `provider_unavailable`.
diff --git a/grok-llm-proxy/test/controlApiClient.test.ts b/grok-llm-proxy/test/controlApiClient.test.ts
index 4b803a4ba..8b33b4298 100644
--- a/grok-llm-proxy/test/controlApiClient.test.ts
+++ b/grok-llm-proxy/test/controlApiClient.test.ts
@@ -1,6 +1,10 @@
-import { describe, expect, it } from 'vitest'
+import { describe, expect, it, vi } from 'vitest'
import { type IncomingMessage, type Server, createServer } from 'node:http'
-import { ControlApiClient, ControlApiClientError } from '../src/controlApiClient.js'
+import {
+ CONTROL_API_REQUEST_TIMEOUT_MS,
+ ControlApiClient,
+ ControlApiClientError,
+} from '../src/controlApiClient.js'
import { GROK_CATALOG_ORIGIN, GROK_COMPLETIONS_ORIGIN } from '../src/originPolicy.js'
const LOOPBACK_V4 = ['127', '0', '0', '1'].join('.')
@@ -266,3 +270,126 @@ describe('ControlApiClient response hardening', () => {
}
})
})
+
+// G1-4 (#720): a redeem that never reached a live control-plane process is
+// control_plane_unavailable; a failure that may have reached one is not.
+describe('ControlApiClient control-plane reachability', () => {
+ async function closedPortUrl(): Promise<string> {
+ const server = createServer()
+ await new Promise<void>(resolve => server.listen(0, '127.0.0.1', () => resolve()))
+ const addr = server.address()
+ const port = typeof addr === 'object' && addr ? addr.port : 0
+ await new Promise<void>(resolve => server.close(() => resolve()))
+ return `http://${LOOPBACK_V4}:${port}`
+ }
+
+ function clientAt(baseUrl: string, fetchFn?: typeof fetch): ControlApiClient {
+ return new ControlApiClient({
+ baseUrl: `${baseUrl}/api/v1`,
+ serviceName: 'grok-llm-proxy',
+ serviceToken: 'dev-grok-llm-proxy-token',
+ ...(fetchFn ? { fetchFn } : {}),
+ })
+ }
+
+ const redeemInput = {
+ executionTicket: 'ticket',
+ requestHash: 'e'.repeat(64),
+ operation: 'completion_stream' as const,
+ }
+
+ // The shape undici gives a fetch that failed before any response: a
+ // TypeError whose cause carries the system or undici error code.
+ function fetchFailure(code: string): TypeError {
+ return Object.assign(new TypeError('fetch failed'), {
+ cause: Object.assign(new Error(code), { code }),
+ })
+ }
+
+ it('G1-4a maps a refused connection to control_plane_unavailable', async () => {
+ const url = await closedPortUrl()
+ // Witness: the platform fetch fails this way against the closed port, so
+ // the mocked shapes below match what undici really produces.
+ const raw = await fetch(`${url}/api/v1/probe`).catch((caught: unknown) => caught)
+ expect(raw).toBeInstanceOf(TypeError)
+ expect((raw as { cause?: { code?: unknown } }).cause?.code).toBe('ECONNREFUSED')
+ await expect(clientAt(url).redeem(redeemInput)).rejects.toMatchObject({
+ name: 'ControlApiClientError',
+ code: 'control_plane_unavailable',
+ causeCode: 'ECONNREFUSED',
+ })
+ })
+
+ it.each(['ENOTFOUND', 'EAI_AGAIN', 'EHOSTUNREACH', 'ENETUNREACH', 'UND_ERR_CONNECT_TIMEOUT'])(
+ 'G1-4b maps a %s fetch failure to control_plane_unavailable',
+ async code => {
+ const fetchFn = vi.fn(async () => {
+ throw fetchFailure(code)
+ })
+ await expect(
+ clientAt('http://control-api.invalid', fetchFn as unknown as typeof fetch).redeem(
+ redeemInput
+ )
+ ).rejects.toMatchObject({ code: 'control_plane_unavailable', causeCode: code })
+ expect(fetchFn).toHaveBeenCalledTimes(1)
+ }
+ )
+
+ it('G1-4c rethrows a reset connection unchanged', async () => {
+ const failure = fetchFailure('ECONNRESET')
+ const fetchFn = vi.fn(async () => {
+ throw failure
+ })
+ await expect(
+ clientAt('http://control-api.invalid', fetchFn as unknown as typeof fetch).redeem(redeemInput)
+ ).rejects.toBe(failure)
+ expect(fetchFn).toHaveBeenCalledTimes(1)
+ })
+
+ it.each([
+ ['a connect-phase code', fetchFailure('UND_ERR_CONNECT_TIMEOUT')],
+ ['the timeout reason', new DOMException('The operation was aborted due to timeout', 'TimeoutError')],
+ ])('G1-4d rethrows %s unchanged once the request timeout fired', async (_label, failure) => {
+ const timeout = vi
+ .spyOn(AbortSignal, 'timeout')
+ .mockReturnValue(
+ AbortSignal.abort(new DOMException('The operation was aborted due to timeout', 'TimeoutError'))
+ )
+ try {
+ const fetchFn = vi.fn(async () => {
+ throw failure
+ })
+ await expect(
+ clientAt('http://control-api.invalid', fetchFn as unknown as typeof fetch).redeem(
+ redeemInput
+ )
+ ).rejects.toBe(failure)
+ expect(fetchFn).toHaveBeenCalledTimes(1)
+ expect(timeout).toHaveBeenCalledWith(CONTROL_API_REQUEST_TIMEOUT_MS)
+ } finally {
+ timeout.mockRestore()
+ }
+ })
+
+ it('G1-4e keeps a non-JSON 502 as provider_unavailable', async () => {
+ const server = await listen((_req, _body, res) => {
+ const raw = res as unknown as {
+ statusCode: number
+ setHeader: (name: string, value: string) => void
+ end: (chunk: string) => void
+ }
+ raw.statusCode = 502
+ raw.setHeader('content-type', 'text/html')
+ raw.end('<html>502 Bad Gateway</html>')
+ })
+ try {
+ await expect(clientAt(server.url).redeem(redeemInput)).rejects.toMatchObject({
+ code: 'provider_unavailable',
+ })
+ // Witness: the request reached the server.
+ expect(server.requests).toHaveLength(1)
+ } finally {
+ await server.close()
+ }
+ })
+})
diff --git a/grok-llm-proxy/test/server.security.test.ts b/grok-llm-proxy/test/server.security.test.ts
index 41b16528d..524beb08f 100644
--- a/grok-llm-proxy/test/server.security.test.ts
+++ b/grok-llm-proxy/test/server.security.test.ts
@@ -1,7 +1,7 @@
import { describe, expect, it, vi } from 'vitest'
import jwt from 'jsonwebtoken'
import { generateKeyPairSync } from 'node:crypto'
-import { request as httpRequest } from 'node:http'
+import { createServer, request as httpRequest } from 'node:http'
import type { AddressInfo } from 'node:net'
import request from 'supertest'
import {
@@ -1088,6 +1088,42 @@ describe('grok-llm-proxy attempt telemetry', () => {
expect(failureCount(metricsText, 'other')).toBe(0)
})
+ // G1-4 (#720): a redeem whose connection nothing accepted (control-api or
+ // its gateway restarting) is reported as the control plane being down.
+ it('(g1-4a) answers 503 control_plane_unavailable when redeem cannot connect', async () => {
+ const closed = createServer()
+ await new Promise<void>(resolve => closed.listen(0, '127.0.0.1', () => resolve()))
+ const { port } = closed.address() as AddressInfo
+ await new Promise<void>(resolve => closed.close(() => resolve()))
+ const upstreamFetch = vi.fn(upstream(1, 0))
+ const { res, lines, metricsText } = await run({
+ providerAttemptId: 'att-control-down',
+ fetchFn: upstreamFetch as unknown as typeof fetch,
+ controlApiClient: new ControlApiClient({
+ baseUrl: `http://127.0.0.1:${port}/api/v1`,
+ serviceName: 'grok-llm-proxy',
+ serviceToken: 'dev-grok-llm-proxy-token',
+ }),
+ })
+ expect(res.status).toBe(503)
+ expect(res.headers['content-type']).toMatch(/^application\/json/)
+ expect(res.body).toEqual({ error: 'control_plane_unavailable' })
+ // Redeem failed, so the upstream was never called.
+ expect(upstreamFetch).not.toHaveBeenCalled()
+ expect(lines).toHaveLength(1)
+ expect(lines[0]).toMatchObject({
+ providerAttemptId: 'att-control-down',
+ outcome: 'failed',
+ code: 'control_plane_unavailable',
+ causeCode: 'ECONNREFUSED',
+ deliveredAs: 'http_status',
+ httpStatus: 503,
+ })
+ expectNoForbiddenKeys(lines[0]!)
+ expect(failureCount(metricsText, 'control_plane_unavailable')).toBe(1)
+ expect(failureCount(metricsText, 'other')).toBe(0)
+ })
+
it('(a) answers 422 and logs one attempt line when 257 calls arrive before any text', async () => {
const { res, receipts, lines, metricsText } = await run({
providerAttemptId: 'att-limit-http',