Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
test(llm-proxy): require a redeem that cannot connect to answer control_plane_unavailable
Alfredo Lopezcdefcaaf995c
diff --git a/codex-llm-proxy/test/controlApiClient.test.ts b/codex-llm-proxy/test/controlApiClient.test.tsindex 30f81abc9..461d64173 100644--- a/codex-llm-proxy/test/controlApiClient.test.ts+++ b/codex-llm-proxy/test/controlApiClient.test.ts@@ -1,6 +1,10 @@import { createServer, type IncomingMessage, type Server } from 'node:http'-import { describe, expect, it } from 'vitest'-import { ControlApiClient, ControlApiClientError } from '../src/controlApiClient.js'+import { describe, expect, it, vi } from 'vitest'+import {+ CONTROL_API_REQUEST_TIMEOUT_MS,+ ControlApiClient,+ ControlApiClientError,+} from '../src/controlApiClient.js'import { CODEX_CATALOG_ORIGIN, CODEX_COMPLETIONS_ORIGIN } from '../src/originPolicy.js'const LOOPBACK_V4 = ['127', '0', '0', '1'].join('.')@@ -259,3 +263,126 @@ describe('ControlApiClient response hardening', () => {}})})++// G1-4 (#720): a redeem that never reached a live control-plane process is+// control_plane_unavailable; a failure that may have reached one is not.+describe('ControlApiClient control-plane reachability', () => {+ async function closedPortUrl(): Promise<string> {+ const server = createServer()+ await new Promise<void>(resolve => server.listen(0, '127.0.0.1', () => resolve()))+ const addr = server.address()+ const port = typeof addr === 'object' && addr ? addr.port : 0+ await new Promise<void>(resolve => server.close(() => resolve()))+ return `http://${LOOPBACK_V4}:${port}`+ }++ function clientAt(baseUrl: string, fetchFn?: typeof fetch): ControlApiClient {+ return new ControlApiClient({+ baseUrl: `${baseUrl}/api/v1`,+ serviceName: 'codex-llm-proxy',+ serviceToken: 'dev-codex-llm-proxy-token',+ ...(fetchFn ? { fetchFn } : {}),+ })+ }++ const redeemInput = {+ executionTicket: 'ticket',+ requestHash: 'e'.repeat(64),+ operation: 'completion_stream' as const,+ }++ // The shape undici gives a fetch that failed before any response: a+ // TypeError whose cause carries the system or undici error code.+ function fetchFailure(code: string): TypeError {+ return Object.assign(new TypeError('fetch failed'), {+ cause: Object.assign(new Error(code), { code }),+ })+ }++ it('G1-4a maps a refused connection to control_plane_unavailable', async () => {+ const url = await closedPortUrl()+ // Witness: the platform fetch fails this way against the closed port, so+ // the mocked shapes below match what undici really produces.+ const raw = await fetch(`${url}/api/v1/probe`).catch((caught: unknown) => caught)+ expect(raw).toBeInstanceOf(TypeError)+ expect((raw as { cause?: { code?: unknown } }).cause?.code).toBe('ECONNREFUSED')+ await expect(clientAt(url).redeem(redeemInput)).rejects.toMatchObject({+ name: 'ControlApiClientError',+ code: 'control_plane_unavailable',+ causeCode: 'ECONNREFUSED',+ })+ })++ it.each(['ENOTFOUND', 'EAI_AGAIN', 'EHOSTUNREACH', 'ENETUNREACH', 'UND_ERR_CONNECT_TIMEOUT'])(+ 'G1-4b maps a %s fetch failure to control_plane_unavailable',+ async code => {+ const fetchFn = vi.fn(async () => {+ throw fetchFailure(code)+ })+ await expect(+ clientAt('http://control-api.invalid', fetchFn as unknown as typeof fetch).redeem(+ redeemInput+ )+ ).rejects.toMatchObject({ code: 'control_plane_unavailable', causeCode: code })+ expect(fetchFn).toHaveBeenCalledTimes(1)+ }+ )++ it('G1-4c rethrows a reset connection unchanged', async () => {+ const failure = fetchFailure('ECONNRESET')+ const fetchFn = vi.fn(async () => {+ throw failure+ })+ await expect(+ clientAt('http://control-api.invalid', fetchFn as unknown as typeof fetch).redeem(redeemInput)+ ).rejects.toBe(failure)+ expect(fetchFn).toHaveBeenCalledTimes(1)+ })++ it.each([+ ['a connect-phase code', fetchFailure('UND_ERR_CONNECT_TIMEOUT')],+ ['the timeout reason', new DOMException('The operation was aborted due to timeout', 'TimeoutError')],+ ])('G1-4d rethrows %s unchanged once the request timeout fired', async (_label, failure) => {+ const timeout = vi+ .spyOn(AbortSignal, 'timeout')+ .mockReturnValue(+ AbortSignal.abort(new DOMException('The operation was aborted due to timeout', 'TimeoutError'))+ )+ try {+ const fetchFn = vi.fn(async () => {+ throw failure+ })+ await expect(+ clientAt('http://control-api.invalid', fetchFn as unknown as typeof fetch).redeem(+ redeemInput+ )+ ).rejects.toBe(failure)+ expect(fetchFn).toHaveBeenCalledTimes(1)+ expect(timeout).toHaveBeenCalledWith(CONTROL_API_REQUEST_TIMEOUT_MS)+ } finally {+ timeout.mockRestore()+ }+ })++ it('G1-4e keeps a non-JSON 502 as provider_unavailable', async () => {+ const server = await listen((_req, _body, res) => {+ const raw = res as unknown as {+ statusCode: number+ setHeader: (name: string, value: string) => void+ end: (chunk: string) => void+ }+ raw.statusCode = 502+ raw.setHeader('content-type', 'text/html')+ raw.end('<html>502 Bad Gateway</html>')+ })+ try {+ await expect(clientAt(server.url).redeem(redeemInput)).rejects.toMatchObject({+ code: 'provider_unavailable',+ })+ // Witness: the request reached the server.+ expect(server.requests).toHaveLength(1)+ } finally {+ await server.close()+ }+ })+})diff --git a/codex-llm-proxy/test/server.security.test.ts b/codex-llm-proxy/test/server.security.test.tsindex c6abd77da..bf8fb88cb 100644--- a/codex-llm-proxy/test/server.security.test.ts+++ b/codex-llm-proxy/test/server.security.test.ts@@ -1,5 +1,5 @@import { generateKeyPairSync } from 'node:crypto'-import { request as httpRequest } from 'node:http'+import { createServer, request as httpRequest } from 'node:http'import type { AddressInfo } from 'node:net'import jwt from 'jsonwebtoken'import request from 'supertest'@@ -1250,6 +1250,48 @@ describe('codex-llm-proxy attempt telemetry', () => {expect(failureCount(metricsText, 'other')).toBe(0)})+ // G1-4 (#720): a redeem whose connection nothing accepted (control-api or+ // its gateway restarting) is reported as the control plane being down.+ it('(g1-4a) answers 503 control_plane_unavailable when redeem cannot connect', async () => {+ const closed = createServer()+ await new Promise<void>(resolve => closed.listen(0, '127.0.0.1', () => resolve()))+ const { port } = closed.address() as AddressInfo+ await new Promise<void>(resolve => closed.close(() => resolve()))+ const upstreamFetch = vi.fn(upstream(1, 0))+ const { res, lines, metricsText } = await run(+ 'att-control-down',+ 0,+ 0,+ undefined,+ undefined,+ {+ fetchFn: upstreamFetch as unknown as typeof fetch,+ controlApiClient: new ControlApiClient({+ baseUrl: `http://127.0.0.1:${port}/api/v1`,+ serviceName: 'codex-llm-proxy',+ serviceToken: 'dev-codex-llm-proxy-token',+ }),+ }+ )+ expect(res.status).toBe(503)+ expect(res.headers['content-type']).toMatch(/^application\/json/)+ expect(res.body).toEqual({ error: 'control_plane_unavailable' })+ // Redeem failed, so the upstream was never called.+ expect(upstreamFetch).not.toHaveBeenCalled()+ expect(lines).toHaveLength(1)+ expect(lines[0]).toMatchObject({+ providerAttemptId: 'att-control-down',+ outcome: 'failed',+ code: 'control_plane_unavailable',+ causeCode: 'ECONNREFUSED',+ deliveredAs: 'http_status',+ httpStatus: 503,+ })+ expectNoForbiddenKeys(lines[0]!)+ expect(failureCount(metricsText, 'control_plane_unavailable')).toBe(1)+ expect(failureCount(metricsText, 'other')).toBe(0)+ })+// #731 — the recorded upstream context-window refusal reaches the Host as a// 400 `context_length_exceeded` (the upstream's own status class), not as a// retryable 503 `provider_unavailable`.diff --git a/grok-llm-proxy/test/controlApiClient.test.ts b/grok-llm-proxy/test/controlApiClient.test.tsindex 4b803a4ba..8b33b4298 100644--- a/grok-llm-proxy/test/controlApiClient.test.ts+++ b/grok-llm-proxy/test/controlApiClient.test.ts@@ -1,6 +1,10 @@-import { describe, expect, it } from 'vitest'+import { describe, expect, it, vi } from 'vitest'import { type IncomingMessage, type Server, createServer } from 'node:http'-import { ControlApiClient, ControlApiClientError } from '../src/controlApiClient.js'+import {+ CONTROL_API_REQUEST_TIMEOUT_MS,+ ControlApiClient,+ ControlApiClientError,+} from '../src/controlApiClient.js'import { GROK_CATALOG_ORIGIN, GROK_COMPLETIONS_ORIGIN } from '../src/originPolicy.js'const LOOPBACK_V4 = ['127', '0', '0', '1'].join('.')@@ -266,3 +270,126 @@ describe('ControlApiClient response hardening', () => {}})})++// G1-4 (#720): a redeem that never reached a live control-plane process is+// control_plane_unavailable; a failure that may have reached one is not.+describe('ControlApiClient control-plane reachability', () => {+ async function closedPortUrl(): Promise<string> {+ const server = createServer()+ await new Promise<void>(resolve => server.listen(0, '127.0.0.1', () => resolve()))+ const addr = server.address()+ const port = typeof addr === 'object' && addr ? addr.port : 0+ await new Promise<void>(resolve => server.close(() => resolve()))+ return `http://${LOOPBACK_V4}:${port}`+ }++ function clientAt(baseUrl: string, fetchFn?: typeof fetch): ControlApiClient {+ return new ControlApiClient({+ baseUrl: `${baseUrl}/api/v1`,+ serviceName: 'grok-llm-proxy',+ serviceToken: 'dev-grok-llm-proxy-token',+ ...(fetchFn ? { fetchFn } : {}),+ })+ }++ const redeemInput = {+ executionTicket: 'ticket',+ requestHash: 'e'.repeat(64),+ operation: 'completion_stream' as const,+ }++ // The shape undici gives a fetch that failed before any response: a+ // TypeError whose cause carries the system or undici error code.+ function fetchFailure(code: string): TypeError {+ return Object.assign(new TypeError('fetch failed'), {+ cause: Object.assign(new Error(code), { code }),+ })+ }++ it('G1-4a maps a refused connection to control_plane_unavailable', async () => {+ const url = await closedPortUrl()+ // Witness: the platform fetch fails this way against the closed port, so+ // the mocked shapes below match what undici really produces.+ const raw = await fetch(`${url}/api/v1/probe`).catch((caught: unknown) => caught)+ expect(raw).toBeInstanceOf(TypeError)+ expect((raw as { cause?: { code?: unknown } }).cause?.code).toBe('ECONNREFUSED')+ await expect(clientAt(url).redeem(redeemInput)).rejects.toMatchObject({+ name: 'ControlApiClientError',+ code: 'control_plane_unavailable',+ causeCode: 'ECONNREFUSED',+ })+ })++ it.each(['ENOTFOUND', 'EAI_AGAIN', 'EHOSTUNREACH', 'ENETUNREACH', 'UND_ERR_CONNECT_TIMEOUT'])(+ 'G1-4b maps a %s fetch failure to control_plane_unavailable',+ async code => {+ const fetchFn = vi.fn(async () => {+ throw fetchFailure(code)+ })+ await expect(+ clientAt('http://control-api.invalid', fetchFn as unknown as typeof fetch).redeem(+ redeemInput+ )+ ).rejects.toMatchObject({ code: 'control_plane_unavailable', causeCode: code })+ expect(fetchFn).toHaveBeenCalledTimes(1)+ }+ )++ it('G1-4c rethrows a reset connection unchanged', async () => {+ const failure = fetchFailure('ECONNRESET')+ const fetchFn = vi.fn(async () => {+ throw failure+ })+ await expect(+ clientAt('http://control-api.invalid', fetchFn as unknown as typeof fetch).redeem(redeemInput)+ ).rejects.toBe(failure)+ expect(fetchFn).toHaveBeenCalledTimes(1)+ })++ it.each([+ ['a connect-phase code', fetchFailure('UND_ERR_CONNECT_TIMEOUT')],+ ['the timeout reason', new DOMException('The operation was aborted due to timeout', 'TimeoutError')],+ ])('G1-4d rethrows %s unchanged once the request timeout fired', async (_label, failure) => {+ const timeout = vi+ .spyOn(AbortSignal, 'timeout')+ .mockReturnValue(+ AbortSignal.abort(new DOMException('The operation was aborted due to timeout', 'TimeoutError'))+ )+ try {+ const fetchFn = vi.fn(async () => {+ throw failure+ })+ await expect(+ clientAt('http://control-api.invalid', fetchFn as unknown as typeof fetch).redeem(+ redeemInput+ )+ ).rejects.toBe(failure)+ expect(fetchFn).toHaveBeenCalledTimes(1)+ expect(timeout).toHaveBeenCalledWith(CONTROL_API_REQUEST_TIMEOUT_MS)+ } finally {+ timeout.mockRestore()+ }+ })++ it('G1-4e keeps a non-JSON 502 as provider_unavailable', async () => {+ const server = await listen((_req, _body, res) => {+ const raw = res as unknown as {+ statusCode: number+ setHeader: (name: string, value: string) => void+ end: (chunk: string) => void+ }+ raw.statusCode = 502+ raw.setHeader('content-type', 'text/html')+ raw.end('<html>502 Bad Gateway</html>')+ })+ try {+ await expect(clientAt(server.url).redeem(redeemInput)).rejects.toMatchObject({+ code: 'provider_unavailable',+ })+ // Witness: the request reached the server.+ expect(server.requests).toHaveLength(1)+ } finally {+ await server.close()+ }+ })+})diff --git a/grok-llm-proxy/test/server.security.test.ts b/grok-llm-proxy/test/server.security.test.tsindex 41b16528d..524beb08f 100644--- a/grok-llm-proxy/test/server.security.test.ts+++ b/grok-llm-proxy/test/server.security.test.ts@@ -1,7 +1,7 @@import { describe, expect, it, vi } from 'vitest'import jwt from 'jsonwebtoken'import { generateKeyPairSync } from 'node:crypto'-import { request as httpRequest } from 'node:http'+import { createServer, request as httpRequest } from 'node:http'import type { AddressInfo } from 'node:net'import request from 'supertest'import {@@ -1088,6 +1088,42 @@ describe('grok-llm-proxy attempt telemetry', () => {expect(failureCount(metricsText, 'other')).toBe(0)})+ // G1-4 (#720): a redeem whose connection nothing accepted (control-api or+ // its gateway restarting) is reported as the control plane being down.+ it('(g1-4a) answers 503 control_plane_unavailable when redeem cannot connect', async () => {+ const closed = createServer()+ await new Promise<void>(resolve => closed.listen(0, '127.0.0.1', () => resolve()))+ const { port } = closed.address() as AddressInfo+ await new Promise<void>(resolve => closed.close(() => resolve()))+ const upstreamFetch = vi.fn(upstream(1, 0))+ const { res, lines, metricsText } = await run({+ providerAttemptId: 'att-control-down',+ fetchFn: upstreamFetch as unknown as typeof fetch,+ controlApiClient: new ControlApiClient({+ baseUrl: `http://127.0.0.1:${port}/api/v1`,+ serviceName: 'grok-llm-proxy',+ serviceToken: 'dev-grok-llm-proxy-token',+ }),+ })+ expect(res.status).toBe(503)+ expect(res.headers['content-type']).toMatch(/^application\/json/)+ expect(res.body).toEqual({ error: 'control_plane_unavailable' })+ // Redeem failed, so the upstream was never called.+ expect(upstreamFetch).not.toHaveBeenCalled()+ expect(lines).toHaveLength(1)+ expect(lines[0]).toMatchObject({+ providerAttemptId: 'att-control-down',+ outcome: 'failed',+ code: 'control_plane_unavailable',+ causeCode: 'ECONNREFUSED',+ deliveredAs: 'http_status',+ httpStatus: 503,+ })+ expectNoForbiddenKeys(lines[0]!)+ expect(failureCount(metricsText, 'control_plane_unavailable')).toBe(1)+ expect(failureCount(metricsText, 'other')).toBe(0)+ })+it('(a) answers 422 and logs one attempt line when 257 calls arrive before any text', async () => {const { res, receipts, lines, metricsText } = await run({providerAttemptId: 'att-limit-http',