Skip to content

7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.

TypeScript

feat(proxy): add OpenAI Responses API endpoint for Codex CLI (#144)

Tashfeen376c2e1b4be8
diff --git a/server/src/__tests__/routes/responses-translate.test.ts b/server/src/__tests__/routes/responses-translate.test.ts
new file mode 100644
index 00000000..bc3f6276
--- /dev/null
+++ b/server/src/__tests__/routes/responses-translate.test.ts
@@ -0,0 +1,89 @@
+import { describe, it, expect } from 'vitest';
+import {
+ toChatMessages,
+ toChatTools,
+ toChatToolChoice,
+ buildResponseObject,
+} from '../../routes/responses.js';
+
+describe('Responses → chat translation (#96)', () => {
+ it('maps a plain string input to a single user message', () => {
+ expect(toChatMessages({ input: 'hello' } as any)).toEqual([
+ { role: 'user', content: 'hello' },
+ ]);
+ });
+
+ it('prepends instructions as a system message', () => {
+ const msgs = toChatMessages({ instructions: 'You are terse.', input: 'hi' } as any);
+ expect(msgs[0]).toEqual({ role: 'system', content: 'You are terse.' });
+ expect(msgs[1]).toEqual({ role: 'user', content: 'hi' });
+ });
+
+ it('flattens message items with content parts and maps the developer role to system', () => {
+ const msgs = toChatMessages({
+ input: [
+ { type: 'message', role: 'developer', content: [{ type: 'input_text', text: 'sys' }] },
+ { type: 'message', role: 'user', content: [{ type: 'input_text', text: 'a' }, { type: 'input_text', text: 'b' }] },
+ ],
+ } as any);
+ expect(msgs).toEqual([
+ { role: 'system', content: 'sys' },
+ { role: 'user', content: 'ab' },
+ ]);
+ });
+
+ it('maps a function_call item to an assistant tool_call', () => {
+ const msgs = toChatMessages({
+ input: [{ type: 'function_call', call_id: 'call_1', name: 'get_weather', arguments: '{"city":"SF"}' }],
+ } as any);
+ expect(msgs[0]).toEqual({
+ role: 'assistant',
+ content: null,
+ tool_calls: [{ id: 'call_1', type: 'function', function: { name: 'get_weather', arguments: '{"city":"SF"}' } }],
+ });
+ });
+
+ it('maps a function_call_output item to a tool message', () => {
+ const msgs = toChatMessages({
+ input: [{ type: 'function_call_output', call_id: 'call_1', output: 'sunny' }],
+ } as any);
+ expect(msgs[0]).toEqual({ role: 'tool', tool_call_id: 'call_1', content: 'sunny' });
+ });
+
+ it('converts flat Responses tools to nested chat tools', () => {
+ const tools = toChatTools([
+ { type: 'function', name: 'f', description: 'd', parameters: { type: 'object' }, strict: true },
+ ] as any);
+ expect(tools).toEqual([
+ { type: 'function', function: { name: 'f', description: 'd', parameters: { type: 'object' }, strict: true } },
+ ]);
+ });
+
+ it('converts tool_choice forms', () => {
+ expect(toChatToolChoice('auto' as any)).toBe('auto');
+ expect(toChatToolChoice({ type: 'function', name: 'f' } as any)).toEqual({ type: 'function', function: { name: 'f' } });
+ expect(toChatToolChoice(undefined)).toBeUndefined();
+ });
+});
+
+describe('chat result → Responses object (#96)', () => {
+ it('builds a message output item plus usage for text', () => {
+ const r = buildResponseObject({ id: 'resp_x', model: 'm', text: 'hi there', toolCalls: [], promptTokens: 5, completionTokens: 2 });
+ expect(r.object).toBe('response');
+ expect(r.status).toBe('completed');
+ expect(r.output_text).toBe('hi there');
+ expect(r.output).toHaveLength(1);
+ expect(r.output[0]).toMatchObject({ type: 'message', role: 'assistant', content: [{ type: 'output_text', text: 'hi there' }] });
+ expect(r.usage).toMatchObject({ input_tokens: 5, output_tokens: 2, total_tokens: 7 });
+ });
+
+ it('emits function_call output items for tool calls', () => {
+ const r = buildResponseObject({
+ id: 'resp_x', model: 'm', text: '',
+ toolCalls: [{ id: 'call_1', type: 'function', function: { name: 'f', arguments: '{}' } }],
+ promptTokens: 1, completionTokens: 1,
+ });
+ expect(r.output).toHaveLength(1);
+ expect(r.output[0]).toMatchObject({ type: 'function_call', call_id: 'call_1', name: 'f', arguments: '{}' });
+ });
+});
diff --git a/server/src/__tests__/routes/responses.test.ts b/server/src/__tests__/routes/responses.test.ts
new file mode 100644
index 00000000..0217e804
--- /dev/null
+++ b/server/src/__tests__/routes/responses.test.ts
@@ -0,0 +1,113 @@
+import { describe, it, expect, beforeAll, vi } from 'vitest';
+
+// Mock only routeRequest so we don't need real provider keys; keep the rest of
+// the router module (recordSuccess / recordRateLimitHit) intact.
+const { mockRouteRequest } = vi.hoisted(() => ({ mockRouteRequest: vi.fn() }));
+vi.mock('../../services/router.js', async (importOriginal) => {
+ const actual = await importOriginal<typeof import('../../services/router.js')>();
+ return { ...actual, routeRequest: mockRouteRequest };
+});
+
+import type { Express } from 'express';
+import { createApp } from '../../app.js';
+import { initDb, getUnifiedApiKey } from '../../db/index.js';
+
+function fakeRoute(provider: any) {
+ return { provider, modelId: 'fake-model', modelDbId: 9999, apiKey: 'k', keyId: 1, platform: 'fake', displayName: 'Fake Model' };
+}
+
+async function post(app: Express, path: string, body: any, key?: string) {
+ const server = app.listen(0);
+ const addr = server.address() as any;
+ const res = await fetch(`http://127.0.0.1:${addr.port}${path}`, {
+ method: 'POST',
+ headers: { 'Content-Type': 'application/json', ...(key ? { Authorization: `Bearer ${key}` } : {}) },
+ body: JSON.stringify(body),
+ });
+ const text = await res.text();
+ server.close();
+ return { status: res.status, text, contentType: res.headers.get('content-type') ?? '' };
+}
+
+describe('POST /v1/responses (#96)', () => {
+ let app: Express;
+ let key: string;
+
+ beforeAll(() => {
+ process.env.ENCRYPTION_KEY = '0'.repeat(64);
+ initDb(':memory:');
+ app = createApp();
+ key = getUnifiedApiKey();
+ });
+
+ it('rejects requests without a valid unified key (401)', async () => {
+ expect((await post(app, '/v1/responses', { input: 'hi' })).status).toBe(401);
+ expect((await post(app, '/v1/responses', { input: 'hi' }, 'wrong')).status).toBe(401);
+ });
+
+ it('rejects an invalid body (missing input) with 400', async () => {
+ expect((await post(app, '/v1/responses', { model: 'auto' }, key)).status).toBe(400);
+ });
+
+ it('non-stream: returns a completed Responses object with usage', async () => {
+ mockRouteRequest.mockReturnValue(fakeRoute({
+ async chatCompletion() {
+ return {
+ id: 'c', object: 'chat.completion', created: 0, model: 'fake-model',
+ choices: [{ index: 0, message: { role: 'assistant', content: 'Hello from fake' }, finish_reason: 'stop' }],
+ usage: { prompt_tokens: 3, completion_tokens: 4, total_tokens: 7 },
+ };
+ },
+ async *streamChatCompletion() { /* unused */ },
+ }));
+
+ const { status, text } = await post(app, '/v1/responses', { input: 'hi', stream: false }, key);
+ expect(status).toBe(200);
+ const body = JSON.parse(text);
+ expect(body.object).toBe('response');
+ expect(body.status).toBe('completed');
+ expect(body.output_text).toBe('Hello from fake');
+ expect(body.output[0]).toMatchObject({ type: 'message', role: 'assistant' });
+ expect(body.usage.total_tokens).toBe(7);
+ });
+
+ it('stream: emits the Responses SSE event sequence', async () => {
+ mockRouteRequest.mockReturnValue(fakeRoute({
+ async chatCompletion() { throw new Error('should not be called'); },
+ async *streamChatCompletion() {
+ yield { id: 'c', object: 'chat.completion.chunk', created: 0, model: 'fake-model', choices: [{ index: 0, delta: { role: 'assistant', content: 'Hel' }, finish_reason: null }] };
+ yield { id: 'c', object: 'chat.completion.chunk', created: 0, model: 'fake-model', choices: [{ index: 0, delta: { content: 'lo' }, finish_reason: null }] };
+ yield { id: 'c', object: 'chat.completion.chunk', created: 0, model: 'fake-model', choices: [{ index: 0, delta: {}, finish_reason: 'stop' }] };
+ },
+ }));
+
+ const { status, text, contentType } = await post(app, '/v1/responses', { input: 'hi', stream: true }, key);
+ expect(status).toBe(200);
+ expect(contentType).toContain('text/event-stream');
+ for (const ev of ['response.created', 'response.output_item.added', 'response.content_part.added',
+ 'response.output_text.delta', 'response.output_text.done', 'response.output_item.done', 'response.completed']) {
+ expect(text).toContain(`event: ${ev}`);
+ }
+ expect(text).toContain('"delta":"Hel"');
+ expect(text).toContain('"delta":"lo"');
+ // the terminal event carries the assembled text
+ const completed = text.split('event: response.completed')[1];
+ expect(completed).toContain('"output_text":"Hello"');
+ });
+
+ it('stream: tool-call deltas produce function_call events with assembled arguments', async () => {
+ mockRouteRequest.mockReturnValue(fakeRoute({
+ async chatCompletion() { throw new Error('nope'); },
+ async *streamChatCompletion() {
+ yield { id: 'c', object: 'chat.completion.chunk', created: 0, model: 'fake-model', choices: [{ index: 0, delta: { tool_calls: [{ index: 0, id: 'call_1', type: 'function', function: { name: 'get_weather', arguments: '{"ci' } }] }, finish_reason: null }] };
+ yield { id: 'c', object: 'chat.completion.chunk', created: 0, model: 'fake-model', choices: [{ index: 0, delta: { tool_calls: [{ index: 0, type: 'function', function: { arguments: 'ty":"SF"}' } }] }, finish_reason: 'tool_calls' }] };
+ },
+ }));
+
+ const { text } = await post(app, '/v1/responses', { input: 'weather?', stream: true }, key);
+ expect(text).toContain('"type":"function_call"');
+ expect(text).toContain('event: response.function_call_arguments.delta');
+ expect(text).toContain('event: response.function_call_arguments.done');
+ expect(text).toContain('"arguments":"{\\"city\\":\\"SF\\"}"');
+ });
+});
diff --git a/server/src/app.ts b/server/src/app.ts
index c3725e5e..718b1fbf 100644
--- a/server/src/app.ts
+++ b/server/src/app.ts
@@ -6,6 +6,7 @@ import { fileURLToPath } from 'url';
import { keysRouter } from './routes/keys.js';
import { modelsRouter } from './routes/models.js';
import { proxyRouter } from './routes/proxy.js';
+import { responsesRouter } from './routes/responses.js';
import { fallbackRouter } from './routes/fallback.js';
import { analyticsRouter } from './routes/analytics.js';
import { healthRouter } from './routes/health.js';
@@ -57,6 +58,8 @@ export function createApp() {
// OpenAI-compatible proxy
app.use('/v1', proxyRouter);
+ // OpenAI Responses API shim (Codex CLI requires wire_api="responses"; see #96)
+ app.use('/v1', responsesRouter);
// Health check
app.get('/api/ping', (_req, res) => {
diff --git a/server/src/routes/proxy.ts b/server/src/routes/proxy.ts
index 884d05d9..370a9ef7 100644
--- a/server/src/routes/proxy.ts
+++ b/server/src/routes/proxy.ts
@@ -23,7 +23,7 @@ function isAutoModel(modelId: string | undefined): boolean {
// Constant-time string comparison for the unified API key. Plain `===` leaks
// length and per-character timing, which a network attacker could in principle
// use to recover the key one byte at a time.
-function timingSafeStringEqual(provided: string, expected: string): boolean {
+export function timingSafeStringEqual(provided: string, expected: string): boolean {
const a = Buffer.from(provided);
const b = Buffer.from(expected);
// Compare against a same-length buffer regardless of input length so the
@@ -50,7 +50,7 @@ function getSessionKey(messages: ChatMessage[]): string {
return `${hash}:${messages.length > 2 ? 'multi' : 'single'}`;
}
-function getStickyModel(messages: ChatMessage[]): number | undefined {
+export function getStickyModel(messages: ChatMessage[]): number | undefined {
// Only apply sticky for multi-turn (has assistant messages = continuation)
const hasAssistant = messages.some(m => m.role === 'assistant');
if (!hasAssistant) return undefined;
@@ -68,7 +68,7 @@ function getStickyModel(messages: ChatMessage[]): number | undefined {
return entry.modelDbId;
}
-function setStickyModel(messages: ChatMessage[], modelDbId: number) {
+export function setStickyModel(messages: ChatMessage[], modelDbId: number) {
const key = getSessionKey(messages);
if (!key) return;
stickySessionMap.set(key, { modelDbId, lastUsed: Date.now() });
@@ -473,7 +473,7 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {
});
});
-function logRequest(
+export function logRequest(
platform: string,
modelId: string,
keyId: number,
diff --git a/server/src/routes/responses.ts b/server/src/routes/responses.ts
new file mode 100644
index 00000000..bbbed628
--- /dev/null
+++ b/server/src/routes/responses.ts
@@ -0,0 +1,463 @@
+import crypto from 'crypto';
+import { Router } from 'express';
+import type { Request, Response } from 'express';
+import { z } from 'zod';
+import type {
+ ChatMessage,
+ ChatToolCall,
+ ChatToolDefinition,
+ ChatToolChoice,
+} from '@freellmapi/shared/types.js';
+import { routeRequest, recordRateLimitHit, recordSuccess, type RouteResult } from '../services/router.js';
+import { recordRequest, recordTokens, setCooldown, getNextCooldownDuration } from '../services/ratelimit.js';
+import { getUnifiedApiKey } from '../db/index.js';
+import { contentToString } from '../lib/content.js';
+import {
+ isRetryableError,
+ timingSafeStringEqual,
+ getStickyModel,
+ setStickyModel,
+ logRequest,
+} from './proxy.js';
+
+export const responsesRouter = Router();
+
+// ─────────────────────────────────────────────────────────────────────────
+// OpenAI Responses API shim (POST /v1/responses).
+//
+// Current Codex versions only speak the Responses API — `wire_api = "chat"`
+// is rejected — so the existing /v1/chat/completions endpoint isn't reachable
+// from Codex (see issue #96). This endpoint accepts a Responses-shaped request,
+// translates it to the internal chat-message format, runs it through the SAME
+// router/retry machinery as the proxy, and translates the result back into the
+// Responses object / SSE event stream that Codex expects.
+//
+// Deliberately self-contained: it duplicates the proxy's retry loop rather than
+// refactoring that battle-tested handler, so the production /chat/completions
+// path is untouched. Shared, side-effect-free helpers (routing, rate-limit
+// bookkeeping, sticky sessions, logging) are imported, not re-implemented.
+// ─────────────────────────────────────────────────────────────────────────
+
+const MAX_RETRIES = 20;
+
+function newId(prefix: string): string {
+ return `${prefix}_${crypto.randomBytes(18).toString('hex')}`;
+}
+
+function nowUnix(): number {
+ return Math.floor(Date.now() / 1000);
+}
+
+// ── Request schema ──────────────────────────────────────────────────────
+// Lenient on purpose: the Responses API surface is large and evolving, and we
+// only consume the fields we can map. Unknown fields (store, reasoning,
+// metadata, previous_response_id, …) are accepted and ignored.
+
+const contentPartSchema = z.object({ type: z.string() }).passthrough();
+
+const messageItemSchema = z.object({
+ type: z.literal('message').optional(),
+ role: z.enum(['system', 'developer', 'user', 'assistant']),
+ content: z.union([z.string(), z.array(contentPartSchema)]),
+});
+
+const functionCallItemSchema = z.object({
+ type: z.literal('function_call'),
+ call_id: z.string(),
+ name: z.string(),
+ arguments: z.string(),
+ id: z.string().optional(),
+});
+
+const functionCallOutputItemSchema = z.object({
+ type: z.literal('function_call_output'),
+ call_id: z.string(),
+ output: z.union([z.string(), z.array(contentPartSchema), z.record(z.string(), z.unknown())]),
+});
+
+const inputItemSchema = z.union([
+ functionCallItemSchema,
+ functionCallOutputItemSchema,
+ messageItemSchema,
+]);
+
+const responsesToolSchema = z.object({
+ type: z.literal('function'),
+ name: z.string(),
+ description: z.string().nullable().optional(),
+ parameters: z.record(z.string(), z.unknown()).nullable().optional(),
+ strict: z.boolean().nullable().optional(),
+}).passthrough();
+
+const responsesRequestSchema = z.object({
+ model: z.string().optional(),
+ instructions: z.string().nullable().optional(),
+ input: z.union([z.string(), z.array(inputItemSchema)]),
+ stream: z.boolean().optional(),
+ temperature: z.number().min(0).max(2).nullable().optional(),
+ top_p: z.number().min(0).max(1).nullable().optional(),
+ max_output_tokens: z.number().int().positive().nullable().optional(),
+ tools: z.array(responsesToolSchema).optional(),
+ tool_choice: z.union([
+ z.enum(['none', 'auto', 'required']),
+ z.object({ type: z.literal('function'), name: z.string() }).passthrough(),
+ ]).optional(),
+ parallel_tool_calls: z.boolean().nullable().optional(),
+}).passthrough();
+
+type ResponsesRequest = z.infer<typeof responsesRequestSchema>;
+
+// Responses content parts → plain text. input_text / output_text both carry
+// `text`; other part types (images, etc.) are dropped (parity with the proxy).
+function partsToString(content: string | Array<{ type: string; text?: unknown }>): string {
+ if (typeof content === 'string') return content;
+ return content
+ .map((p) => (typeof p.text === 'string' ? p.text : ''))
+ .join('');
+}
+
+// ── Translate a Responses request → internal chat messages + options ──────
+export function toChatMessages(req: ResponsesRequest): ChatMessage[] {
+ const messages: ChatMessage[] = [];
+
+ if (req.instructions) {
+ messages.push({ role: 'system', content: req.instructions });
+ }
+
+ if (typeof req.input === 'string') {
+ messages.push({ role: 'user', content: req.input });
+ return messages;
+ }
+
+ for (const item of req.input) {
+ if ('type' in item && item.type === 'function_call') {
+ messages.push({
+ role: 'assistant',
+ content: null,
+ tool_calls: [{
+ id: item.call_id,
+ type: 'function',
+ function: { name: item.name, arguments: item.arguments },
+ }],
+ });
+ } else if ('type' in item && item.type === 'function_call_output') {
+ const output = typeof item.output === 'string'
+ ? item.output
+ : Array.isArray(item.output)
+ ? partsToString(item.output as any)
+ : JSON.stringify(item.output);
+ messages.push({ role: 'tool', tool_call_id: item.call_id, content: output });
+ } else {
+ // message item
+ const m = item as z.infer<typeof messageItemSchema>;
+ // 'developer' is the Responses-era system role.
+ const role = m.role === 'developer' ? 'system' : m.role;
+ messages.push({ role, content: partsToString(m.content) });
+ }
+ }
+
+ return messages;
+}
+
+export function toChatTools(tools?: ResponsesRequest['tools']): ChatToolDefinition[] | undefined {
+ if (!tools?.length) return undefined;
+ return tools.map((t) => ({
+ type: 'function',
+ function: {
+ name: t.name,
+ ...(t.description ? { description: t.description } : {}),
+ ...(t.parameters ? { parameters: t.parameters } : {}),
+ ...(t.strict != null ? { strict: t.strict } : {}),
+ },
+ }));
+}
+
+export function toChatToolChoice(tc?: ResponsesRequest['tool_choice']): ChatToolChoice | undefined {
+ if (!tc) return undefined;
+ if (typeof tc === 'string') return tc;
+ return { type: 'function', function: { name: tc.name } };
+}
+
+// ── Build the final (non-stream) Responses object ─────────────────────────
+export function buildResponseObject(opts: {
+ id: string;
+ model: string;
+ text: string;
+ toolCalls: ChatToolCall[];
+ promptTokens: number;
+ completionTokens: number;
+}) {
+ const output: any[] = [];
+ if (opts.text.length > 0) {
+ output.push({
+ type: 'message',
+ id: newId('msg'),
+ status: 'completed',
+ role: 'assistant',
+ content: [{ type: 'output_text', text: opts.text, annotations: [] }],
+ });
+ }
+ for (const tc of opts.toolCalls) {
+ output.push({
+ type: 'function_call',
+ id: newId('fc'),
+ call_id: tc.id,
+ name: tc.function.name,
+ arguments: tc.function.arguments,
+ status: 'completed',
+ });
+ }
+
+ return {
+ id: opts.id,
+ object: 'response',
+ created_at: nowUnix(),
+ status: 'completed',
+ model: opts.model,
+ output,
+ output_text: opts.text,
+ usage: {
+ input_tokens: opts.promptTokens,
+ input_tokens_details: { cached_tokens: 0 },
+ output_tokens: opts.completionTokens,
+ output_tokens_details: { reasoning_tokens: 0 },
+ total_tokens: opts.promptTokens + opts.completionTokens,
+ },
+ };
+}
+
+responsesRouter.post('/responses', async (req: Request, res: Response) => {
+ const start = Date.now();
+
+ // Same unified-key bearer auth as the proxy.
+ const token = req.headers.authorization?.replace(/^Bearer\s+/i, '');
+ const unifiedKey = getUnifiedApiKey();
+ if (!token || !timingSafeStringEqual(token, unifiedKey)) {
+ res.status(401).json({ error: { message: 'Invalid API key', type: 'authentication_error' } });
+ return;
+ }
+
+ const parsed = responsesRequestSchema.safeParse(req.body);
+ if (!parsed.success) {
+ res.status(400).json({
+ error: {
+ message: `Invalid request: ${parsed.error.errors.map((e) => `${e.path.join('.')}: ${e.message}`).join(', ')}`,
+ type: 'invalid_request_error',
+ },
+ });
+ return;
+ }
+
+ const reqData = parsed.data;
+ const stream = reqData.stream ?? false;
+ const messages = toChatMessages(reqData);
+ const tools = toChatTools(reqData.tools);
+ const tool_choice = toChatToolChoice(reqData.tool_choice);
+ const completionOpts = {
+ temperature: reqData.temperature ?? undefined,
+ max_tokens: reqData.max_output_tokens ?? undefined,
+ top_p: reqData.top_p ?? undefined,
+ tools,
+ tool_choice,
+ parallel_tool_calls: reqData.parallel_tool_calls ?? undefined,
+ };
+
+ const estimatedInputTokens = messages.reduce(
+ (sum, m) => sum + Math.ceil(contentToString(m.content).length / 4),
+ 0,
+ );
+ const estimatedTotal = estimatedInputTokens + (reqData.max_output_tokens ?? 1000);
+ const preferredModel = getStickyModel(messages);
+
+ const responseId = newId('resp');
+ const skipKeys = new Set<string>();
+ let lastError: any = null;
+
+ // Stream bookkeeping (used only when stream === true).
+ let seq = 0;
+ let streamStarted = false;
+ const sse = (event: string, payload: Record<string, unknown>) => {
+ res.write(`event: ${event}\n`);
+ res.write(`data: ${JSON.stringify({ type: event, sequence_number: seq++, ...payload })}\n\n`);
+ };
+
+ for (let attempt = 0; attempt < MAX_RETRIES; attempt++) {
+ let route: RouteResult;
+ try {
+ route = routeRequest(estimatedTotal, skipKeys.size > 0 ? skipKeys : undefined, preferredModel);
+ } catch (err: any) {
+ const status = lastError ? 429 : (err.status ?? 503);
+ const message = lastError
+ ? `All models rate-limited. Last error: ${lastError.message}`
+ : err.message;
+ const type = lastError ? 'rate_limit_error' : 'routing_error';
+ if (streamStarted) {
+ sse('response.failed', { response: { id: responseId, object: 'response', status: 'failed', error: { message, type } } });
+ res.end();
+ } else {
+ res.status(status).json({ error: { message, type } });
+ }
+ return;
+ }
+
+ recordRequest(route.platform, route.modelId, route.keyId);
+
+ try {
+ if (stream) {
+ // Headers + response.created on the first attempt that gets this far.
+ if (!streamStarted) {
+ res.setHeader('Content-Type', 'text/event-stream');
+ res.setHeader('Cache-Control', 'no-cache');
+ res.setHeader('Connection', 'keep-alive');
+ res.setHeader('X-Routed-Via', `${route.platform}/${route.modelId}`);
+ const skeleton = {
+ id: responseId, object: 'response', created_at: nowUnix(),
+ status: 'in_progress', model: route.modelId, output: [], output_text: '',
+ };
+ sse('response.created', { response: skeleton });
+ sse('response.in_progress', { response: skeleton });
+ streamStarted = true;
+ }
+
+ let outputIndex = 0;
+ let msgItemId: string | null = null;
+ let msgText = '';
+ // tool-call accumulator keyed by the provider's tool_call index
+ const toolAcc = new Map<number, { outputIndex: number; itemId: string; callId: string; name: string; args: string }>();
+ let totalOutputTokens = 0;
+
+ const gen = route.provider.streamChatCompletion(route.apiKey, messages, route.modelId, completionOpts);
+
+ for await (const chunk of gen) {
+ const delta = chunk.choices[0]?.delta;
+ if (!delta) continue;
+
+ // Text deltas → output_text events on a single message item.
+ const text = delta.content ?? '';
+ if (text) {
+ if (msgItemId === null) {
+ msgItemId = newId('msg');
+ sse('response.output_item.added', {
+ output_index: outputIndex,
+ item: { id: msgItemId, type: 'message', status: 'in_progress', role: 'assistant', content: [] },
+ });
+ sse('response.content_part.added', {
+ item_id: msgItemId, output_index: outputIndex, content_index: 0,
+ part: { type: 'output_text', text: '', annotations: [] },
+ });
+ }
+ sse('response.output_text.delta', {
+ item_id: msgItemId, output_index: outputIndex, content_index: 0, delta: text,
+ });
+ msgText += text;
+ totalOutputTokens += Math.ceil(text.length / 4);
+ }
+
+ // Tool-call deltas → function_call item + argument deltas.
+ for (const tc of delta.tool_calls ?? []) {
+ const idx = (tc as any).index ?? 0;
+ let acc = toolAcc.get(idx);
+ if (!acc) {
+ // First time we see this tool call: open a new output item.
+ if (msgItemId !== null && msgText.length > 0) {
+ // close the text item (always output index 0) before starting a function_call item
+ sse('response.output_text.done', { item_id: msgItemId, output_index: 0, content_index: 0, text: msgText });
+ sse('response.content_part.done', { item_id: msgItemId, output_index: 0, content_index: 0, part: { type: 'output_text', text: msgText, annotations: [] } });
+ sse('response.output_item.done', { output_index: 0, item: { id: msgItemId, type: 'message', status: 'completed', role: 'assistant', content: [{ type: 'output_text', text: msgText, annotations: [] }] } });
+ msgItemId = null;
+ }
+ outputIndex = toolAcc.size + (msgText.length > 0 ? 1 : 0);
+ acc = { outputIndex, itemId: newId('fc'), callId: tc.id || newId('call'), name: tc.function?.name ?? '', args: '' };
+ toolAcc.set(idx, acc);
+ sse('response.output_item.added', {
+ output_index: acc.outputIndex,
+ item: { id: acc.itemId, type: 'function_call', status: 'in_progress', call_id: acc.callId, name: acc.name, arguments: '' },
+ });
+ }
+ const argFrag = tc.function?.arguments ?? '';
+ if (tc.function?.name && !acc.name) acc.name = tc.function.name;
+ if (argFrag) {
+ acc.args += argFrag;
+ sse('response.function_call_arguments.delta', { item_id: acc.itemId, output_index: acc.outputIndex, delta: argFrag });
+ }
+ }
+ }
+
+ // Finalize any open text item.
+ if (msgItemId !== null) {
+ sse('response.output_text.done', { item_id: msgItemId, output_index: 0, content_index: 0, text: msgText });
+ sse('response.content_part.done', { item_id: msgItemId, output_index: 0, content_index: 0, part: { type: 'output_text', text: msgText, annotations: [] } });
+ sse('response.output_item.done', { output_index: 0, item: { id: msgItemId, type: 'message', status: 'completed', role: 'assistant', content: [{ type: 'output_text', text: msgText, annotations: [] }] } });
+ }
+ // Finalize tool-call items.
+ const finalToolCalls: ChatToolCall[] = [];
+ for (const acc of toolAcc.values()) {
+ sse('response.function_call_arguments.done', { item_id: acc.itemId, output_index: acc.outputIndex, arguments: acc.args });
+ sse('response.output_item.done', { output_index: acc.outputIndex, item: { id: acc.itemId, type: 'function_call', status: 'completed', call_id: acc.callId, name: acc.name, arguments: acc.args } });
+ finalToolCalls.push({ id: acc.callId, type: 'function', function: { name: acc.name, arguments: acc.args } });
+ }
+
+ const finalResponse = buildResponseObject({
+ id: responseId, model: route.modelId, text: msgText,
+ toolCalls: finalToolCalls, promptTokens: estimatedInputTokens, completionTokens: totalOutputTokens,
+ });
+ sse('response.completed', { response: finalResponse });
+ res.end();
+
+ recordTokens(route.platform, route.modelId, route.keyId, estimatedInputTokens + totalOutputTokens);
+ recordSuccess(route.modelDbId);
+ setStickyModel(messages, route.modelDbId);
+ logRequest(route.platform, route.modelId, route.keyId, 'success', estimatedInputTokens, totalOutputTokens, Date.now() - start, null);
+ return;
+ } else {
+ const result = await route.provider.chatCompletion(route.apiKey, messages, route.modelId, completionOpts);
+
+ const msg = result.choices[0]?.message;
+ const text = contentToString(msg?.content ?? '');
+ const toolCalls = msg?.tool_calls ?? [];
+ const promptTokens = result.usage?.prompt_tokens ?? estimatedInputTokens;
+ const completionTokens = result.usage?.completion_tokens ?? Math.ceil(text.length / 4);
+
+ recordTokens(route.platform, route.modelId, route.keyId, result.usage?.total_tokens ?? 0);
+ recordSuccess(route.modelDbId);
+ setStickyModel(messages, route.modelDbId);
+
+ res.setHeader('X-Routed-Via', `${route.platform}/${route.modelId}`);
+ if (attempt > 0) res.setHeader('X-Fallback-Attempts', String(attempt));
+ res.json(buildResponseObject({
+ id: responseId, model: route.modelId, text, toolCalls,
+ promptTokens, completionTokens,
+ }));
+
+ logRequest(route.platform, route.modelId, route.keyId, 'success',
+ promptTokens, completionTokens, Date.now() - start, null);
+ return;
+ }
+ } catch (err: any) {
+ const latency = Date.now() - start;
+ logRequest(route.platform, route.modelId, route.keyId, 'error', estimatedInputTokens, 0, latency, err.message);
+
+ // Mid-stream failures can't be retried (bytes already sent) — close cleanly.
+ if (stream && streamStarted) {
+ sse('response.failed', { response: { id: responseId, object: 'response', status: 'failed', error: { message: `Provider error (${route.displayName}): stream interrupted`, type: 'stream_error' } } });
+ res.end();
+ return;
+ }
+
+ if (isRetryableError(err)) {
+ skipKeys.add(`${route.platform}:${route.modelId}:${route.keyId}`);
+ setCooldown(route.platform, route.modelId, route.keyId, getNextCooldownDuration(route.platform, route.modelId, route.keyId));
+ recordRateLimitHit(route.modelDbId);
+ lastError = err;
+ continue;
+ }
+
+ res.status(502).json({ error: { message: `Provider error (${route.displayName}): ${err.message}`, type: 'provider_error' } });
+ return;
+ }
+ }
+
+ res.status(429).json({
+ error: { message: `All models rate-limited after ${MAX_RETRIES} attempts. Last: ${lastError?.message}`, type: 'rate_limit_error' },
+ });
+});