7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.
TypeScript
feat(proxy): add OpenAI Responses API endpoint for Codex CLI (#144)
Tashfeen376c2e1b4be8
diff --git a/server/src/__tests__/routes/responses-translate.test.ts b/server/src/__tests__/routes/responses-translate.test.tsnew file mode 100644index 00000000..bc3f6276--- /dev/null+++ b/server/src/__tests__/routes/responses-translate.test.ts@@ -0,0 +1,89 @@+import { describe, it, expect } from 'vitest';+import {+ toChatMessages,+ toChatTools,+ toChatToolChoice,+ buildResponseObject,+} from '../../routes/responses.js';++describe('Responses → chat translation (#96)', () => {+ it('maps a plain string input to a single user message', () => {+ expect(toChatMessages({ input: 'hello' } as any)).toEqual([+ { role: 'user', content: 'hello' },+ ]);+ });++ it('prepends instructions as a system message', () => {+ const msgs = toChatMessages({ instructions: 'You are terse.', input: 'hi' } as any);+ expect(msgs[0]).toEqual({ role: 'system', content: 'You are terse.' });+ expect(msgs[1]).toEqual({ role: 'user', content: 'hi' });+ });++ it('flattens message items with content parts and maps the developer role to system', () => {+ const msgs = toChatMessages({+ input: [+ { type: 'message', role: 'developer', content: [{ type: 'input_text', text: 'sys' }] },+ { type: 'message', role: 'user', content: [{ type: 'input_text', text: 'a' }, { type: 'input_text', text: 'b' }] },+ ],+ } as any);+ expect(msgs).toEqual([+ { role: 'system', content: 'sys' },+ { role: 'user', content: 'ab' },+ ]);+ });++ it('maps a function_call item to an assistant tool_call', () => {+ const msgs = toChatMessages({+ input: [{ type: 'function_call', call_id: 'call_1', name: 'get_weather', arguments: '{"city":"SF"}' }],+ } as any);+ expect(msgs[0]).toEqual({+ role: 'assistant',+ content: null,+ tool_calls: [{ id: 'call_1', type: 'function', function: { name: 'get_weather', arguments: '{"city":"SF"}' } }],+ });+ });++ it('maps a function_call_output item to a tool message', () => {+ const msgs = toChatMessages({+ input: [{ type: 'function_call_output', call_id: 'call_1', output: 'sunny' }],+ } as any);+ expect(msgs[0]).toEqual({ role: 'tool', tool_call_id: 'call_1', content: 'sunny' });+ });++ it('converts flat Responses tools to nested chat tools', () => {+ const tools = toChatTools([+ { type: 'function', name: 'f', description: 'd', parameters: { type: 'object' }, strict: true },+ ] as any);+ expect(tools).toEqual([+ { type: 'function', function: { name: 'f', description: 'd', parameters: { type: 'object' }, strict: true } },+ ]);+ });++ it('converts tool_choice forms', () => {+ expect(toChatToolChoice('auto' as any)).toBe('auto');+ expect(toChatToolChoice({ type: 'function', name: 'f' } as any)).toEqual({ type: 'function', function: { name: 'f' } });+ expect(toChatToolChoice(undefined)).toBeUndefined();+ });+});++describe('chat result → Responses object (#96)', () => {+ it('builds a message output item plus usage for text', () => {+ const r = buildResponseObject({ id: 'resp_x', model: 'm', text: 'hi there', toolCalls: [], promptTokens: 5, completionTokens: 2 });+ expect(r.object).toBe('response');+ expect(r.status).toBe('completed');+ expect(r.output_text).toBe('hi there');+ expect(r.output).toHaveLength(1);+ expect(r.output[0]).toMatchObject({ type: 'message', role: 'assistant', content: [{ type: 'output_text', text: 'hi there' }] });+ expect(r.usage).toMatchObject({ input_tokens: 5, output_tokens: 2, total_tokens: 7 });+ });++ it('emits function_call output items for tool calls', () => {+ const r = buildResponseObject({+ id: 'resp_x', model: 'm', text: '',+ toolCalls: [{ id: 'call_1', type: 'function', function: { name: 'f', arguments: '{}' } }],+ promptTokens: 1, completionTokens: 1,+ });+ expect(r.output).toHaveLength(1);+ expect(r.output[0]).toMatchObject({ type: 'function_call', call_id: 'call_1', name: 'f', arguments: '{}' });+ });+});diff --git a/server/src/__tests__/routes/responses.test.ts b/server/src/__tests__/routes/responses.test.tsnew file mode 100644index 00000000..0217e804--- /dev/null+++ b/server/src/__tests__/routes/responses.test.ts@@ -0,0 +1,113 @@+import { describe, it, expect, beforeAll, vi } from 'vitest';++// Mock only routeRequest so we don't need real provider keys; keep the rest of+// the router module (recordSuccess / recordRateLimitHit) intact.+const { mockRouteRequest } = vi.hoisted(() => ({ mockRouteRequest: vi.fn() }));+vi.mock('../../services/router.js', async (importOriginal) => {+ const actual = await importOriginal<typeof import('../../services/router.js')>();+ return { ...actual, routeRequest: mockRouteRequest };+});++import type { Express } from 'express';+import { createApp } from '../../app.js';+import { initDb, getUnifiedApiKey } from '../../db/index.js';++function fakeRoute(provider: any) {+ return { provider, modelId: 'fake-model', modelDbId: 9999, apiKey: 'k', keyId: 1, platform: 'fake', displayName: 'Fake Model' };+}++async function post(app: Express, path: string, body: any, key?: string) {+ const server = app.listen(0);+ const addr = server.address() as any;+ const res = await fetch(`http://127.0.0.1:${addr.port}${path}`, {+ method: 'POST',+ headers: { 'Content-Type': 'application/json', ...(key ? { Authorization: `Bearer ${key}` } : {}) },+ body: JSON.stringify(body),+ });+ const text = await res.text();+ server.close();+ return { status: res.status, text, contentType: res.headers.get('content-type') ?? '' };+}++describe('POST /v1/responses (#96)', () => {+ let app: Express;+ let key: string;++ beforeAll(() => {+ process.env.ENCRYPTION_KEY = '0'.repeat(64);+ initDb(':memory:');+ app = createApp();+ key = getUnifiedApiKey();+ });++ it('rejects requests without a valid unified key (401)', async () => {+ expect((await post(app, '/v1/responses', { input: 'hi' })).status).toBe(401);+ expect((await post(app, '/v1/responses', { input: 'hi' }, 'wrong')).status).toBe(401);+ });++ it('rejects an invalid body (missing input) with 400', async () => {+ expect((await post(app, '/v1/responses', { model: 'auto' }, key)).status).toBe(400);+ });++ it('non-stream: returns a completed Responses object with usage', async () => {+ mockRouteRequest.mockReturnValue(fakeRoute({+ async chatCompletion() {+ return {+ id: 'c', object: 'chat.completion', created: 0, model: 'fake-model',+ choices: [{ index: 0, message: { role: 'assistant', content: 'Hello from fake' }, finish_reason: 'stop' }],+ usage: { prompt_tokens: 3, completion_tokens: 4, total_tokens: 7 },+ };+ },+ async *streamChatCompletion() { /* unused */ },+ }));++ const { status, text } = await post(app, '/v1/responses', { input: 'hi', stream: false }, key);+ expect(status).toBe(200);+ const body = JSON.parse(text);+ expect(body.object).toBe('response');+ expect(body.status).toBe('completed');+ expect(body.output_text).toBe('Hello from fake');+ expect(body.output[0]).toMatchObject({ type: 'message', role: 'assistant' });+ expect(body.usage.total_tokens).toBe(7);+ });++ it('stream: emits the Responses SSE event sequence', async () => {+ mockRouteRequest.mockReturnValue(fakeRoute({+ async chatCompletion() { throw new Error('should not be called'); },+ async *streamChatCompletion() {+ yield { id: 'c', object: 'chat.completion.chunk', created: 0, model: 'fake-model', choices: [{ index: 0, delta: { role: 'assistant', content: 'Hel' }, finish_reason: null }] };+ yield { id: 'c', object: 'chat.completion.chunk', created: 0, model: 'fake-model', choices: [{ index: 0, delta: { content: 'lo' }, finish_reason: null }] };+ yield { id: 'c', object: 'chat.completion.chunk', created: 0, model: 'fake-model', choices: [{ index: 0, delta: {}, finish_reason: 'stop' }] };+ },+ }));++ const { status, text, contentType } = await post(app, '/v1/responses', { input: 'hi', stream: true }, key);+ expect(status).toBe(200);+ expect(contentType).toContain('text/event-stream');+ for (const ev of ['response.created', 'response.output_item.added', 'response.content_part.added',+ 'response.output_text.delta', 'response.output_text.done', 'response.output_item.done', 'response.completed']) {+ expect(text).toContain(`event: ${ev}`);+ }+ expect(text).toContain('"delta":"Hel"');+ expect(text).toContain('"delta":"lo"');+ // the terminal event carries the assembled text+ const completed = text.split('event: response.completed')[1];+ expect(completed).toContain('"output_text":"Hello"');+ });++ it('stream: tool-call deltas produce function_call events with assembled arguments', async () => {+ mockRouteRequest.mockReturnValue(fakeRoute({+ async chatCompletion() { throw new Error('nope'); },+ async *streamChatCompletion() {+ yield { id: 'c', object: 'chat.completion.chunk', created: 0, model: 'fake-model', choices: [{ index: 0, delta: { tool_calls: [{ index: 0, id: 'call_1', type: 'function', function: { name: 'get_weather', arguments: '{"ci' } }] }, finish_reason: null }] };+ yield { id: 'c', object: 'chat.completion.chunk', created: 0, model: 'fake-model', choices: [{ index: 0, delta: { tool_calls: [{ index: 0, type: 'function', function: { arguments: 'ty":"SF"}' } }] }, finish_reason: 'tool_calls' }] };+ },+ }));++ const { text } = await post(app, '/v1/responses', { input: 'weather?', stream: true }, key);+ expect(text).toContain('"type":"function_call"');+ expect(text).toContain('event: response.function_call_arguments.delta');+ expect(text).toContain('event: response.function_call_arguments.done');+ expect(text).toContain('"arguments":"{\\"city\\":\\"SF\\"}"');+ });+});diff --git a/server/src/app.ts b/server/src/app.tsindex c3725e5e..718b1fbf 100644--- a/server/src/app.ts+++ b/server/src/app.ts@@ -6,6 +6,7 @@ import { fileURLToPath } from 'url';import { keysRouter } from './routes/keys.js';import { modelsRouter } from './routes/models.js';import { proxyRouter } from './routes/proxy.js';+import { responsesRouter } from './routes/responses.js';import { fallbackRouter } from './routes/fallback.js';import { analyticsRouter } from './routes/analytics.js';import { healthRouter } from './routes/health.js';@@ -57,6 +58,8 @@ export function createApp() {// OpenAI-compatible proxyapp.use('/v1', proxyRouter);+ // OpenAI Responses API shim (Codex CLI requires wire_api="responses"; see #96)+ app.use('/v1', responsesRouter);// Health checkapp.get('/api/ping', (_req, res) => {diff --git a/server/src/routes/proxy.ts b/server/src/routes/proxy.tsindex 884d05d9..370a9ef7 100644--- a/server/src/routes/proxy.ts+++ b/server/src/routes/proxy.ts@@ -23,7 +23,7 @@ function isAutoModel(modelId: string | undefined): boolean {// Constant-time string comparison for the unified API key. Plain `===` leaks// length and per-character timing, which a network attacker could in principle// use to recover the key one byte at a time.-function timingSafeStringEqual(provided: string, expected: string): boolean {+export function timingSafeStringEqual(provided: string, expected: string): boolean {const a = Buffer.from(provided);const b = Buffer.from(expected);// Compare against a same-length buffer regardless of input length so the@@ -50,7 +50,7 @@ function getSessionKey(messages: ChatMessage[]): string {return `${hash}:${messages.length > 2 ? 'multi' : 'single'}`;}-function getStickyModel(messages: ChatMessage[]): number | undefined {+export function getStickyModel(messages: ChatMessage[]): number | undefined {// Only apply sticky for multi-turn (has assistant messages = continuation)const hasAssistant = messages.some(m => m.role === 'assistant');if (!hasAssistant) return undefined;@@ -68,7 +68,7 @@ function getStickyModel(messages: ChatMessage[]): number | undefined {return entry.modelDbId;}-function setStickyModel(messages: ChatMessage[], modelDbId: number) {+export function setStickyModel(messages: ChatMessage[], modelDbId: number) {const key = getSessionKey(messages);if (!key) return;stickySessionMap.set(key, { modelDbId, lastUsed: Date.now() });@@ -473,7 +473,7 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {});});-function logRequest(+export function logRequest(platform: string,modelId: string,keyId: number,diff --git a/server/src/routes/responses.ts b/server/src/routes/responses.tsnew file mode 100644index 00000000..bbbed628--- /dev/null+++ b/server/src/routes/responses.ts@@ -0,0 +1,463 @@+import crypto from 'crypto';+import { Router } from 'express';+import type { Request, Response } from 'express';+import { z } from 'zod';+import type {+ ChatMessage,+ ChatToolCall,+ ChatToolDefinition,+ ChatToolChoice,+} from '@freellmapi/shared/types.js';+import { routeRequest, recordRateLimitHit, recordSuccess, type RouteResult } from '../services/router.js';+import { recordRequest, recordTokens, setCooldown, getNextCooldownDuration } from '../services/ratelimit.js';+import { getUnifiedApiKey } from '../db/index.js';+import { contentToString } from '../lib/content.js';+import {+ isRetryableError,+ timingSafeStringEqual,+ getStickyModel,+ setStickyModel,+ logRequest,+} from './proxy.js';++export const responsesRouter = Router();++// ─────────────────────────────────────────────────────────────────────────+// OpenAI Responses API shim (POST /v1/responses).+//+// Current Codex versions only speak the Responses API — `wire_api = "chat"`+// is rejected — so the existing /v1/chat/completions endpoint isn't reachable+// from Codex (see issue #96). This endpoint accepts a Responses-shaped request,+// translates it to the internal chat-message format, runs it through the SAME+// router/retry machinery as the proxy, and translates the result back into the+// Responses object / SSE event stream that Codex expects.+//+// Deliberately self-contained: it duplicates the proxy's retry loop rather than+// refactoring that battle-tested handler, so the production /chat/completions+// path is untouched. Shared, side-effect-free helpers (routing, rate-limit+// bookkeeping, sticky sessions, logging) are imported, not re-implemented.+// ─────────────────────────────────────────────────────────────────────────++const MAX_RETRIES = 20;++function newId(prefix: string): string {+ return `${prefix}_${crypto.randomBytes(18).toString('hex')}`;+}++function nowUnix(): number {+ return Math.floor(Date.now() / 1000);+}++// ── Request schema ──────────────────────────────────────────────────────+// Lenient on purpose: the Responses API surface is large and evolving, and we+// only consume the fields we can map. Unknown fields (store, reasoning,+// metadata, previous_response_id, …) are accepted and ignored.++const contentPartSchema = z.object({ type: z.string() }).passthrough();++const messageItemSchema = z.object({+ type: z.literal('message').optional(),+ role: z.enum(['system', 'developer', 'user', 'assistant']),+ content: z.union([z.string(), z.array(contentPartSchema)]),+});++const functionCallItemSchema = z.object({+ type: z.literal('function_call'),+ call_id: z.string(),+ name: z.string(),+ arguments: z.string(),+ id: z.string().optional(),+});++const functionCallOutputItemSchema = z.object({+ type: z.literal('function_call_output'),+ call_id: z.string(),+ output: z.union([z.string(), z.array(contentPartSchema), z.record(z.string(), z.unknown())]),+});++const inputItemSchema = z.union([+ functionCallItemSchema,+ functionCallOutputItemSchema,+ messageItemSchema,+]);++const responsesToolSchema = z.object({+ type: z.literal('function'),+ name: z.string(),+ description: z.string().nullable().optional(),+ parameters: z.record(z.string(), z.unknown()).nullable().optional(),+ strict: z.boolean().nullable().optional(),+}).passthrough();++const responsesRequestSchema = z.object({+ model: z.string().optional(),+ instructions: z.string().nullable().optional(),+ input: z.union([z.string(), z.array(inputItemSchema)]),+ stream: z.boolean().optional(),+ temperature: z.number().min(0).max(2).nullable().optional(),+ top_p: z.number().min(0).max(1).nullable().optional(),+ max_output_tokens: z.number().int().positive().nullable().optional(),+ tools: z.array(responsesToolSchema).optional(),+ tool_choice: z.union([+ z.enum(['none', 'auto', 'required']),+ z.object({ type: z.literal('function'), name: z.string() }).passthrough(),+ ]).optional(),+ parallel_tool_calls: z.boolean().nullable().optional(),+}).passthrough();++type ResponsesRequest = z.infer<typeof responsesRequestSchema>;++// Responses content parts → plain text. input_text / output_text both carry+// `text`; other part types (images, etc.) are dropped (parity with the proxy).+function partsToString(content: string | Array<{ type: string; text?: unknown }>): string {+ if (typeof content === 'string') return content;+ return content+ .map((p) => (typeof p.text === 'string' ? p.text : ''))+ .join('');+}++// ── Translate a Responses request → internal chat messages + options ──────+export function toChatMessages(req: ResponsesRequest): ChatMessage[] {+ const messages: ChatMessage[] = [];++ if (req.instructions) {+ messages.push({ role: 'system', content: req.instructions });+ }++ if (typeof req.input === 'string') {+ messages.push({ role: 'user', content: req.input });+ return messages;+ }++ for (const item of req.input) {+ if ('type' in item && item.type === 'function_call') {+ messages.push({+ role: 'assistant',+ content: null,+ tool_calls: [{+ id: item.call_id,+ type: 'function',+ function: { name: item.name, arguments: item.arguments },+ }],+ });+ } else if ('type' in item && item.type === 'function_call_output') {+ const output = typeof item.output === 'string'+ ? item.output+ : Array.isArray(item.output)+ ? partsToString(item.output as any)+ : JSON.stringify(item.output);+ messages.push({ role: 'tool', tool_call_id: item.call_id, content: output });+ } else {+ // message item+ const m = item as z.infer<typeof messageItemSchema>;+ // 'developer' is the Responses-era system role.+ const role = m.role === 'developer' ? 'system' : m.role;+ messages.push({ role, content: partsToString(m.content) });+ }+ }++ return messages;+}++export function toChatTools(tools?: ResponsesRequest['tools']): ChatToolDefinition[] | undefined {+ if (!tools?.length) return undefined;+ return tools.map((t) => ({+ type: 'function',+ function: {+ name: t.name,+ ...(t.description ? { description: t.description } : {}),+ ...(t.parameters ? { parameters: t.parameters } : {}),+ ...(t.strict != null ? { strict: t.strict } : {}),+ },+ }));+}++export function toChatToolChoice(tc?: ResponsesRequest['tool_choice']): ChatToolChoice | undefined {+ if (!tc) return undefined;+ if (typeof tc === 'string') return tc;+ return { type: 'function', function: { name: tc.name } };+}++// ── Build the final (non-stream) Responses object ─────────────────────────+export function buildResponseObject(opts: {+ id: string;+ model: string;+ text: string;+ toolCalls: ChatToolCall[];+ promptTokens: number;+ completionTokens: number;+}) {+ const output: any[] = [];+ if (opts.text.length > 0) {+ output.push({+ type: 'message',+ id: newId('msg'),+ status: 'completed',+ role: 'assistant',+ content: [{ type: 'output_text', text: opts.text, annotations: [] }],+ });+ }+ for (const tc of opts.toolCalls) {+ output.push({+ type: 'function_call',+ id: newId('fc'),+ call_id: tc.id,+ name: tc.function.name,+ arguments: tc.function.arguments,+ status: 'completed',+ });+ }++ return {+ id: opts.id,+ object: 'response',+ created_at: nowUnix(),+ status: 'completed',+ model: opts.model,+ output,+ output_text: opts.text,+ usage: {+ input_tokens: opts.promptTokens,+ input_tokens_details: { cached_tokens: 0 },+ output_tokens: opts.completionTokens,+ output_tokens_details: { reasoning_tokens: 0 },+ total_tokens: opts.promptTokens + opts.completionTokens,+ },+ };+}++responsesRouter.post('/responses', async (req: Request, res: Response) => {+ const start = Date.now();++ // Same unified-key bearer auth as the proxy.+ const token = req.headers.authorization?.replace(/^Bearer\s+/i, '');+ const unifiedKey = getUnifiedApiKey();+ if (!token || !timingSafeStringEqual(token, unifiedKey)) {+ res.status(401).json({ error: { message: 'Invalid API key', type: 'authentication_error' } });+ return;+ }++ const parsed = responsesRequestSchema.safeParse(req.body);+ if (!parsed.success) {+ res.status(400).json({+ error: {+ message: `Invalid request: ${parsed.error.errors.map((e) => `${e.path.join('.')}: ${e.message}`).join(', ')}`,+ type: 'invalid_request_error',+ },+ });+ return;+ }++ const reqData = parsed.data;+ const stream = reqData.stream ?? false;+ const messages = toChatMessages(reqData);+ const tools = toChatTools(reqData.tools);+ const tool_choice = toChatToolChoice(reqData.tool_choice);+ const completionOpts = {+ temperature: reqData.temperature ?? undefined,+ max_tokens: reqData.max_output_tokens ?? undefined,+ top_p: reqData.top_p ?? undefined,+ tools,+ tool_choice,+ parallel_tool_calls: reqData.parallel_tool_calls ?? undefined,+ };++ const estimatedInputTokens = messages.reduce(+ (sum, m) => sum + Math.ceil(contentToString(m.content).length / 4),+ 0,+ );+ const estimatedTotal = estimatedInputTokens + (reqData.max_output_tokens ?? 1000);+ const preferredModel = getStickyModel(messages);++ const responseId = newId('resp');+ const skipKeys = new Set<string>();+ let lastError: any = null;++ // Stream bookkeeping (used only when stream === true).+ let seq = 0;+ let streamStarted = false;+ const sse = (event: string, payload: Record<string, unknown>) => {+ res.write(`event: ${event}\n`);+ res.write(`data: ${JSON.stringify({ type: event, sequence_number: seq++, ...payload })}\n\n`);+ };++ for (let attempt = 0; attempt < MAX_RETRIES; attempt++) {+ let route: RouteResult;+ try {+ route = routeRequest(estimatedTotal, skipKeys.size > 0 ? skipKeys : undefined, preferredModel);+ } catch (err: any) {+ const status = lastError ? 429 : (err.status ?? 503);+ const message = lastError+ ? `All models rate-limited. Last error: ${lastError.message}`+ : err.message;+ const type = lastError ? 'rate_limit_error' : 'routing_error';+ if (streamStarted) {+ sse('response.failed', { response: { id: responseId, object: 'response', status: 'failed', error: { message, type } } });+ res.end();+ } else {+ res.status(status).json({ error: { message, type } });+ }+ return;+ }++ recordRequest(route.platform, route.modelId, route.keyId);++ try {+ if (stream) {+ // Headers + response.created on the first attempt that gets this far.+ if (!streamStarted) {+ res.setHeader('Content-Type', 'text/event-stream');+ res.setHeader('Cache-Control', 'no-cache');+ res.setHeader('Connection', 'keep-alive');+ res.setHeader('X-Routed-Via', `${route.platform}/${route.modelId}`);+ const skeleton = {+ id: responseId, object: 'response', created_at: nowUnix(),+ status: 'in_progress', model: route.modelId, output: [], output_text: '',+ };+ sse('response.created', { response: skeleton });+ sse('response.in_progress', { response: skeleton });+ streamStarted = true;+ }++ let outputIndex = 0;+ let msgItemId: string | null = null;+ let msgText = '';+ // tool-call accumulator keyed by the provider's tool_call index+ const toolAcc = new Map<number, { outputIndex: number; itemId: string; callId: string; name: string; args: string }>();+ let totalOutputTokens = 0;++ const gen = route.provider.streamChatCompletion(route.apiKey, messages, route.modelId, completionOpts);++ for await (const chunk of gen) {+ const delta = chunk.choices[0]?.delta;+ if (!delta) continue;++ // Text deltas → output_text events on a single message item.+ const text = delta.content ?? '';+ if (text) {+ if (msgItemId === null) {+ msgItemId = newId('msg');+ sse('response.output_item.added', {+ output_index: outputIndex,+ item: { id: msgItemId, type: 'message', status: 'in_progress', role: 'assistant', content: [] },+ });+ sse('response.content_part.added', {+ item_id: msgItemId, output_index: outputIndex, content_index: 0,+ part: { type: 'output_text', text: '', annotations: [] },+ });+ }+ sse('response.output_text.delta', {+ item_id: msgItemId, output_index: outputIndex, content_index: 0, delta: text,+ });+ msgText += text;+ totalOutputTokens += Math.ceil(text.length / 4);+ }++ // Tool-call deltas → function_call item + argument deltas.+ for (const tc of delta.tool_calls ?? []) {+ const idx = (tc as any).index ?? 0;+ let acc = toolAcc.get(idx);+ if (!acc) {+ // First time we see this tool call: open a new output item.+ if (msgItemId !== null && msgText.length > 0) {+ // close the text item (always output index 0) before starting a function_call item+ sse('response.output_text.done', { item_id: msgItemId, output_index: 0, content_index: 0, text: msgText });+ sse('response.content_part.done', { item_id: msgItemId, output_index: 0, content_index: 0, part: { type: 'output_text', text: msgText, annotations: [] } });+ sse('response.output_item.done', { output_index: 0, item: { id: msgItemId, type: 'message', status: 'completed', role: 'assistant', content: [{ type: 'output_text', text: msgText, annotations: [] }] } });+ msgItemId = null;+ }+ outputIndex = toolAcc.size + (msgText.length > 0 ? 1 : 0);+ acc = { outputIndex, itemId: newId('fc'), callId: tc.id || newId('call'), name: tc.function?.name ?? '', args: '' };+ toolAcc.set(idx, acc);+ sse('response.output_item.added', {+ output_index: acc.outputIndex,+ item: { id: acc.itemId, type: 'function_call', status: 'in_progress', call_id: acc.callId, name: acc.name, arguments: '' },+ });+ }+ const argFrag = tc.function?.arguments ?? '';+ if (tc.function?.name && !acc.name) acc.name = tc.function.name;+ if (argFrag) {+ acc.args += argFrag;+ sse('response.function_call_arguments.delta', { item_id: acc.itemId, output_index: acc.outputIndex, delta: argFrag });+ }+ }+ }++ // Finalize any open text item.+ if (msgItemId !== null) {+ sse('response.output_text.done', { item_id: msgItemId, output_index: 0, content_index: 0, text: msgText });+ sse('response.content_part.done', { item_id: msgItemId, output_index: 0, content_index: 0, part: { type: 'output_text', text: msgText, annotations: [] } });+ sse('response.output_item.done', { output_index: 0, item: { id: msgItemId, type: 'message', status: 'completed', role: 'assistant', content: [{ type: 'output_text', text: msgText, annotations: [] }] } });+ }+ // Finalize tool-call items.+ const finalToolCalls: ChatToolCall[] = [];+ for (const acc of toolAcc.values()) {+ sse('response.function_call_arguments.done', { item_id: acc.itemId, output_index: acc.outputIndex, arguments: acc.args });+ sse('response.output_item.done', { output_index: acc.outputIndex, item: { id: acc.itemId, type: 'function_call', status: 'completed', call_id: acc.callId, name: acc.name, arguments: acc.args } });+ finalToolCalls.push({ id: acc.callId, type: 'function', function: { name: acc.name, arguments: acc.args } });+ }++ const finalResponse = buildResponseObject({+ id: responseId, model: route.modelId, text: msgText,+ toolCalls: finalToolCalls, promptTokens: estimatedInputTokens, completionTokens: totalOutputTokens,+ });+ sse('response.completed', { response: finalResponse });+ res.end();++ recordTokens(route.platform, route.modelId, route.keyId, estimatedInputTokens + totalOutputTokens);+ recordSuccess(route.modelDbId);+ setStickyModel(messages, route.modelDbId);+ logRequest(route.platform, route.modelId, route.keyId, 'success', estimatedInputTokens, totalOutputTokens, Date.now() - start, null);+ return;+ } else {+ const result = await route.provider.chatCompletion(route.apiKey, messages, route.modelId, completionOpts);++ const msg = result.choices[0]?.message;+ const text = contentToString(msg?.content ?? '');+ const toolCalls = msg?.tool_calls ?? [];+ const promptTokens = result.usage?.prompt_tokens ?? estimatedInputTokens;+ const completionTokens = result.usage?.completion_tokens ?? Math.ceil(text.length / 4);++ recordTokens(route.platform, route.modelId, route.keyId, result.usage?.total_tokens ?? 0);+ recordSuccess(route.modelDbId);+ setStickyModel(messages, route.modelDbId);++ res.setHeader('X-Routed-Via', `${route.platform}/${route.modelId}`);+ if (attempt > 0) res.setHeader('X-Fallback-Attempts', String(attempt));+ res.json(buildResponseObject({+ id: responseId, model: route.modelId, text, toolCalls,+ promptTokens, completionTokens,+ }));++ logRequest(route.platform, route.modelId, route.keyId, 'success',+ promptTokens, completionTokens, Date.now() - start, null);+ return;+ }+ } catch (err: any) {+ const latency = Date.now() - start;+ logRequest(route.platform, route.modelId, route.keyId, 'error', estimatedInputTokens, 0, latency, err.message);++ // Mid-stream failures can't be retried (bytes already sent) — close cleanly.+ if (stream && streamStarted) {+ sse('response.failed', { response: { id: responseId, object: 'response', status: 'failed', error: { message: `Provider error (${route.displayName}): stream interrupted`, type: 'stream_error' } } });+ res.end();+ return;+ }++ if (isRetryableError(err)) {+ skipKeys.add(`${route.platform}:${route.modelId}:${route.keyId}`);+ setCooldown(route.platform, route.modelId, route.keyId, getNextCooldownDuration(route.platform, route.modelId, route.keyId));+ recordRateLimitHit(route.modelDbId);+ lastError = err;+ continue;+ }++ res.status(502).json({ error: { message: `Provider error (${route.displayName}): ${err.message}`, type: 'provider_error' } });+ return;+ }+ }++ res.status(429).json({+ error: { message: `All models rate-limited after ${MAX_RETRIES} attempts. Last: ${lastError?.message}`, type: 'rate_limit_error' },+ });+});