7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.
TypeScript
import crypto from 'crypto';
import { Router } from 'express';
import type { Request, Response } from 'express';
import { z } from 'zod';
import type {
ChatMessage,
ChatToolCall,
ChatToolDefinition,
ChatToolChoice,
Platform,
} from '@freellmapi/shared/types.js';
import { routeRequest, hasEnabledVisionModel, hasEnabledToolsModel, resolveStickyPreference, routingReserveTokens, resolveModelGroupCandidates, resolveRoutingChain, failureAttributionFor, type RouteResult, type ResolvedChain, type ChainRow } from '../services/router.js';
import { getDb } from '../db/index.js';
import { resolveAuth, prependSystemPrompt } from '../lib/system-prompt.js';
import { isUnifyEnabled, getModelGroups, resolveRequestedIdForDispatch } from '../services/model-groups.js';
import { contentToString, estimateInputTokens, messageHasImage } from '../lib/content.js';
import { routeOutputBudget } from '../lib/output-cap.js';
import { resolveTaskType } from '../lib/task-type.js';
import { normalizeMessageImages } from '../lib/image-normalize.js';
import { repairToolArguments, toolSchemaMap } from '../lib/tool-args.js';
import { invalidToolArgumentsError, invalidToolCallReasons, isToolArgumentValidationEnabled } from '../lib/tool-validate.js';
import { rescueInlineToolCalls, startsWithDialectMarker, couldBecomeDialectMarker, containsDialectMarker } from '../lib/tool-call-rescue.js';
import {
extractApiToken,
getRequestGroupId,
getStickyModel,
setStickyModel,
streamReasoningText,
completionReasoningText,
traceRouteEvent,
logRequest,
} from './proxy.js';
import { runFallbackLoop, newFallbackState, fallbackRoutingTokens, recordUpstreamSuccess, setFallbackHeaders, exhaustionErrorPayload, setExhaustionHeaders, type AttemptRecord } from '../lib/fallback-loop.js';
import { routedViaValue, safeHeaderValue } from '../lib/header-value.js';
import { startStreamKeepalive, streamKeepaliveMs } from '../lib/stream-keepalive.js';
import { applyTokenBudget, tokenBudgetMessage } from '../lib/guardrails.js';
import { samplingParamSchemaFields, pickSamplingParams, type ResponseFormat } from '../lib/sampling-params.js';
import { enforceJsonContent } from '../lib/structured-output.js';
import { sanitizeProviderErrorMessage } from '../lib/error-redaction.js';
import { isClientAbortError, newClientAbortError, newHedgeAbortError } from '../lib/error-classify.js';
import { inferQuotaPoolKey, type QuotaObservationContext } from '../services/provider-quota.js';
import { compressRequest, formatCompressionHeader } from '../services/compression/pipeline.js';
import { maybeServeResponsesWebSearch, webSearchCallToText } from '../lib/responses-web-search.js';
import {
FUSION_MODEL_ID,
FusionError,
fusionConfigSchema,
isFusionModel,
runFusion,
type FusionResult,
} from '../services/fusion.js';
import { applyDecisionRouting, routingStrategyHeader } from '../services/routing-decision.js';
import { resolveListedClaudeAlias } from '../services/anthropic-map.js';
export const responsesRouter = Router();
const AUTO_MODEL_ID = 'auto';
function isAutoModel(modelId: string | undefined): boolean {
if (!modelId) return true;
const lower = modelId.toLowerCase();
return lower === AUTO_MODEL_ID || lower.startsWith(`${AUTO_MODEL_ID}:`);
}
// ─────────────────────────────────────────────────────────────────────────
// OpenAI Responses API shim (POST /v1/responses).
//
// Current Codex versions only speak the Responses API — `wire_api = "chat"`
// is rejected — so the existing /v1/chat/completions endpoint isn't reachable
// from Codex (see issue #96). This endpoint accepts a Responses-shaped request,
// translates it to the internal chat-message format, runs it through the SAME
// shared fallback loop as the proxy (lib/fallback-loop.ts), and translates the
// result back into the Responses object / SSE event stream Codex expects.
//
// A thin adapter: the cooldown/skip/penalty/exhaustion machinery is shared, and
// only the Responses request/stream translation lives here. This is what fixed
// the drift where /v1/responses ignored a provider Retry-After and under-benched
// a 403 (both are now handled identically to /chat/completions by the shared
// loop) and committed the SSE skeleton on the first raw chunk (the commit point
// is now held until the first meaningful content, so a pre-content failure fails
// over invisibly).
// ─────────────────────────────────────────────────────────────────────────
const MAX_RETRIES = 20;
function newId(prefix: string): string {
return `${prefix}_${crypto.randomBytes(18).toString('hex')}`;
}
function nowUnix(): number {
return Math.floor(Date.now() / 1000);
}
// ── Request schema ──────────────────────────────────────────────────────
// Lenient on purpose: the Responses API surface is large and evolving, and we
// only consume the fields we can map. Unknown fields (store, reasoning,
// metadata, previous_response_id, …) are accepted and ignored.
const contentPartSchema = z.object({ type: z.string() }).passthrough();
const messageItemSchema = z.object({
type: z.literal('message').optional(),
role: z.enum(['system', 'developer', 'user', 'assistant']),
content: z.union([z.string(), z.array(contentPartSchema)]),
});
const functionCallItemSchema = z.object({
type: z.literal('function_call'),
call_id: z.string(),
name: z.string(),
arguments: z.string(),
id: z.string().optional(),
});
const functionCallOutputItemSchema = z.object({
type: z.literal('function_call_output'),
call_id: z.string(),
output: z.union([z.string(), z.array(contentPartSchema), z.record(z.string(), z.unknown())]),
});
// Remaining official ResponseInputItemParam kinds. Codex computer-use round-trips
// `computer_call` (the model's action request) and `computer_call_output` (the
// harness's result, incl. screenshots); multi-turn sessions also replay
// `reasoning` / `local_shell_call` items. We accept them all so validation never
// 400s on a standard payload; each is then either mapped (below) or dropped
// because chat-completions upstreams have no equivalent (computer/local_shell).
// Each schema is permissive — we only consume the fields that matter.
const computerCallItemSchema = z.object({
type: z.literal('computer_call'),
call_id: z.string(),
action: z.record(z.string(), z.unknown()).optional(),
id: z.string().optional(),
}).passthrough();
const computerCallOutputItemSchema = z.object({
type: z.literal('computer_call_output'),
call_id: z.string(),
output: z.union([
z.string(),
z.array(contentPartSchema),
z.record(z.string(), z.unknown()),
]).optional(),
id: z.string().optional(),
}).passthrough();
const reasoningItemSchema = z.object({
type: z.literal('reasoning'),
summary: z.union([z.string(), z.array(contentPartSchema)]).optional(),
content: z.union([z.string(), z.array(contentPartSchema)]).optional(),
id: z.string().optional(),
}).passthrough();
const localShellCallItemSchema = z.object({
type: z.literal('local_shell_call'),
call_id: z.string().optional(),
action: z.record(z.string(), z.unknown()).optional(),
id: z.string().optional(),
}).passthrough();
// Codex sends its client-side tool inventory as an `additional_tools` input
// item on every Responses turn. It is metadata for the harness, not a chat
// message, so the translator deliberately drops it below. Keep the schema
// permissive because Codex may add fields (or tool shapes) as the inventory
// evolves; rejecting the whole request here turns an otherwise valid launch
// into a misleading `input: Invalid input` 400.
const additionalToolsItemSchema = z.object({
type: z.literal('additional_tools'),
id: z.string().optional(),
role: z.string().optional(),
tools: z.array(z.record(z.string(), z.unknown())).optional(),
}).passthrough();
// The rest of the official ResponseInputItemParam union: built-in tool calls
// (web_search, file_search, code interpreter, image generation), MCP items,
// and item references. None has a chat-completions equivalent — validated
// loosely so a standard replay never 400s, then skipped at conversion like
// the kinds above.
const otherKnownItemSchema = z.object({
type: z.enum([
'web_search_call', 'file_search_call', 'code_interpreter_call',
'image_generation_call', 'mcp_call', 'mcp_list_tools',
'mcp_approval_request', 'mcp_approval_response', 'item_reference',
]),
id: z.string().optional(),
}).passthrough();
const inputItemSchema = z.union([
functionCallItemSchema,
functionCallOutputItemSchema,
computerCallItemSchema,
computerCallOutputItemSchema,
reasoningItemSchema,
localShellCallItemSchema,
additionalToolsItemSchema,
otherKnownItemSchema,
messageItemSchema,
]);
// Accept ANY tool type, not just 'function'. Codex (Responses API) sends
// built-in tools like `web_search` / `local_shell` alongside function tools;
// a strict z.literal('function') rejected the whole request. We validate
// loosely here and drop non-function tools at conversion (toChatTools), since
// chat-completions providers only accept type:'function'.
const responsesToolSchema = z.object({
type: z.string(),
name: z.string().optional(),
description: z.string().nullable().optional(),
parameters: z.record(z.string(), z.unknown()).nullable().optional(),
strict: z.boolean().nullable().optional(),
}).passthrough();
const responsesRequestSchema = z.object({
model: z.string().optional(),
instructions: z.string().nullable().optional(),
input: z.union([z.string(), z.array(inputItemSchema)]),
stream: z.boolean().optional(),
temperature: z.number().min(0).max(2).nullable().optional(),
top_p: z.number().min(0).max(1).nullable().optional(),
max_output_tokens: z.number().int().positive().nullable().optional(),
tools: z.array(responsesToolSchema).optional(),
// The virtual Fusion model fans the translated conversation out to a
// diverse panel, then optionally synthesizes the survivors. Keep the
// Responses surface in parity with /v1/chat/completions.
fusion: fusionConfigSchema.optional(),
tool_choice: z.union([
z.enum(['none', 'auto', 'required']),
z.object({ type: z.literal('function'), name: z.string() }).passthrough(),
]).optional(),
parallel_tool_calls: z.boolean().nullable().optional(),
// Extended sampling params, validated the same way as /chat/completions.
// Responses clients express structured output as `text.format` rather than
// `response_format` — mapped where completionOpts is built.
...samplingParamSchemaFields,
text: z.object({
format: z.object({
type: z.enum(['text', 'json_object', 'json_schema']),
name: z.string().optional(),
strict: z.boolean().nullable().optional(),
schema: z.record(z.string(), z.unknown()).optional(),
}).passthrough().optional(),
}).passthrough().nullable().optional(),
}).passthrough();
type ResponsesRequest = z.infer<typeof responsesRequestSchema>;
// Responses content parts → plain text. input_text / output_text both carry
// `text`; other part types (images, etc.) are dropped (parity with the proxy).
function partsToString(content: string | Array<{ type: string; text?: unknown }>): string {
if (typeof content === 'string') return content;
return content
.map((p) => (typeof p.text === 'string' ? p.text : ''))
.join('');
}
// Responses content parts → internal chat content. Text parts map to text
// blocks, image parts (Responses `input_image`, chat-style `image_url`/`image`,
// and computer-use `computer_screenshot`) map to `image_url` blocks so vision
// routing and the provider adapters see them (parity with /chat/completions).
// `refusal` parts (assistant history replay) fold into text so the turn isn't
// silently emptied. All-text content collapses back to a plain string (the
// shape upstream chat providers and compression expect); an array comes back
// only once a message actually carries an image.
function partsToChatContent(content: string | Array<{ type: string; [k: string]: unknown }>): string | Array<{ type: 'text'; text: string } | { type: 'image_url'; image_url: { url: string; detail?: string } }> {
if (typeof content === 'string') return content;
const blocks: Array<{ type: 'text'; text: string } | { type: 'image_url'; image_url: { url: string; detail?: string } }> = [];
for (const p of content) {
const type = p.type;
const text = p.text;
if (type === 'text' || type === 'input_text' || type === 'output_text' || type === 'summary_text' || (type === undefined && typeof text === 'string')) {
if (typeof text === 'string') blocks.push({ type: 'text', text });
continue;
}
if (type === 'refusal') {
// The model's refusal text on a replayed assistant turn; chat providers
// have no refusal concept, so carry it as ordinary text.
const refusal = (p as { refusal?: unknown }).refusal;
if (typeof refusal === 'string') blocks.push({ type: 'text', text: refusal });
continue;
}
if (type === 'input_image' || type === 'computer_screenshot' || type === 'image_url' || type === 'image') {
// The image lives under `image_url`, as a bare data URL string
// (Responses `input_image` / `computer_screenshot`) or as
// `{ url }` (chat-style `image_url`). The Responses `detail` hint
// (low/high/auto/original) rides along; adapters without an equivalent
// (Gemini inlineData) ignore it. Unresolvable shapes (missing/empty
// url, file_id-only) are rejected up front by the route's pre-check,
// so dropping here never answers blind.
const url = extractPartImageUrl(p);
if (url) {
const detail = (p as { detail?: unknown }).detail;
blocks.push({ type: 'image_url', image_url: { url, ...(typeof detail === 'string' ? { detail } : {}) } });
}
continue;
}
}
if (blocks.every((b) => b.type === 'text')) {
return blocks.map((b) => (b as { type: 'text'; text: string }).text).join('');
}
return blocks;
}
// Shared url extraction so the translation and the pre-check below always
// agree on what counts as a resolvable image.
function extractPartImageUrl(p: { [k: string]: unknown }): string | undefined {
const raw = p.image_url;
const url = typeof raw === 'string' ? raw : (raw as { url?: unknown } | undefined)?.url;
return typeof url === 'string' && url.length > 0 ? url : undefined;
}
// Responses `input_image` references an image by `image_url` OR a Files API
// `file_id`. This proxy has no OpenAI Files backend, and the schema is
// deliberately lenient, so an image part with no resolvable url (file_id-only,
// or no url at all) can't survive translation — reject up front instead of
// silently dropping it and answering blind to an image the client believes
// was sent. (`computer_screenshot` shares these forms but is intentionally
// not checked: computer use is rejected wholesale up front, today and until
// it's a supported feature.)
export function responsesInputHasFileIdImage(req: ResponsesRequest): boolean {
if (typeof req.input === 'string') return false;
return req.input.some((item) => {
const content = (item as { content?: unknown }).content;
if (!Array.isArray(content)) return false;
return content.some((p) => {
const part = p as { type?: string; [k: string]: unknown };
if (part.type !== 'input_image') return false;
return extractPartImageUrl(part) === undefined;
});
});
}
// Computer use (the Responses `computer` / `computer_use_preview` tool + its
// computer_call/computer_call_output items) has no chat-completions equivalent:
// the harness loop needs the model's computer_actions and screenshot context,
// neither of which survives translation. Fail clearly (mirroring the image 422)
// rather than silently dropping the calls and breaking the tool loop.
export function responsesInputRequestsComputerUse(req: ResponsesRequest): boolean {
if ((req.tools ?? []).some((t) => t.type === 'computer' || t.type === 'computer_use_preview')) return true;
if (typeof req.input === 'string' || req.input == null) return false;
return req.input.some((item) => {
const type = (item as { type?: string })?.type;
return type === 'computer_call' || type === 'computer_call_output';
});
}
// ── Translate a Responses request → internal chat messages + options ──────
export function toChatMessages(req: ResponsesRequest): ChatMessage[] {
const systemMessages: ChatMessage[] = [];
if (req.instructions) {
systemMessages.push({ role: 'system', content: req.instructions });
}
if (typeof req.input === 'string') {
const messages = [{ role: 'user' as const, content: req.input }];
return [...systemMessages, ...messages];
}
const messages: ChatMessage[] = [];
// Replayed web_search_call items (served by the gateway's search pool, or
// by OpenAI) become a short note on the assistant turn that follows, so the
// model keeps the context of what it searched without an extra assistant
// turn (consecutive assistant turns 400 on strict upstreams, #96).
let pendingSearchNotes: string[] = [];
const flushSearchNotes = () => {
if (pendingSearchNotes.length === 0) return;
messages.push({ role: 'assistant', content: pendingSearchNotes.join('\n') });
pendingSearchNotes = [];
};
const items = req.input;
for (let i = 0; i < items.length; i++) {
const item = items[i];
if ('type' in item && item.type === 'web_search_call') {
const note = webSearchCallToText(item as Record<string, unknown>);
if (note) pendingSearchNotes.push(note);
continue;
}
if (!('type' in item && item.type === 'message' && (item as { role?: string }).role === 'assistant')) {
flushSearchNotes();
}
if ('type' in item && item.type === 'function_call') {
// Parallel tool calls replay as consecutive standalone function_call
// items (no preceding assistant message item). Fold the run into one
// assistant turn, for the same reason as the message+function_call
// merge below: consecutive assistant turns 400 on strict upstreams.
const toolCalls: ChatToolCall[] = [];
let j = i;
while (j < items.length && (items[j] as { type?: string }).type === 'function_call') {
const fc = items[j] as z.infer<typeof functionCallItemSchema>;
toolCalls.push({
id: fc.call_id,
type: 'function',
function: { name: fc.name, arguments: fc.arguments },
});
j++;
}
messages.push({ role: 'assistant', content: null, tool_calls: toolCalls });
i = j - 1;
continue;
}
if ('type' in item && item.type === 'function_call_output') {
const output = typeof item.output === 'string'
? item.output
: Array.isArray(item.output)
? partsToString(item.output as any)
: JSON.stringify(item.output);
messages.push({ role: 'tool', tool_call_id: item.call_id, content: output });
continue;
}
if ('type' in item && item.type !== 'message') {
// computer_call / computer_call_output / reasoning / local_shell_call:
// no chat-message equivalent (the route 422s computer use up front).
// Skip rather than mis-parse as a message item.
continue;
}
// message item
const m = item as z.infer<typeof messageItemSchema>;
// 'developer' is the Responses-era system role.
const role = m.role === 'developer' ? 'system' : m.role;
let content = partsToChatContent(m.content);
if (role === 'system') {
// Hoist system/developer messages to the start of the conversation:
// chat providers (Gemini, Claude, Mistral) reject a system message that
// appears after a user turn. Codex history replay often emits developer
// items mid-conversation.
systemMessages.push({ role: 'system', content });
continue;
}
if (role === 'assistant') {
const searchNotes = pendingSearchNotes;
pendingSearchNotes = [];
if (searchNotes.length > 0) {
const notes = searchNotes.join('\n');
content = typeof content === 'string'
? (content.length > 0 ? `${notes}\n\n${content}` : notes)
: [{ type: 'text', text: notes }, ...content];
}
// A Responses assistant turn is a message item followed by its
// function_call items. Merge them into a single chat assistant message
// (content + tool_calls); emitting consecutive assistant turns makes
// Gemini map them to consecutive model turns and strict upstreams
// (Mistral/Cohere) 400. Drop empty assistant items — an empty turn means
// nothing to a chat provider. (#96)
const toolCalls: ChatToolCall[] = [];
let j = i + 1;
while (j < items.length && (items[j] as { type?: string }).type === 'function_call') {
const fc = items[j] as z.infer<typeof functionCallItemSchema>;
toolCalls.push({
id: fc.call_id,
type: 'function',
function: { name: fc.name, arguments: fc.arguments },
});
j++;
}
if (toolCalls.length > 0) {
messages.push({
role: 'assistant',
content: content.length > 0 ? content : null,
tool_calls: toolCalls,
});
i = j - 1;
continue;
}
if (content.length === 0) continue;
messages.push({ role: 'assistant', content });
continue;
}
messages.push({ role, content });
}
flushSearchNotes();
return [...systemMessages, ...messages];
}
export function toChatTools(tools?: ResponsesRequest['tools']): ChatToolDefinition[] | undefined {
if (!tools?.length) return undefined;
// Forward only function tools — chat-completions upstreams reject other
// Responses-API tool types (web_search, local_shell, etc.). Codex sends those
// extras alongside its function tools (shell/exec, apply_patch); dropping them
// keeps the request valid without losing the tools that actually do the work.
const fns = tools.filter((t): t is typeof t & { name: string } => t.type === 'function' && typeof t.name === 'string');
if (!fns.length) return undefined;
return fns.map((t) => ({
type: 'function',
function: {
name: t.name,
...(t.description ? { description: t.description } : {}),
...(t.parameters ? { parameters: t.parameters } : {}),
...(t.strict != null ? { strict: t.strict } : {}),
},
}));
}
export function toChatToolChoice(tc?: ResponsesRequest['tool_choice']): ChatToolChoice | undefined {
if (!tc) return undefined;
if (typeof tc === 'string') return tc;
return { type: 'function', function: { name: tc.name } };
}
function requestDeclaresToolUse(req: ResponsesRequest): boolean {
return (req.tools?.length ?? 0) > 0 && req.tool_choice !== 'none';
}
// ── Build the final (non-stream) Responses object ─────────────────────────
/**
* Responses-API `incomplete_details.reason` for an upstream chat finish
* reason, or undefined when the response ran to completion.
*/
export function incompleteReason(finishReason: string | null | undefined): 'max_output_tokens' | 'content_filter' | undefined {
if (finishReason === 'length') return 'max_output_tokens';
if (finishReason === 'content_filter') return 'content_filter';
return undefined;
}
export function buildResponseObject(opts: {
id: string;
model: string;
text: string;
toolCalls: ChatToolCall[];
promptTokens: number;
completionTokens: number;
reasoningTokens?: number;
metadata?: Record<string, string>;
incompleteReason?: 'max_output_tokens' | 'content_filter';
}) {
const output: any[] = [];
if (opts.text.length > 0) {
output.push({
type: 'message',
id: newId('msg'),
status: opts.incompleteReason ? 'incomplete' : 'completed',
role: 'assistant',
content: [{ type: 'output_text', text: opts.text, annotations: [] }],
});
}
for (const tc of opts.toolCalls) {
output.push({
type: 'function_call',
id: newId('fc'),
call_id: tc.id,
name: tc.function.name,
arguments: tc.function.arguments,
status: 'completed',
});
}
return {
id: opts.id,
object: 'response',
created_at: nowUnix(),
status: opts.incompleteReason ? 'incomplete' : 'completed',
...(opts.incompleteReason ? { incomplete_details: { reason: opts.incompleteReason } } : {}),
model: opts.model,
output,
output_text: opts.text,
usage: {
input_tokens: opts.promptTokens,
input_tokens_details: { cached_tokens: 0 },
output_tokens: opts.completionTokens,
output_tokens_details: { reasoning_tokens: opts.reasoningTokens ?? 0 },
total_tokens: opts.promptTokens + opts.completionTokens,
},
...(opts.metadata ? { metadata: opts.metadata } : {}),
};
}
/**
* Translate the chat-shaped result returned by the Fusion service into the
* Responses object clients expect. Chat Fusion uses additive `_fusion` and
* `x_fusion` fields, but those are not part of the Responses schema and strict
* SDKs (including Codex) reject them while decoding `response.completed`.
* Preserve the portable routing summary in the standard string-valued
* `metadata` map instead; the chat surface keeps its richer fields unchanged.
*/
function buildFusionResponseObject(id: string, result: FusionResult): Record<string, unknown> {
const fusionResponse = result.response as FusionResult['response'] & {
_fusion?: unknown;
x_fusion?: unknown;
};
const message = fusionResponse.choices?.[0]?.message;
const usage = fusionResponse.usage;
const response = buildResponseObject({
id,
model: FUSION_MODEL_ID,
text: contentToString(message?.content ?? ''),
toolCalls: message?.tool_calls ?? [],
promptTokens: usage?.prompt_tokens ?? 0,
completionTokens: usage?.completion_tokens ?? 0,
reasoningTokens: usage?.completion_tokens_details?.reasoning_tokens,
metadata: fusionResponseMetadata(fusionResponse),
}) as Record<string, unknown>;
return response;
}
function fusionResponseMetadata(result: FusionResult['response'] & { _fusion?: unknown; x_fusion?: unknown }): Record<string, string> {
const metadata: Record<string, string> = { fusion: 'true' };
const summary = result._fusion && typeof result._fusion === 'object' && !Array.isArray(result._fusion)
? result._fusion as Record<string, unknown>
: undefined;
const details = result.x_fusion && typeof result.x_fusion === 'object' && !Array.isArray(result.x_fusion)
? result.x_fusion as Record<string, unknown>
: undefined;
if (Array.isArray(summary?.panel)) {
const panel = summary.panel
.map((entry) => {
if (!entry || typeof entry !== 'object') return null;
const item = entry as Record<string, unknown>;
return typeof item.platform === 'string' && typeof item.model === 'string'
? `${item.platform}/${item.model}`
: null;
})
.filter((entry): entry is string => entry !== null);
if (panel.length > 0) metadata.fusion_panel = panel.join(',');
}
if (summary && 'judge' in summary) {
const judge = summary.judge;
if (judge && typeof judge === 'object') {
const item = judge as Record<string, unknown>;
if (typeof item.platform === 'string' && typeof item.model === 'string') {
metadata.fusion_judge = `${item.platform}/${item.model}`;
}
} else {
metadata.fusion_judge = 'none';
}
}
if (typeof summary?.synthesized === 'boolean') metadata.fusion_synthesized = String(summary.synthesized);
if (typeof details?.strategy === 'string') metadata.fusion_strategy = details.strategy;
return metadata;
}
/** Apply the same structured-output contract to Fusion as the chat route. */
function enforceFusionResponseFormat(result: FusionResult, responseFormat: ResponseFormat | undefined): void {
if (!responseFormat) return;
const message = result.response.choices?.[0]?.message as any;
if (!message || message.tool_calls?.length) return;
const text = contentToString(message.content ?? '');
if (!text) return;
const enforced = enforceJsonContent(text);
if (!enforced.ok) {
throw new Error(`fusion produced non-JSON output despite response_format=${responseFormat.type} — retry, or pin a structured-output-capable model instead of "fusion"`);
}
if (enforced.healed) message.content = enforced.content;
}
function quotaContextForRoute(route: RouteResult, endpoint: string): QuotaObservationContext {
return {
platform: route.platform as Platform,
keyId: route.keyId,
modelId: route.modelId,
quotaPoolKey: inferQuotaPoolKey(route.platform as Platform, route.modelId),
endpoint,
origin: 'responses',
};
}
// A `web_search` tool is served through the search loop
// (lib/responses-web-search.ts) when a search key exists; it re-enters
// handleResponses once per model turn. Everything else goes straight through.
responsesRouter.post('/responses', async (req: Request, res: Response) => {
if (await maybeServeResponsesWebSearch(handleResponses, req, res)) return;
await handleResponses(req, res);
});
export async function handleResponses(req: Request, res: Response): Promise<void> {
const start = Date.now();
const requestGroupId = getRequestGroupId(req);
res.setHeader('X-Request-ID', requestGroupId);
// Same auth as the proxy (accepts Bearer or x-api-key): unified key, or a
// client-profile key carrying a server-enforced system prompt (#411).
const auth = resolveAuth(extractApiToken(req));
if (!auth) {
res.status(401).json({ error: { message: 'Invalid API key', type: 'authentication_error' } });
return;
}
const parsed = responsesRequestSchema.safeParse(req.body);
if (!parsed.success) {
res.status(400).json({
error: {
message: `Invalid request: ${parsed.error.errors.map((e) => `${e.path.join('.')}: ${e.message}`).join(', ')}`,
type: 'invalid_request_error',
},
});
return;
}
const reqData = parsed.data;
// Computer use can't survive the chat-completions translation either (no
// computer tool, no screenshot context). Fail clearly instead of silently
// dropping the calls and breaking Codex's computer-use tool loop.
if (responsesInputRequestsComputerUse(reqData)) {
res.status(422).json({
error: {
message: 'Computer use is not yet supported on /v1/responses (the computer / computer_use_preview tool and computer_call items have no chat-completions equivalent).',
type: 'invalid_request_error',
code: 'no_computer_use_model',
},
});
return;
}
// Unresolvable image parts (file_id-only, or no usable url) can't survive
// translation — fail clearly rather than dropping the image and answering
// blind.
if (responsesInputHasFileIdImage(reqData)) {
res.status(422).json({
error: {
message: "This request contains an image part that can't be resolved: input_image needs an image_url (an https URL or a base64 data URL); Files API file_id references aren't supported by this proxy.",
type: 'invalid_request_error',
code: 'unsupported_image_input',
},
});
return;
}
const stream = reqData.stream ?? false;
let messages = toChatMessages(reqData);
const tools = toChatTools(reqData.tools);
// name → parameter schema, for repairing double-encoded tool arguments on
// the way back out (see lib/tool-args.ts).
const toolSchemas = toolSchemaMap(tools);
const tool_choice = tools?.length ? toChatToolChoice(reqData.tool_choice) : undefined;
// Responses-API structured output arrives as `text.format`; translate it to
// the internal response_format shape (an explicit response_format on the
// body, unusual for this surface but valid, wins).
const samplingParams = pickSamplingParams(reqData);
const textFormat = reqData.text?.format;
if (!samplingParams.response_format && textFormat && textFormat.type !== 'text') {
samplingParams.response_format = textFormat.type === 'json_schema'
? { type: 'json_schema', json_schema: { name: textFormat.name, strict: textFormat.strict, schema: textFormat.schema } }
: { type: 'json_object' } as ResponseFormat;
}
const completionOpts = {
temperature: reqData.temperature ?? undefined,
max_tokens: reqData.max_output_tokens ?? undefined,
top_p: reqData.top_p ?? undefined,
tools,
tool_choice,
parallel_tool_calls: reqData.parallel_tool_calls ?? undefined,
...samplingParams,
};
const hasCacheControl = typeof reqData.input !== 'string' && reqData.input.some(item => {
const content = (item as { content?: unknown }).content;
return Array.isArray(content)
&& content.some(block => block && typeof block === 'object' && 'cache_control' in block);
});
const compressionResult = compressRequest(messages, {
header: req.headers['x-freellm-compress'],
tools,
cacheControlPrefixLength: hasCacheControl ? messages.length : 0,
});
messages = compressionResult.messages;
res.setHeader('X-FreeLLM-Compress', formatCompressionHeader(compressionResult));
// Server-enforced system prompt (#411): after compression so it is never
// compressed away, first in the list so the caller's own instructions
// (`instructions` / system input items) follow it and cannot override it.
messages = prependSystemPrompt(messages, auth.systemPrompt);
// Downscale over-threshold inline images before estimation/routing so the
// token budget, payload limits, and upstream transfer all see the shrunk
// bytes (see lib/image-normalize.ts). Mutates the image blocks in place.
await normalizeMessageImages(messages);
const estimatedInputTokens = estimateInputTokens(messages, tools);
// Image requests must route to a vision-capable model (mirrors
// /chat/completions, proxy.ts). Reject up front with a clear message when
// none is enabled; when vision models are available, requireVision routing
// skips text-only models — including a pinned/sticky one — and falls back to
// a vision-capable peer (#118, #125). A rough per-image token cost keeps
// budget routing from being skewed by content the text heuristic can't see.
const hasImage = messageHasImage(messages);
if (hasImage && !hasEnabledVisionModel()) {
res.status(422).json({
error: {
message: 'This request includes an image, but no vision-capable model is enabled. Enable a vision model (e.g. Gemini 2.5 Flash, Llama 4 Scout) in the Fallback Chain.',
type: 'invalid_request_error',
code: 'no_vision_model',
},
});
return;
}
const IMAGE_TOKEN_ESTIMATE = 1000;
const imageCount = messages.reduce((n, m) =>
n + (Array.isArray(m.content) ? m.content.filter(b => (b as { type?: string })?.type === 'image_url' || (b as { type?: string })?.type === 'image').length : 0), 0);
// Capped output reserve so a large max_output_tokens can't falsely exclude the
// model pool (#470); input counts in full. Threaded to the router separately:
// it is exact and must not be inflated by the context-window safety margin
// (#956 review).
const outputReserve = routingReserveTokens(reqData.max_output_tokens);
const estimatedTotal = estimatedInputTokens + imageCount * IMAGE_TOKEN_ESTIMATE + outputReserve;
// Guardrail: per-request token budget (request_max_tokens_budget, default
// off). A request with no max_output_tokens gets its output capped to the
// budget remainder instead of a rejection.
const budgetCheck = applyTokenBudget(estimatedInputTokens + imageCount * IMAGE_TOKEN_ESTIMATE, completionOpts.max_tokens);
if (budgetCheck.rejection) {
res.status(413).json({
error: { message: tokenBudgetMessage(budgetCheck.rejection), type: 'invalid_request_error', code: 'request_token_budget' },
});
return;
}
completionOpts.max_tokens = budgetCheck.maxTokens;
// Optional client-managed session affinity (mirrors /chat/completions).
const rawSessionId = req.headers['x-codex-session-id']
?? req.headers['session-id']
?? req.headers['x-session-id'];
const sessionIdHeader = Array.isArray(rawSessionId) ? rawSessionId[0] : rawSessionId;
const requestedModelLabel = reqData.model ?? 'auto';
// What routing acts on. Usually the id the client sent; a Claude alias from
// GET /v1/models routes through the Claude family map instead (a pinned
// model, auto, or a tier pool), the same as on /v1/messages.
const claudeAlias = resolveListedClaudeAlias(reqData.model);
const routedModel = claudeAlias?.kind === 'model' ? claudeAlias.modelId : claudeAlias ? 'auto' : requestedModelLabel;
const aliasTierChain = claudeAlias?.kind === 'tier' ? resolveModelGroupCandidates(claudeAlias.dbIds) : [];
// Explicit `model` field pins routing. If the catalog has no enabled row
// matching the requested id, return 400 — silently auto-routing to a
// different model would be surprising to Responses API clients.
// Priority: explicit model > sticky session > auto routing.
let preferredModel: number | undefined;
let groupChain: ChainRow[] | undefined;
let resolvedChain: ResolvedChain | undefined;
// Sticky scope for a named auto:<profile> (or auto:smart) chain. Plain `auto`
// stays on the unscoped key so existing session affinity is unchanged.
let stickyStrategyKey: string | undefined;
if (aliasTierChain.length > 0) {
// Strict over the tier's pool; an empty tier falls through to plain auto.
groupChain = aliasTierChain;
stickyStrategyKey = `tier:${aliasTierChain[0]!.model_db_id}`;
const sticky = getStickyModel(messages, sessionIdHeader, stickyStrategyKey);
preferredModel = (sticky != null && groupChain.some(r => r.model_db_id === sticky)) ? sticky : undefined;
} else if (isAutoModel(routedModel)) {
try {
resolvedChain = resolveRoutingChain(routedModel);
} catch (err: any) {
if (err?.status === 400) {
res.status(400).json({
error: { message: err.message, type: 'invalid_request_error' },
});
return;
}
throw err;
}
stickyStrategyKey = resolvedChain.strategyKey === 'auto' ? undefined : resolvedChain.strategyKey;
preferredModel = resolveStickyPreference(
getStickyModel(messages, sessionIdHeader, stickyStrategyKey),
resolvedChain.chain,
);
} else if (isFusionModel(routedModel)) {
// Fusion is a virtual model, not a row in the catalog. Its panel and
// judge each resolve through the normal router inside runFusion below.
} else {
const db = getDb();
const resolved = isUnifyEnabled() ? resolveRequestedIdForDispatch(routedModel, getModelGroups()) : null;
const members = resolved?.memberDbIds ?? null;
if (members && members.length > 0) {
groupChain = resolveModelGroupCandidates(members, resolved!.demotedDbIds);
if (groupChain.length === 0) {
const placeholders = members.map(() => '?').join(',');
const anyEnabled = db.prepare(`SELECT 1 FROM models WHERE id IN (${placeholders}) AND enabled = 1 LIMIT 1`).get(...members);
const reason = anyEnabled ? 'has no providers with an enabled key' : 'is disabled';
res.status(400).json({
error: {
message: `Model '${routedModel}' ${reason}. Use 'auto' (or omit the 'model' field) to auto-route, or call /v1/models for the available list.`,
type: 'invalid_request_error',
code: 'model_not_found',
},
});
return;
}
const sticky = getStickyModel(messages, sessionIdHeader, routedModel);
preferredModel = (sticky != null && groupChain.some(r => r.model_db_id === sticky)) ? sticky : undefined;
} else {
const enabled = db.prepare('SELECT id FROM models WHERE model_id = ? AND enabled = 1').get(routedModel) as { id: number } | undefined;
if (enabled) {
preferredModel = enabled.id;
} else {
const disabled = db.prepare('SELECT id FROM models WHERE model_id = ?').get(routedModel) as { id: number } | undefined;
const reason = disabled ? 'is disabled' : 'is not in the catalog';
res.status(400).json({
error: {
message: `Model '${routedModel}' ${reason}. Use 'auto' (or omit the 'model' field) to auto-route, or call /v1/models for the available list.`,
type: 'invalid_request_error',
code: 'model_not_found',
},
});
return;
}
}
}
// Tool-bearing requests (the normal case for Codex/agent clients on this
// endpoint) must stay on models that emit structured tool_calls. Make the
// routing decision from the original Responses payload, not the subset of
// function tools we can forward to chat providers, because Codex may include
// built-in tool descriptors alongside or instead of function descriptors.
const wantsTools = requestDeclaresToolUse(reqData);
if (wantsTools && !hasEnabledToolsModel()) {
res.status(422).json({
error: {
message: 'This request includes tools, but no tool-capable model is enabled. Enable a tool-calling model (e.g. GPT-OSS 120B, Gemini 3.5 Flash, GLM-4.7) in the Fallback Chain.',
type: 'invalid_request_error',
code: 'no_tools_model',
},
});
return;
}
const responseId = newId('resp');
const state = newFallbackState();
// Lets the failover loop learn which models reject tool calls (#1230).
state.wantsTools = wantsTools;
const attemptLog: AttemptRecord[] = [];
// Client-disconnect fan-out: the flag stops the loop before the NEXT
// attempt; the AbortController (threaded to the provider as
// CompletionOptions.signal) additionally cancels the IN-FLIGHT upstream
// fetch and any body/stream read, so tokens stop burning and the in-flight
// lease frees immediately. 'close' also fires on normal completion —
// writableEnded distinguishes a real disconnect.
let clientGone = false;
const clientAbort = new AbortController();
// Fallback-v2 hedging: the loop aborts this controller (via abortInFlight)
// when the wall-clock retry budget expires mid-attempt, canceling the
// in-flight upstream instead of waiting for a stalled attempt to time out.
const hedgeAbort = new AbortController();
res.on('close', () => {
if (!res.writableEnded) {
clientGone = true;
clientAbort.abort(newClientAbortError());
}
});
const dispatchOpts = { ...completionOpts, signal: AbortSignal.any([clientAbort.signal, hedgeAbort.signal]) };
// ── Fusion: Responses-API virtual model ────────────────────────────────
// `fusion` is intentionally not a catalog row. The chat route handles this
// virtual id before normal model dispatch; do the same here after the common
// Responses translation/guardrails so Codex and other Responses clients can
// opt into the exact same panel + judge behavior.
if (isFusionModel(routedModel)) {
const fusionConfig = reqData.fusion ?? {};
const fusionOptions = { ...dispatchOpts };
if (!stream) {
try {
const result = await runFusion({
messages,
config: fusionConfig,
options: fusionOptions,
estimatedTokens: estimatedTotal,
vision: hasImage,
});
enforceFusionResponseFormat(result, samplingParams.response_format);
res.setHeader('X-Routed-Via', safeHeaderValue(result.routedVia));
res.json(buildFusionResponseObject(responseId, result));
} catch (err: any) {
if (err instanceof FusionError) {
res.status(err.status).json({
error: {
message: err.message,
type: err.status === 429 ? 'rate_limit_error' : err.status >= 500 ? 'server_error' : 'invalid_request_error',
...(err.code ? { code: err.code } : {}),
},
});
} else {
res.status(502).json({
error: {
message: `fusion error: ${sanitizeProviderErrorMessage(err?.message)}`,
type: 'server_error',
},
});
}
}
return;
}
// A streamed Fusion judge can emit deltas through runFusion's hook. Panel
// and best-of/single-survivor paths remain valid Responses streams too:
// they simply emit the completed answer once the fan-out settles.
res.setHeader('Content-Type', 'text/event-stream');
res.setHeader('Cache-Control', 'no-cache');
res.setHeader('Connection', 'keep-alive');
// The detailed panel is only known after runFusion settles, but the
// standard header can still identify this stream as Fusion before the
// response is committed. The final metadata carries panel/judge details.
res.setHeader('X-Routed-Via', FUSION_MODEL_ID);
let fusionSequence = 0;
const fusionSse = (event: string, payload: Record<string, unknown>) => {
if (res.writableEnded) return;
try {
res.write(`event: ${event}\n`);
res.write(`data: ${JSON.stringify({ type: event, sequence_number: fusionSequence++, ...payload })}\n\n`);
} catch {
// The close listener aborts the provider signal; a late write can still
// race the socket teardown and is intentionally ignored.
}
};
const skeleton = {
id: responseId,
object: 'response',
created_at: nowUnix(),
status: 'in_progress',
model: FUSION_MODEL_ID,
output: [],
output_text: '',
metadata: { fusion: 'true' },
};
fusionSse('response.created', { response: skeleton });
fusionSse('response.in_progress', { response: skeleton });
// Fusion fans out to several upstream models before the judge can emit a
// token. That gap can exceed the ~15s idle timeout used by Codex's
// Responses client even though the HTTP connection is healthy. A standard
// in-progress event keeps the client-side event watchdog alive; comments
// are ignored by Codex's watchdog even though they refresh curl's stream.
const fusionHeartbeat = setInterval(() => {
if (res.writableEnded || res.destroyed) return;
try {
fusionSse('response.in_progress', { response: skeleton });
} catch {
// The close listener aborts the in-flight fan-out; a late heartbeat
// racing socket teardown is harmless.
}
}, 5000);
fusionHeartbeat.unref?.();
let textItemId: string | null = null;
const textOutputIndex = 0;
let streamedText = '';
const openTextItem = (id: string = newId('msg')) => {
if (textItemId !== null) return;
textItemId = id;
fusionSse('response.output_item.added', {
output_index: textOutputIndex,
item: { id, type: 'message', status: 'in_progress', role: 'assistant', content: [] },
});
fusionSse('response.content_part.added', {
item_id: id,
output_index: textOutputIndex,
content_index: 0,
part: { type: 'output_text', text: '', annotations: [] },
});
};
try {
const result = await runFusion({
messages,
config: fusionConfig,
options: fusionOptions,
estimatedTokens: estimatedTotal,
vision: hasImage,
hooks: {
onJudgeDelta: (delta) => {
if (!delta) return;
openTextItem();
fusionSse('response.output_text.delta', {
item_id: textItemId,
output_index: textOutputIndex,
content_index: 0,
delta,
});
streamedText += delta;
},
},
});
// Like the chat route's streamed Fusion branch, do not run a structured
// output verdict after judge deltas have already reached the client: a
// failed verdict could not retract the partial answer. Non-streaming
// Fusion is checked above before the response is committed.
const responseObject = buildFusionResponseObject(responseId, result) as any;
const outputs = Array.isArray(responseObject.output) ? responseObject.output : [];
const messageOutput = outputs.find((item: any) => item?.type === 'message');
const finalText = typeof responseObject.output_text === 'string' ? responseObject.output_text : '';
if (messageOutput) {
// Keep the final Response object's message id aligned with any judge
// deltas already emitted on the wire.
if (textItemId === null) openTextItem(messageOutput.id);
else messageOutput.id = textItemId;
if (finalText && !streamedText) {
fusionSse('response.output_text.delta', {
item_id: textItemId,
output_index: textOutputIndex,
content_index: 0,
delta: finalText,
});
streamedText = finalText;
} else if (finalText.length > streamedText.length && finalText.startsWith(streamedText)) {
fusionSse('response.output_text.delta', {
item_id: textItemId,
output_index: textOutputIndex,
content_index: 0,
delta: finalText.slice(streamedText.length),
});
streamedText = finalText;
}
fusionSse('response.output_text.done', {
item_id: textItemId,
output_index: textOutputIndex,
content_index: 0,
text: finalText,
});
fusionSse('response.content_part.done', {
item_id: textItemId,
output_index: textOutputIndex,
content_index: 0,
part: { type: 'output_text', text: finalText, annotations: [] },
});
fusionSse('response.output_item.done', {
output_index: textOutputIndex,
item: messageOutput,
});
}
let outputIndex = messageOutput ? 1 : 0;
for (const output of outputs) {
if (output?.type !== 'function_call') continue;
const item = { ...output, status: 'in_progress', arguments: '' };
fusionSse('response.output_item.added', { output_index: outputIndex, item });
if (output.arguments) {
fusionSse('response.function_call_arguments.delta', {
item_id: output.id,
output_index: outputIndex,
delta: output.arguments,
});
}
fusionSse('response.function_call_arguments.done', {
item_id: output.id,
output_index: outputIndex,
arguments: output.arguments ?? '',
});
fusionSse('response.output_item.done', { output_index: outputIndex, item: output });
outputIndex++;
}
fusionSse('response.completed', { response: responseObject });
res.end();
} catch (err: any) {
const message = err instanceof FusionError
? err.message
: `fusion error: ${sanitizeProviderErrorMessage(err?.message)}`;
const type = err instanceof FusionError
? (err.status === 429 ? 'rate_limit_error' : err.status >= 500 ? 'server_error' : 'invalid_request_error')
: 'server_error';
const code = err instanceof FusionError ? err.code : undefined;
fusionSse('response.failed', {
response: {
...skeleton,
status: 'failed',
error: { message, type, ...(code ? { code } : {}) },
},
});
res.end();
} finally {
clearInterval(fusionHeartbeat);
}
return;
}
// Stream bookkeeping (used only when stream === true). `streamStarted` is the
// commit flag: true once the response.created/in_progress skeleton has left,
// after which failover is no longer possible. seq/streamStarted span attempts
// so the SSE sequence numbers stay monotonic and a committed stream can't be
// re-committed by a later attempt.
let seq = 0;
let streamStarted = false;
const sse = (event: string, payload: Record<string, unknown>) => {
res.write(`event: ${event}\n`);
res.write(`data: ${JSON.stringify({ type: event, sequence_number: seq++, ...payload })}\n\n`);
};
// Decision routing (#1354): with the 'decision' strategy on, a decision
// model picks which preset (Smartest / Fastest / Balanced / Most reliable)
// this turn routes with. Skipped when the client pinned a model or the
// conversation is sticky to one. Any doubt routes as Balanced.
const routingPick = await applyDecisionRouting(messages, { pinned: !isAutoModel(routedModel) || preferredModel != null, signal: clientAbort.signal });
if (routingPick && !res.headersSent) res.setHeader('X-FreeLLM-Routing-Strategy', routingStrategyHeader(routingPick));
await runFallbackLoop({
maxRetries: MAX_RETRIES,
state,
attemptLog,
logIdentity: { surface: 'responses', requestId: requestGroupId, requestedModel: requestedModelLabel },
clientGone: () => clientGone,
abortInFlight: () => hedgeAbort.abort(newHedgeAbortError()),
route: () => {
// Task-type routing (#1127): same header/derivation as /chat/completions.
const taskType = resolveTaskType(req, tools, messages);
// #507: inflate the routing estimate from any provider-reported REQUESTED
// size latched onto state so the existing size gates in router.ts skip
// low-TPM / small-context models on retry.
const routingTotal = fallbackRoutingTokens(state, estimatedTotal, outputReserve);
return routeRequest(routingTotal, state.skipKeys.size > 0 ? state.skipKeys : undefined, preferredModel, hasImage, wantsTools, state.skipModels.size > 0 ? state.skipModels : undefined, groupChain ?? resolvedChain?.chain, completionOpts.response_format !== undefined, state.skipPlatforms.size > 0 ? state.skipPlatforms : undefined, outputReserve, taskType);
},
dispatch: async (route, attempt, ctx) => {
const contextBudget = routeOutputBudget(route, estimatedInputTokens);
const routeOpts = contextBudget != null ? { ...dispatchOpts, contextBudget } : dispatchOpts;
traceRouteEvent('Responses', {
event: attempt === 0 ? 'start' : 'next',
requestId: requestGroupId,
attempt,
platform: route.platform,
model: route.modelId,
requestedModel: attempt === 0 ? requestedModelLabel : undefined,
});
if (stream) {
// Every output item (message text + each function_call) claims the next
// free index. OpenAI's streaming SDK indexes snapshot.output by
// output_index, so indices MUST be dense & unique — reusing 0 for the
// text item after a tool-call item had already taken it makes the SDK
// crash on `snapshot.output[output_index]` (#96, Codex computer-use).
let outputIndex = 0;
let msgItemId: string | null = null;
let msgText = '';
// output_index of the open text item (valid while msgItemId !== null).
let textOutputIndex = 0;
// tool-call accumulator keyed by the provider's tool_call index
const toolAcc = new Map<number, { outputIndex: number; itemId: string; callId: string; name: string; args: string }>();
let totalOutputTokens = 0;
// #764: thinking tokens are tracked separately so the final Response
// object can report `output_tokens_details.reasoning_tokens` truthfully
// instead of a hardcoded 0.
let totalReasoningTokens = 0;
// #764: ttfb = first token of ANY kind (content or reasoning), recorded
// in the pump loop; commit() only backfills streams that never produced
// one. This path previously logged no ttfb at all, so Analytics showed
// a null speed for every /v1/responses turn.
let ttfbMs: number | null = null;
// Inline-dialect hold window (#231): first text is held until it
// either matches a tool-call dialect marker (held to the end and
// rescued into function_call items) or provably cannot (flushed and
// streamed normally). Mirrors the /chat/completions stream loop.
let dialectMode: 'undecided' | 'passthrough' | 'dialect' = 'undecided';
let heldText = '';
let upstreamFinish: string | null = null;
// Commit point: headers + the response.created/in_progress skeleton go
// out only when the first MEANINGFUL output item is about to be emitted
// (converged with /chat/completions — responses previously committed on
// the first raw chunk, even a role-only one). Until then a connect-time
// error, an empty completion, or an unparseable dialect turn fails over
// on the same connection with no bytes on the wire. Idempotent.
const commit = () => {
if (streamStarted) return;
if (ttfbMs === null) ttfbMs = Date.now() - start;
sendSseHeaders();
const skeleton = {
id: responseId, object: 'response', created_at: nowUnix(),
status: 'in_progress', model: route.modelId, output: [], output_text: '',
};
sse('response.created', { response: skeleton });
sse('response.in_progress', { response: skeleton });
streamStarted = true;
// Committed: the answer is on its way, so the retry budget must no
// longer cancel this attempt (it could not fail over now anyway).
ctx.disarmHedge();
};
// Send the SSE headers once. Split out of commit so the #1642
// keep-alive can commit the response with a content-free ping while
// the hold window is still open, without sending the event skeleton
// early.
let headersSentEarly = false;
const sendSseHeaders = () => {
if (headersSentEarly) return;
res.setHeader('Content-Type', 'text/event-stream');
res.setHeader('Cache-Control', 'no-cache');
res.setHeader('Connection', 'keep-alive');
res.setHeader('X-Routed-Via', routedViaValue(route.platform, route.modelId));
setFallbackHeaders(res, attempt, attemptLog);
headersSentEarly = true;
};
// #1642: optional heartbeat for the silent reasoning window. A bare
// SSE comment resets client idle watchdogs without committing content.
let streamSettled = false;
const keepaliveStop = startStreamKeepalive(streamKeepaliveMs(),
() => streamSettled,
() => {
sendSseHeaders();
// Bytes on the wire = the response can no longer fail over
// invisibly; stop the retry budget from cancelling it.
ctx.disarmHedge();
res.write(': keepalive\n\n');
});
// Open the text output item and stream `text` as its first delta.
// The text item takes the next free output index (it is NOT always 0 —
// when the model emits tool_calls first, the function_call items own
// the low indices). Every later text delta/done must reference this
// same index or the SDK snapshot lookup misroutes the deltas.
const openTextItem = (text: string) => {
commit();
msgItemId = newId('msg');
textOutputIndex = outputIndex++;
sse('response.output_item.added', {
output_index: textOutputIndex,
item: { id: msgItemId, type: 'message', status: 'in_progress', role: 'assistant', content: [] },
});
sse('response.content_part.added', {
item_id: msgItemId, output_index: textOutputIndex, content_index: 0,
part: { type: 'output_text', text: '', annotations: [] },
});
if (text) {
sse('response.output_text.delta', { item_id: msgItemId, output_index: textOutputIndex, content_index: 0, delta: text });
msgText += text;
}
};
try {
const gen = route.provider.streamChatCompletion(
route.apiKey,
messages,
route.modelId,
routeOpts,
quotaContextForRoute(route, 'responses'),
);
for await (const chunk of gen) {
if (clientGone) break; // client hung up: stop pulling; reader.cancel() aborts upstream
// In-band upstream error frame ({"error":...} inside a 200 SSE
// stream — observed live from Groq). Throwing hands it to the catch
// below: pre-commit it fails over, post-commit it surfaces
// response.failed.
const anyChunk = chunk as Record<string, any>;
if (anyChunk.error && !anyChunk.choices) {
throw new Error(`in-band provider error from ${route.displayName}: ${anyChunk.error.message ?? 'provider error'}`);
}
const choice0 = chunk.choices?.[0];
if (choice0?.finish_reason) upstreamFinish = choice0.finish_reason;
const delta = choice0?.delta;
if (!delta) continue;
// Text deltas → output_text events on a single message item, after
// the dialect hold window has decided the text is real prose.
const text = delta.content ?? '';
// #764: reasoning models stream thinking before the first answer
// token — count that first token as ttfb so Analytics speed
// reflects the real head-of-stream, and count thinking tokens as
// real output consumption.
const reasoning = streamReasoningText(chunk);
if (ttfbMs === null && (text.length > 0 || reasoning.length > 0)) {
ttfbMs = Date.now() - start;
}
if (text) {
totalOutputTokens += Math.ceil((text.length + reasoning.length) / 4);
if (reasoning.length > 0) totalReasoningTokens += Math.ceil(reasoning.length / 4);
if (dialectMode === 'passthrough') {
if (msgItemId === null) openTextItem('');
sse('response.output_text.delta', {
item_id: msgItemId, output_index: textOutputIndex, content_index: 0, delta: text,
});
msgText += text;
} else {
heldText += text;
if (dialectMode === 'undecided') {
const probe = heldText.trimStart();
if (startsWithDialectMarker(probe)) {
dialectMode = 'dialect';
} else if (!couldBecomeDialectMarker(probe) || heldText.length > 256) {
dialectMode = 'passthrough';
openTextItem(heldText);
heldText = '';
}
}
}
} else if (reasoning.length > 0) {
// #764: thinking-only chunk (no visible text yet) — count tokens.
totalOutputTokens += Math.ceil(reasoning.length / 4);
totalReasoningTokens += Math.ceil(reasoning.length / 4);
}
// Tool-call deltas → function_call item + argument deltas.
for (const tc of delta.tool_calls ?? []) {
const idx = (tc as any).index ?? 0;
let acc = toolAcc.get(idx);
if (!acc) {
// First time we see this tool call: open a new output item.
commit();
if (msgItemId !== null && msgText.length > 0) {
// close the text item (at its own output index) before starting a function_call item
sse('response.output_text.done', { item_id: msgItemId, output_index: textOutputIndex, content_index: 0, text: msgText });
sse('response.content_part.done', { item_id: msgItemId, output_index: textOutputIndex, content_index: 0, part: { type: 'output_text', text: msgText, annotations: [] } });
sse('response.output_item.done', { output_index: textOutputIndex, item: { id: msgItemId, type: 'message', status: 'completed', role: 'assistant', content: [{ type: 'output_text', text: msgText, annotations: [] }] } });
msgItemId = null;
}
acc = { outputIndex: outputIndex++, itemId: newId('fc'), callId: tc.id || newId('call'), name: tc.function?.name ?? '', args: '' };
toolAcc.set(idx, acc);
sse('response.output_item.added', {
output_index: acc.outputIndex,
item: { id: acc.itemId, type: 'function_call', status: 'in_progress', call_id: acc.callId, name: acc.name, arguments: '' },
});
}
const argFrag = tc.function?.arguments ?? '';
if (tc.function?.name && !acc.name) acc.name = tc.function.name;
if (argFrag) {
acc.args += argFrag;
sse('response.function_call_arguments.delta', { item_id: acc.itemId, output_index: acc.outputIndex, delta: argFrag });
}
}
}
// Resolve the dialect hold window now that the full text is known.
// Held text was never emitted, so a dead dialect turn can still fail
// over on the same SSE stream (nothing has been committed yet).
if (heldText.length > 0) {
const rescue = (wantsTools && (dialectMode === 'dialect' || containsDialectMarker(heldText)))
? rescueInlineToolCalls(heldText, new Set((tools ?? []).map(t => t.function.name)))
: { detected: false as const, calls: null, cleanText: heldText };
if (rescue.detected && !rescue.calls) {
// Unparseable dialect turn: throw so the shared loop cooldowns this
// model+key and fails over (streamStarted is still false).
throw new Error(`unparseable inline tool-call dialect from ${route.displayName}: ${heldText.slice(0, 120)}`);
}
if (rescue.detected && rescue.calls) {
// Rescued calls become function_call items, exactly as if the
// provider had streamed them structurally.
console.log(`[Responses] Rescued ${rescue.calls.length} inline tool call(s) from ${route.displayName}`);
if (rescue.cleanText.length > 0 && msgItemId === null) openTextItem(rescue.cleanText);
let rescuedIdx = 0;
for (const c of rescue.calls) {
const idx = 1000 + rescuedIdx++; // synthetic accumulator keys, past any provider index
commit();
const acc = {
outputIndex: outputIndex++,
itemId: newId('fc'), callId: newId('call'), name: c.name, args: c.arguments,
};
toolAcc.set(idx, acc);
sse('response.output_item.added', {
output_index: acc.outputIndex,
item: { id: acc.itemId, type: 'function_call', status: 'in_progress', call_id: acc.callId, name: acc.name, arguments: '' },
});
}
} else if (msgItemId === null) {
// Plain short answer that never left the hold window (e.g. "Hi").
openTextItem(heldText);
}
heldText = '';
}
// Empty completion — the provider returned 200 with no text AND no
// tool calls. Seen in production from nemotron-3-super on ~65k-token
// contexts: transport-level "success", zero usable output. Nothing has
// been committed yet (the skeleton is lazy), so throwing lets the
// shared loop fail over to the next model on the same SSE connection.
if (msgText.length === 0 && toolAcc.size === 0) {
// Disconnect before the commit point: the break above fired with
// nothing accumulated — CLIENT behavior, not a provider failure.
// Falling through to the empty-completion throw benched a healthy
// model+key for 90s on every Ctrl-C during a reasoning model's
// TTFB window.
if (clientGone && !streamStarted) {
console.log(`[Responses] client disconnected before first token from ${route.displayName} — dropping attempt without benching`);
return 'committed';
}
// finish_reason 'length' = the model spent the whole output budget
// on hidden reasoning before any visible text: fail over, but skip
// the cooldown/penalty (not a provider-health signal).
throw Object.assign(
new Error(`empty completion from ${route.displayName}`),
upstreamFinish === 'length' ? { skipBench: true } : {},
);
}
// A truncated or filtered turn ends as `incomplete`, not `completed`,
// so clients don't take a cut-off answer as the whole answer.
const incomplete = incompleteReason(upstreamFinish);
// Finalize any open text item.
if (msgItemId !== null) {
sse('response.output_text.done', { item_id: msgItemId, output_index: textOutputIndex, content_index: 0, text: msgText });
sse('response.content_part.done', { item_id: msgItemId, output_index: textOutputIndex, content_index: 0, part: { type: 'output_text', text: msgText, annotations: [] } });
sse('response.output_item.done', { output_index: textOutputIndex, item: { id: msgItemId, type: 'message', status: incomplete ? 'incomplete' : 'completed', role: 'assistant', content: [{ type: 'output_text', text: msgText, annotations: [] }] } });
}
// Finalize tool-call items. Arguments are repaired against the tool's
// parameter schema at this point (after the full string accumulated):
// models like GLM double-encode nested arrays/objects as strings, and
// Codex hard-rejects the call ("invalid type: string, expected a
// sequence"). Clients consume the *.done events / final response for
// arguments, so repairing here covers the streamed path too.
//
// No schema verdict here, deliberately. This surface commits on the
// FIRST tool-call delta (`commit()` above, where the function_call
// item is opened) — the arguments are not complete until this point,
// which is long after the skeleton and the item.added events have
// left. A verdict here could only turn a delivered-but-invalid call
// into a `response.failed` on a stream the client is already reading,
// which is strictly worse. Wiring it would mean holding the
// function_call item until its arguments finish, the way
// /chat/completions buffers — a change to this route's commit point,
// not to validation.
const finalToolCalls: ChatToolCall[] = [];
for (const acc of toolAcc.values()) {
const repairedArgs = repairToolArguments(acc.args, toolSchemas.get(acc.name));
sse('response.function_call_arguments.done', { item_id: acc.itemId, output_index: acc.outputIndex, arguments: repairedArgs });
sse('response.output_item.done', { output_index: acc.outputIndex, item: { id: acc.itemId, type: 'function_call', status: 'completed', call_id: acc.callId, name: acc.name, arguments: repairedArgs } });
finalToolCalls.push({ id: acc.callId, type: 'function', function: { name: acc.name, arguments: repairedArgs } });
}
const finalResponse = buildResponseObject({
id: responseId, model: route.modelId, text: msgText,
toolCalls: finalToolCalls, promptTokens: estimatedInputTokens, completionTokens: totalOutputTokens,
reasoningTokens: totalReasoningTokens, incompleteReason: incomplete,
});
sse(incomplete ? 'response.incomplete' : 'response.completed', { response: finalResponse });
res.end();
recordUpstreamSuccess(route, estimatedInputTokens + totalOutputTokens, state);
setStickyModel(messages, route.modelDbId, sessionIdHeader, stickyStrategyKey);
traceRouteEvent('Responses', {
event: 'ok',
requestId: requestGroupId,
attempt,
platform: route.platform,
model: route.modelId,
latencyMs: Date.now() - start,
inputTokens: estimatedInputTokens,
outputTokens: totalOutputTokens,
});
logRequest(route.platform, route.modelId, route.keyId, 'success', estimatedInputTokens, totalOutputTokens, Date.now() - start, null, ttfbMs, null, null, 'http');
return 'done';
} catch (streamErr: any) {
// Client abort mid-stream: the pump's own `if (clientGone) break`
// can lose the race against the fetch-signal rejection, so the
// abort may surface here instead. Rethrow — the shared loop's
// client-abort branch stops the ladder without benching or an
// error log row (the socket is gone; nothing to render).
if (isClientAbortError(streamErr)) throw streamErr;
// A committed stream can't fail over (bytes already sent) — surface a
// response.failed event honestly and stop. A pre-commit failure throws
// through to the shared loop for cooldown + failover.
// streamStarted = real events went out. headersSentEarly also counts:
// a #1642 keep-alive ping flushed the SSE headers, so this attempt is
// on the wire and can no longer fail over invisibly.
if (streamStarted || headersSentEarly) {
const safe = sanitizeProviderErrorMessage(streamErr.message);
traceRouteEvent('Responses', {
event: 'fail',
requestId: requestGroupId,
attempt,
platform: route.platform,
model: route.modelId,
latencyMs: Date.now() - start,
error: safe,
});
logRequest(route.platform, route.modelId, route.keyId, 'error', estimatedInputTokens, 0, Date.now() - start, safe, ttfbMs, null, null, 'http', failureAttributionFor(streamErr));
sse('response.failed', { response: { id: responseId, object: 'response', status: 'failed', error: { message: `Provider error (${route.displayName}): stream interrupted`, type: 'stream_error' } } });
res.end();
return 'committed';
}
throw streamErr;
} finally {
// #1642: this attempt is over whichever path it left by — the
// keep-alive interval must not outlive it.
streamSettled = true;
keepaliveStop();
}
}
const result = await route.provider.chatCompletion(
route.apiKey,
messages,
route.modelId,
routeOpts,
quotaContextForRoute(route, 'responses'),
);
const msg = result.choices[0]?.message;
let text = contentToString(msg?.content ?? '');
let toolCalls = (msg?.tool_calls ?? []).map((tc) => ({
...tc,
function: { ...tc.function, arguments: repairToolArguments(tc.function.arguments, toolSchemas.get(tc.function.name)) },
}));
// Inline tool-call dialect rescue (#231) — see /chat/completions.
if (wantsTools && toolCalls.length === 0 && text) {
const rescue = rescueInlineToolCalls(text, new Set((tools ?? []).map(t => t.function.name)));
if (rescue.detected) {
if (!rescue.calls) {
throw new Error(`unparseable inline tool-call dialect from ${route.displayName}: ${text.slice(0, 120)}`);
}
console.log(`[Responses] Rescued ${rescue.calls.length} inline tool call(s) from ${route.displayName}`);
toolCalls = rescue.calls.map((c, i) => ({
id: `call_rescued_${i + 1}`,
type: 'function' as const,
function: { name: c.name, arguments: repairToolArguments(c.arguments, toolSchemas.get(c.name)) },
}));
text = rescue.cleanText;
}
}
// Opt-in schema verdict on what the repair could not fix. Thrown before
// anything is written, so the failover hop is invisible to the client.
if (isToolArgumentValidationEnabled() && toolCalls.length > 0) {
const invalid = invalidToolCallReasons(toolCalls, toolSchemas);
if (invalid.length > 0) throw invalidToolArgumentsError(route.displayName, invalid);
}
const promptTokens = result.usage?.prompt_tokens ?? estimatedInputTokens;
// #764: include reasoning tokens (message.reasoning_content / reasoning)
// in the chars/4 estimate so thinking models aren't undercounted when the
// provider omits `usage`.
const completionTokens = result.usage?.completion_tokens
?? Math.ceil((text.length + completionReasoningText(result).length) / 4);
// #764: report reasoning_tokens truthfully — the provider's own count
// when advertised, else the same chars/4 estimate of the thinking text.
const reasoningTokens = result.usage?.completion_tokens_details?.reasoning_tokens
?? Math.ceil(completionReasoningText(result).length / 4);
// Empty completion → fail over via the shared loop (see the streaming
// path); finish_reason 'length' skips the cooldown/penalty.
if (!text && toolCalls.length === 0) {
throw Object.assign(
new Error(`empty completion from ${route.displayName}`),
result.choices[0]?.finish_reason === 'length' ? { skipBench: true } : {},
);
}
// Structured-output enforcement — see /chat/completions. Heal fenced or
// prose-wrapped JSON in place, fail over (skipBench + skipModelForRequest
// — a sibling key would misbehave identically) when the model ignored the
// requested format; finish_reason 'length' gets an honest "truncated"
// class instead, since the JSON was cut off by max_tokens, not ignored.
if (completionOpts.response_format && text && toolCalls.length === 0) {
const enforced = enforceJsonContent(text);
if (!enforced.ok) {
const truncated = result.choices[0]?.finish_reason === 'length';
throw Object.assign(
new Error(truncated
? `truncated JSON from ${route.displayName} (finish_reason=length — raise max_tokens for this ${completionOpts.response_format.type} request)`
: `${route.displayName} ignored response_format (returned non-JSON despite ${completionOpts.response_format.type})`),
{ skipBench: true, skipModelForRequest: true },
);
}
if (enforced.healed) text = enforced.content;
}
// Usage fallback: a missing provider `usage` block used to record 0
// tokens against the rate-limit ledger; promptTokens/completionTokens
// above already carry the chars/4 estimate.
recordUpstreamSuccess(route, result.usage?.total_tokens ?? (promptTokens + completionTokens), state);
setStickyModel(messages, route.modelDbId, sessionIdHeader, stickyStrategyKey);
res.setHeader('X-Routed-Via', routedViaValue(route.platform, route.modelId));
setFallbackHeaders(res, attempt, attemptLog);
res.json(buildResponseObject({
id: responseId, model: route.modelId, text, toolCalls,
promptTokens, completionTokens, reasoningTokens,
incompleteReason: incompleteReason(result.choices[0]?.finish_reason),
}));
traceRouteEvent('Responses', {
event: 'ok',
requestId: requestGroupId,
attempt,
platform: route.platform,
model: route.modelId,
latencyMs: Date.now() - start,
inputTokens: promptTokens,
outputTokens: completionTokens,
});
logRequest(route.platform, route.modelId, route.keyId, 'success', promptTokens, completionTokens, Date.now() - start, null, null, null, null, 'http');
return 'done';
},
logFailure: (route, err, attempt) => {
const latency = Date.now() - start;
const safeError = sanitizeProviderErrorMessage(err.message);
traceRouteEvent('Responses', {
event: 'fail',
requestId: requestGroupId,
attempt,
platform: route.platform,
model: route.modelId,
latencyMs: latency,
error: safeError,
});
logRequest(route.platform, route.modelId, route.keyId, 'error', estimatedInputTokens, 0, latency, safeError, null, null, null, 'http', failureAttributionFor(err));
},
onFatal: (route, err, attempt) => {
setFallbackHeaders(res, attempt, attemptLog);
res.status(502).json({ error: { message: `Provider error (${route.displayName}): ${sanitizeProviderErrorMessage(err.message)}`, type: 'provider_error' } });
},
onRoutingExhausted: (lastError, routeErr, exhaustion, info) => {
if (streamStarted) {
// Headers are already on the wire — the honest status/Retry-After can
// only travel in the failed-event payload.
sse('response.failed', { response: { id: responseId, object: 'response', status: 'failed', error: exhaustionErrorPayload(exhaustion) } });
res.end();
} else {
setFallbackHeaders(res, info.attempts.length, info.attempts);
setExhaustionHeaders(res, exhaustion);
res.status(exhaustion.status).json({ error: exhaustionErrorPayload(exhaustion) });
}
},
onExhausted: (exhaustion, info) => {
// The streaming skeleton may already be on the wire — close the SSE stream
// with a failed event instead of writing JSON onto a committed response.
if (streamStarted) {
sse('response.failed', { response: { id: responseId, object: 'response', status: 'failed', error: exhaustionErrorPayload(exhaustion) } });
res.end();
} else {
setFallbackHeaders(res, info.attempts.length, info.attempts);
setExhaustionHeaders(res, exhaustion);
res.status(exhaustion.status).json({ error: exhaustionErrorPayload(exhaustion) });
}
},
});
}