7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.
TypeScript
Add chronological fallback logging for chat and responses (#313)
JammyJames1234a7e9a28c5596
diff --git a/server/src/__tests__/routes/proxy-empty-completion.test.ts b/server/src/__tests__/routes/proxy-empty-completion.test.tsindex 77411fa2..9405763c 100644--- a/server/src/__tests__/routes/proxy-empty-completion.test.ts+++ b/server/src/__tests__/routes/proxy-empty-completion.test.ts@@ -83,6 +83,9 @@ describe('Empty-completion failover', () => {.mockResolvedValueOnce(EMPTY_RESULT).mockResolvedValueOnce(GOOD_RESULT);+ const db = getDb();+ db.prepare('DELETE FROM requests').run();+const { status, body, headers } = await post(app, '/v1/chat/completions', {messages: [{ role: 'user', content: 'hi' }],}, key);@@ -93,6 +96,12 @@ describe('Empty-completion failover', () => {expect(chatCompletion).toHaveBeenCalledTimes(2);// First model must differ from the one that answered.expect(chatCompletion.mock.calls[0][2]).not.toBe(chatCompletion.mock.calls[1][2]);++ const rows = db.prepare('SELECT status, error FROM requests ORDER BY id').all() as Array<{ status: string; error: string | null }>;+ expect(rows.length).toBe(2);+ expect(rows[0].status).toBe('error');+ expect(rows[0].error).toContain('empty completion');+ expect(rows[1].status).toBe('success');});it('/v1/chat/completions (stream): zero-chunk stream fails over instead of emitting an empty stream', async () => {@@ -171,15 +180,9 @@ describe('Empty-completion failover', () => {.mockResolvedValueOnce(EMPTY_RESULT).mockResolvedValueOnce(GOOD_RESULT);- const db = getDb();- db.prepare('DELETE FROM requests').run();- await post(app, '/v1/chat/completions', { messages: [{ role: 'user', content: 'hi' }] }, key);+ const { headers } = await post(app, '/v1/chat/completions', { messages: [{ role: 'user', content: 'hi' }] }, key);- const rows = db.prepare('SELECT status, error FROM requests ORDER BY id').all() as Array<{ status: string; error: string | null }>;- expect(rows.length).toBe(2);- expect(rows[0].status).toBe('error');- expect(rows[0].error).toContain('empty completion');- expect(rows[1].status).toBe('success');+ expect(headers.get('x-request-id')).toMatch(/\S+/);});it('a tool-calls-only completion (no text) is NOT treated as empty', async () => {diff --git a/server/src/__tests__/routes/proxy-stream-integrity.test.ts b/server/src/__tests__/routes/proxy-stream-integrity.test.tsindex 4f815106..a4f7cf34 100644--- a/server/src/__tests__/routes/proxy-stream-integrity.test.ts+++ b/server/src/__tests__/routes/proxy-stream-integrity.test.ts@@ -111,7 +111,8 @@ describe('proxy stream turn-integrity', () => {const fs = frames(r.text);expect(fs.some(f => f.choices?.[0]?.delta?.content?.includes('All good'))).toBe(true);expect(fs.some(f => f.error)).toBe(false);- // The dead turn is recorded as an error, not a success.+ expect(r.headers.get('x-request-id')).toMatch(/\S+/);+const rows = getDb().prepare("SELECT status, error FROM requests ORDER BY id").all() as any[];expect(rows[0].status).toBe('error');expect(rows[0].error).toMatch(/in-band provider error/);diff --git a/server/src/__tests__/routes/responses.test.ts b/server/src/__tests__/routes/responses.test.tsindex c37a2589..88449646 100644--- a/server/src/__tests__/routes/responses.test.ts+++ b/server/src/__tests__/routes/responses.test.ts@@ -26,7 +26,7 @@ async function post(app: Express, path: string, body: any, key?: string) {});const text = await res.text();server.close();- return { status: res.status, text, contentType: res.headers.get('content-type') ?? '' };+ return { status: res.status, text, contentType: res.headers.get('content-type') ?? '', headers: res.headers };}describe('POST /v1/responses (#96)', () => {@@ -92,8 +92,10 @@ describe('POST /v1/responses (#96)', () => {async *streamChatCompletion() { /* unused */ },}));- const { status, text } = await post(app, '/v1/responses', { input: 'hi', stream: false }, key);+ const { status, text, contentType } = await post(app, '/v1/responses', { input: 'hi', stream: false }, key);expect(status).toBe(200);+ expect(contentType).toContain('application/json');+ expect(text.length).toBeGreaterThan(0);const body = JSON.parse(text);expect(body.object).toBe('response');expect(body.status).toBe('completed');diff --git a/server/src/routes/proxy.ts b/server/src/routes/proxy.tsindex 0c713f2f..2b7613ab 100644--- a/server/src/routes/proxy.ts+++ b/server/src/routes/proxy.ts@@ -60,6 +60,52 @@ export function extractApiToken(req: Request): string | undefined {return trimmed || undefined;}+export function getRequestGroupId(req: Request): string {+ const raw = req.headers['x-request-id'];+ const value = Array.isArray(raw) ? raw[0] : raw;+ const trimmed = value?.trim();+ return trimmed || crypto.randomUUID();+}++function shortRequestId(requestId: string): string {+ return requestId.replace(/-/g, '').slice(0, 6);+}++type TraceEvent = 'start' | 'next' | 'ok' | 'fail';++export function traceRouteEvent(+ scope: 'Proxy' | 'Responses',+ opts: {+ event: TraceEvent;+ requestId: string;+ attempt: number;+ platform: string;+ model: string;+ requestedModel?: string;+ latencyMs?: number;+ inputTokens?: number;+ outputTokens?: number;+ error?: string;+ },+) {+ const parts = [+ `[${scope}]`,+ new Date().toISOString().slice(11, 19),+ opts.event,+ shortRequestId(opts.requestId),+ `a${opts.attempt}`,+ opts.platform,+ '-',+ opts.model,+ ];+ if (opts.requestedModel) parts.push(`req=${opts.requestedModel}`);+ if (opts.latencyMs != null) parts.push(`lat=${opts.latencyMs}ms`);+ if (opts.inputTokens != null) parts.push(`in=${opts.inputTokens}`);+ if (opts.outputTokens != null) parts.push(`out=${opts.outputTokens}`);+ if (opts.error) parts.push(`err=${JSON.stringify(opts.error)}`);+ console.log(parts.join(' '));+}+// Sticky sessions: track which model served each "session"// Key: hash of first user message → model_db_id// This prevents model switching mid-conversation which causes hallucination@@ -415,6 +461,8 @@ proxyRouter.post('/embeddings', async (req: Request, res: Response) => {proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {const start = Date.now();+ const requestGroupId = getRequestGroupId(req);+ res.setHeader('X-Request-ID', requestGroupId);// Authenticate with the unified API key for every proxy request, including// loopback callers. Browser pages can reach localhost, so socket locality is@@ -449,6 +497,7 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {}const { model: requestedModel, temperature, top_p, stream } = parsed.data;+ const requestedModelLabel = requestedModel ?? 'auto';// Agent-tolerant knob normalization (#200): max_tokens <= 0 means "no// limit" in several clients → unset; tool_choice 'any' is OpenAI's// 'required'; tool definitions get their 'function' type re-defaulted.@@ -779,6 +828,14 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {}const modelKey = `${route.platform}:${route.modelId}`;+ traceRouteEvent('Proxy', {+ event: attempt === 0 ? 'start' : 'next',+ requestId: requestGroupId,+ attempt,+ platform: route.platform,+ model: route.modelId,+ requestedModel: attempt === 0 ? requestedModelLabel : undefined,+ });let outboundMessages = messages;// Extra input tokens the injected handoff adds on this turn (0 when not// injected). Folded into the streaming success accounting, where token@@ -866,6 +923,15 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {console.error(`[Proxy] In-band error frame from ${route.displayName} mid-stream:`, msg);writeChunk({ error: { message: `Provider error (${route.displayName}): ${sanitizeProviderErrorMessage(String(msg))}`, type: 'stream_error' } });try { res.write('data: [DONE]\n\n'); res.end(); } catch { /* socket gone */ }+ traceRouteEvent('Proxy', {+ event: 'fail',+ requestId: requestGroupId,+ attempt,+ platform: route.platform,+ model: route.modelId,+ latencyMs: Date.now() - start,+ error: sanitizeProviderErrorMessage(String(msg)),+ });logRequest(route.platform, route.modelId, route.keyId, 'error', estimatedInputTokens, totalOutputTokens, Date.now() - start, `in-band error frame: ${sanitizeProviderErrorMessage(String(msg))}`, ttfbMs, pinnedModelId);return;}@@ -995,6 +1061,16 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {recordSuccess(route.modelDbId);setStickyModel(messages, route.modelDbId, sessionIdHeader, strategyKey);if (handoffMode !== 'off' && sessionKey) recordSuccessfulModel({ sessionKey, modelKey });+ traceRouteEvent('Proxy', {+ event: 'ok',+ requestId: requestGroupId,+ attempt,+ platform: route.platform,+ model: route.modelId,+ latencyMs: Date.now() - start,+ inputTokens: estimatedInputTokens + injectedHandoffTokens,+ outputTokens: totalOutputTokens,+ });logRequest(route.platform, route.modelId, route.keyId, 'success', estimatedInputTokens + injectedHandoffTokens, totalOutputTokens, Date.now() - start, null, ttfbMs, pinnedModelId);return;} catch (streamErr: any) {@@ -1005,6 +1081,15 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {const payload = { error: { message: `Provider error (${route.displayName}): stream interrupted`, type: 'stream_error' } };try { res.write(`data: ${JSON.stringify(payload)}\n\n`); } catch { /* socket gone */ }try { res.write('data: [DONE]\n\n'); res.end(); } catch { /* socket gone */ }+ traceRouteEvent('Proxy', {+ event: 'fail',+ requestId: requestGroupId,+ attempt,+ platform: route.platform,+ model: route.modelId,+ latencyMs: Date.now() - start,+ error: sanitizeProviderErrorMessage(streamErr.message),+ });logRequest(route.platform, route.modelId, route.keyId, 'error', estimatedInputTokens, totalOutputTokens, Date.now() - start, sanitizeProviderErrorMessage(streamErr.message), ttfbMs, pinnedModelId);return;}@@ -1026,6 +1111,15 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {const respMsg = result.choices?.[0]?.message;const respText = contentToString(respMsg?.content ?? '');if (!respText && (respMsg?.tool_calls?.length ?? 0) === 0) {+ traceRouteEvent('Proxy', {+ event: 'fail',+ requestId: requestGroupId,+ attempt,+ platform: route.platform,+ model: route.modelId,+ latencyMs: Date.now() - start,+ error: 'empty completion',+ });logRequest(route.platform, route.modelId, route.keyId, 'error', estimatedInputTokens, 0, Date.now() - start, 'empty completion (no content, no tool_calls)', null, pinnedModelId);skipKeys.add(`${route.platform}:${route.modelId}:${route.keyId}`);setCooldown(route.platform, route.modelId, route.keyId, getCooldownDurationForLimit(route.platform, route.modelId, route.keyId, { rpd: route.rpdLimit, tpd: route.tpdLimit }));@@ -1082,17 +1176,31 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {// Normalize array-shaped message.content to a string on the way out (#166).res.json(normalizeOutboundContent(result));- logRequest(- route.platform, route.modelId, route.keyId, 'success',- result.usage?.prompt_tokens ?? 0,- result.usage?.completion_tokens ?? 0,- Date.now() - start, null, null, pinnedModelId,- );+ traceRouteEvent('Proxy', {+ event: 'ok',+ requestId: requestGroupId,+ attempt,+ platform: route.platform,+ model: route.modelId,+ latencyMs: Date.now() - start,+ inputTokens: result.usage?.prompt_tokens ?? 0,+ outputTokens: result.usage?.completion_tokens ?? 0,+ });+ logRequest(route.platform, route.modelId, route.keyId, 'success', result.usage?.prompt_tokens ?? 0, result.usage?.completion_tokens ?? 0, Date.now() - start, null, null, pinnedModelId);return;}} catch (err: any) {const latency = Date.now() - start;const safeError = sanitizeProviderErrorMessage(err.message);+ traceRouteEvent('Proxy', {+ event: 'fail',+ requestId: requestGroupId,+ attempt,+ platform: route.platform,+ model: route.modelId,+ latencyMs: latency,+ error: safeError,+ });logRequest(route.platform, route.modelId, route.keyId, 'error', estimatedInputTokens, 0, latency, safeError, null, pinnedModelId);if (isRetryableError(err)) {@@ -1126,7 +1234,6 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {);recordRateLimitHit(route.modelDbId);lastError = err;- console.log(`[Proxy] ${safeError.slice(0, 60)} from ${route.displayName}, falling back (attempt ${attempt + 1}/${MAX_RETRIES})`);continue;}diff --git a/server/src/routes/responses.ts b/server/src/routes/responses.tsindex 59daf12c..baa05613 100644--- a/server/src/routes/responses.ts+++ b/server/src/routes/responses.ts@@ -21,8 +21,10 @@ import {isModelAccessForbiddenError,timingSafeStringEqual,extractApiToken,+ getRequestGroupId,getStickyModel,setStickyModel,+ traceRouteEvent,logRequest,} from './proxy.js';import { sanitizeProviderErrorMessage } from '../lib/error-redaction.js';@@ -264,6 +266,8 @@ export function buildResponseObject(opts: {responsesRouter.post('/responses', async (req: Request, res: Response) => {const start = Date.now();+ const requestGroupId = getRequestGroupId(req);+ res.setHeader('X-Request-ID', requestGroupId);// Same unified-key auth as the proxy (accepts Bearer or x-api-key).const token = extractApiToken(req);@@ -324,6 +328,7 @@ responsesRouter.post('/responses', async (req: Request, res: Response) => {const rawSessionId = req.headers['x-session-id'];const sessionIdHeader = Array.isArray(rawSessionId) ? rawSessionId[0] : rawSessionId;const preferredModel = getStickyModel(messages, sessionIdHeader);+ const requestedModelLabel = reqData.model ?? 'auto';// Tool-bearing requests (the normal case for Codex/agent clients on this// endpoint) must stay on models that emit structured tool_calls — a model@@ -374,6 +379,14 @@ responsesRouter.post('/responses', async (req: Request, res: Response) => {}try {+ traceRouteEvent('Responses', {+ event: attempt === 0 ? 'start' : 'next',+ requestId: requestGroupId,+ attempt,+ platform: route.platform,+ model: route.modelId,+ requestedModel: attempt === 0 ? requestedModelLabel : undefined,+ });if (stream) {let outputIndex = 0;let msgItemId: string | null = null;@@ -510,6 +523,15 @@ responsesRouter.post('/responses', async (req: Request, res: Response) => {? rescueInlineToolCalls(heldText, new Set((tools ?? []).map(t => t.function.name))): { detected: false as const, calls: null, cleanText: heldText };if (rescue.detected && !rescue.calls) {+ traceRouteEvent('Responses', {+ event: 'fail',+ requestId: requestGroupId,+ attempt,+ platform: route.platform,+ model: route.modelId,+ latencyMs: Date.now() - start,+ error: 'unparseable inline tool-call dialect',+ });logRequest(route.platform, route.modelId, route.keyId, 'error', estimatedInputTokens, 0, Date.now() - start, `unparseable inline tool-call dialect: ${heldText.slice(0, 120)}`);skipKeys.add(`${route.platform}:${route.modelId}:${route.keyId}`);setCooldown(route.platform, route.modelId, route.keyId, getCooldownDurationForLimit(route.platform, route.modelId, route.keyId, { rpd: route.rpdLimit, tpd: route.tpdLimit }));@@ -551,6 +573,15 @@ responsesRouter.post('/responses', async (req: Request, res: Response) => {// skeletons are out), so it's safe to fail over to the next model on// the same SSE stream.if (msgText.length === 0 && toolAcc.size === 0) {+ traceRouteEvent('Responses', {+ event: 'fail',+ requestId: requestGroupId,+ attempt,+ platform: route.platform,+ model: route.modelId,+ latencyMs: Date.now() - start,+ error: 'empty completion',+ });logRequest(route.platform, route.modelId, route.keyId, 'error', estimatedInputTokens, 0, Date.now() - start, 'empty completion (no content, no tool_calls)');skipKeys.add(`${route.platform}:${route.modelId}:${route.keyId}`);setCooldown(route.platform, route.modelId, route.keyId, getCooldownDurationForLimit(route.platform, route.modelId, route.keyId, { rpd: route.rpdLimit, tpd: route.tpdLimit }));@@ -590,6 +621,16 @@ responsesRouter.post('/responses', async (req: Request, res: Response) => {recordTokens(route.platform, route.modelId, route.keyId, estimatedInputTokens + totalOutputTokens);recordSuccess(route.modelDbId);setStickyModel(messages, route.modelDbId, sessionIdHeader);+ traceRouteEvent('Responses', {+ event: 'ok',+ requestId: requestGroupId,+ attempt,+ platform: route.platform,+ model: route.modelId,+ latencyMs: Date.now() - start,+ inputTokens: estimatedInputTokens,+ outputTokens: totalOutputTokens,+ });logRequest(route.platform, route.modelId, route.keyId, 'success', estimatedInputTokens, totalOutputTokens, Date.now() - start, null);return;} else {@@ -623,7 +664,16 @@ responsesRouter.post('/responses', async (req: Request, res: Response) => {// Empty completion → fail over (see the streaming-path comment above).if (!text && toolCalls.length === 0) {- logRequest(route.platform, route.modelId, route.keyId, 'error', estimatedInputTokens, 0, Date.now() - start, 'empty completion (no content, no tool_calls)');+ traceRouteEvent('Responses', {+ event: 'fail',+ requestId: requestGroupId,+ attempt,+ platform: route.platform,+ model: route.modelId,+ latencyMs: Date.now() - start,+ error: 'empty completion',+ });+ logRequest(route.platform, route.modelId, route.keyId, 'error', promptTokens, 0, Date.now() - start, 'empty completion (no content, no tool_calls)');skipKeys.add(`${route.platform}:${route.modelId}:${route.keyId}`);setCooldown(route.platform, route.modelId, route.keyId, getCooldownDurationForLimit(route.platform, route.modelId, route.keyId, { rpd: route.rpdLimit, tpd: route.tpdLimit }));recordRateLimitHit(route.modelDbId);@@ -643,13 +693,31 @@ responsesRouter.post('/responses', async (req: Request, res: Response) => {promptTokens, completionTokens,}));- logRequest(route.platform, route.modelId, route.keyId, 'success',- promptTokens, completionTokens, Date.now() - start, null);+ traceRouteEvent('Responses', {+ event: 'ok',+ requestId: requestGroupId,+ attempt,+ platform: route.platform,+ model: route.modelId,+ latencyMs: Date.now() - start,+ inputTokens: promptTokens,+ outputTokens: completionTokens,+ });+ logRequest(route.platform, route.modelId, route.keyId, 'success', promptTokens, completionTokens, Date.now() - start, null);return;}} catch (err: any) {const latency = Date.now() - start;const safeError = sanitizeProviderErrorMessage(err.message);+ traceRouteEvent('Responses', {+ event: 'fail',+ requestId: requestGroupId,+ attempt,+ platform: route.platform,+ model: route.modelId,+ latencyMs: latency,+ error: safeError,+ });logRequest(route.platform, route.modelId, route.keyId, 'error', estimatedInputTokens, 0, latency, safeError);// Mid-stream failures can't be retried (bytes already sent) — close cleanly.