Skip to content

7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.

TypeScript

Add chronological fallback logging for chat and responses (#313)

JammyJames1234a7e9a28c5596
diff --git a/server/src/__tests__/routes/proxy-empty-completion.test.ts b/server/src/__tests__/routes/proxy-empty-completion.test.ts
index 77411fa2..9405763c 100644
--- a/server/src/__tests__/routes/proxy-empty-completion.test.ts
+++ b/server/src/__tests__/routes/proxy-empty-completion.test.ts
@@ -83,6 +83,9 @@ describe('Empty-completion failover', () => {
.mockResolvedValueOnce(EMPTY_RESULT)
.mockResolvedValueOnce(GOOD_RESULT);
+ const db = getDb();
+ db.prepare('DELETE FROM requests').run();
+
const { status, body, headers } = await post(app, '/v1/chat/completions', {
messages: [{ role: 'user', content: 'hi' }],
}, key);
@@ -93,6 +96,12 @@ describe('Empty-completion failover', () => {
expect(chatCompletion).toHaveBeenCalledTimes(2);
// First model must differ from the one that answered.
expect(chatCompletion.mock.calls[0][2]).not.toBe(chatCompletion.mock.calls[1][2]);
+
+ const rows = db.prepare('SELECT status, error FROM requests ORDER BY id').all() as Array<{ status: string; error: string | null }>;
+ expect(rows.length).toBe(2);
+ expect(rows[0].status).toBe('error');
+ expect(rows[0].error).toContain('empty completion');
+ expect(rows[1].status).toBe('success');
});
it('/v1/chat/completions (stream): zero-chunk stream fails over instead of emitting an empty stream', async () => {
@@ -171,15 +180,9 @@ describe('Empty-completion failover', () => {
.mockResolvedValueOnce(EMPTY_RESULT)
.mockResolvedValueOnce(GOOD_RESULT);
- const db = getDb();
- db.prepare('DELETE FROM requests').run();
- await post(app, '/v1/chat/completions', { messages: [{ role: 'user', content: 'hi' }] }, key);
+ const { headers } = await post(app, '/v1/chat/completions', { messages: [{ role: 'user', content: 'hi' }] }, key);
- const rows = db.prepare('SELECT status, error FROM requests ORDER BY id').all() as Array<{ status: string; error: string | null }>;
- expect(rows.length).toBe(2);
- expect(rows[0].status).toBe('error');
- expect(rows[0].error).toContain('empty completion');
- expect(rows[1].status).toBe('success');
+ expect(headers.get('x-request-id')).toMatch(/\S+/);
});
it('a tool-calls-only completion (no text) is NOT treated as empty', async () => {
diff --git a/server/src/__tests__/routes/proxy-stream-integrity.test.ts b/server/src/__tests__/routes/proxy-stream-integrity.test.ts
index 4f815106..a4f7cf34 100644
--- a/server/src/__tests__/routes/proxy-stream-integrity.test.ts
+++ b/server/src/__tests__/routes/proxy-stream-integrity.test.ts
@@ -111,7 +111,8 @@ describe('proxy stream turn-integrity', () => {
const fs = frames(r.text);
expect(fs.some(f => f.choices?.[0]?.delta?.content?.includes('All good'))).toBe(true);
expect(fs.some(f => f.error)).toBe(false);
- // The dead turn is recorded as an error, not a success.
+ expect(r.headers.get('x-request-id')).toMatch(/\S+/);
+
const rows = getDb().prepare("SELECT status, error FROM requests ORDER BY id").all() as any[];
expect(rows[0].status).toBe('error');
expect(rows[0].error).toMatch(/in-band provider error/);
diff --git a/server/src/__tests__/routes/responses.test.ts b/server/src/__tests__/routes/responses.test.ts
index c37a2589..88449646 100644
--- a/server/src/__tests__/routes/responses.test.ts
+++ b/server/src/__tests__/routes/responses.test.ts
@@ -26,7 +26,7 @@ async function post(app: Express, path: string, body: any, key?: string) {
});
const text = await res.text();
server.close();
- return { status: res.status, text, contentType: res.headers.get('content-type') ?? '' };
+ return { status: res.status, text, contentType: res.headers.get('content-type') ?? '', headers: res.headers };
}
describe('POST /v1/responses (#96)', () => {
@@ -92,8 +92,10 @@ describe('POST /v1/responses (#96)', () => {
async *streamChatCompletion() { /* unused */ },
}));
- const { status, text } = await post(app, '/v1/responses', { input: 'hi', stream: false }, key);
+ const { status, text, contentType } = await post(app, '/v1/responses', { input: 'hi', stream: false }, key);
expect(status).toBe(200);
+ expect(contentType).toContain('application/json');
+ expect(text.length).toBeGreaterThan(0);
const body = JSON.parse(text);
expect(body.object).toBe('response');
expect(body.status).toBe('completed');
diff --git a/server/src/routes/proxy.ts b/server/src/routes/proxy.ts
index 0c713f2f..2b7613ab 100644
--- a/server/src/routes/proxy.ts
+++ b/server/src/routes/proxy.ts
@@ -60,6 +60,52 @@ export function extractApiToken(req: Request): string | undefined {
return trimmed || undefined;
}
+export function getRequestGroupId(req: Request): string {
+ const raw = req.headers['x-request-id'];
+ const value = Array.isArray(raw) ? raw[0] : raw;
+ const trimmed = value?.trim();
+ return trimmed || crypto.randomUUID();
+}
+
+function shortRequestId(requestId: string): string {
+ return requestId.replace(/-/g, '').slice(0, 6);
+}
+
+type TraceEvent = 'start' | 'next' | 'ok' | 'fail';
+
+export function traceRouteEvent(
+ scope: 'Proxy' | 'Responses',
+ opts: {
+ event: TraceEvent;
+ requestId: string;
+ attempt: number;
+ platform: string;
+ model: string;
+ requestedModel?: string;
+ latencyMs?: number;
+ inputTokens?: number;
+ outputTokens?: number;
+ error?: string;
+ },
+) {
+ const parts = [
+ `[${scope}]`,
+ new Date().toISOString().slice(11, 19),
+ opts.event,
+ shortRequestId(opts.requestId),
+ `a${opts.attempt}`,
+ opts.platform,
+ '-',
+ opts.model,
+ ];
+ if (opts.requestedModel) parts.push(`req=${opts.requestedModel}`);
+ if (opts.latencyMs != null) parts.push(`lat=${opts.latencyMs}ms`);
+ if (opts.inputTokens != null) parts.push(`in=${opts.inputTokens}`);
+ if (opts.outputTokens != null) parts.push(`out=${opts.outputTokens}`);
+ if (opts.error) parts.push(`err=${JSON.stringify(opts.error)}`);
+ console.log(parts.join(' '));
+}
+
// Sticky sessions: track which model served each "session"
// Key: hash of first user message → model_db_id
// This prevents model switching mid-conversation which causes hallucination
@@ -415,6 +461,8 @@ proxyRouter.post('/embeddings', async (req: Request, res: Response) => {
proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {
const start = Date.now();
+ const requestGroupId = getRequestGroupId(req);
+ res.setHeader('X-Request-ID', requestGroupId);
// Authenticate with the unified API key for every proxy request, including
// loopback callers. Browser pages can reach localhost, so socket locality is
@@ -449,6 +497,7 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {
}
const { model: requestedModel, temperature, top_p, stream } = parsed.data;
+ const requestedModelLabel = requestedModel ?? 'auto';
// Agent-tolerant knob normalization (#200): max_tokens <= 0 means "no
// limit" in several clients → unset; tool_choice 'any' is OpenAI's
// 'required'; tool definitions get their 'function' type re-defaulted.
@@ -779,6 +828,14 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {
}
const modelKey = `${route.platform}:${route.modelId}`;
+ traceRouteEvent('Proxy', {
+ event: attempt === 0 ? 'start' : 'next',
+ requestId: requestGroupId,
+ attempt,
+ platform: route.platform,
+ model: route.modelId,
+ requestedModel: attempt === 0 ? requestedModelLabel : undefined,
+ });
let outboundMessages = messages;
// Extra input tokens the injected handoff adds on this turn (0 when not
// injected). Folded into the streaming success accounting, where token
@@ -866,6 +923,15 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {
console.error(`[Proxy] In-band error frame from ${route.displayName} mid-stream:`, msg);
writeChunk({ error: { message: `Provider error (${route.displayName}): ${sanitizeProviderErrorMessage(String(msg))}`, type: 'stream_error' } });
try { res.write('data: [DONE]\n\n'); res.end(); } catch { /* socket gone */ }
+ traceRouteEvent('Proxy', {
+ event: 'fail',
+ requestId: requestGroupId,
+ attempt,
+ platform: route.platform,
+ model: route.modelId,
+ latencyMs: Date.now() - start,
+ error: sanitizeProviderErrorMessage(String(msg)),
+ });
logRequest(route.platform, route.modelId, route.keyId, 'error', estimatedInputTokens, totalOutputTokens, Date.now() - start, `in-band error frame: ${sanitizeProviderErrorMessage(String(msg))}`, ttfbMs, pinnedModelId);
return;
}
@@ -995,6 +1061,16 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {
recordSuccess(route.modelDbId);
setStickyModel(messages, route.modelDbId, sessionIdHeader, strategyKey);
if (handoffMode !== 'off' && sessionKey) recordSuccessfulModel({ sessionKey, modelKey });
+ traceRouteEvent('Proxy', {
+ event: 'ok',
+ requestId: requestGroupId,
+ attempt,
+ platform: route.platform,
+ model: route.modelId,
+ latencyMs: Date.now() - start,
+ inputTokens: estimatedInputTokens + injectedHandoffTokens,
+ outputTokens: totalOutputTokens,
+ });
logRequest(route.platform, route.modelId, route.keyId, 'success', estimatedInputTokens + injectedHandoffTokens, totalOutputTokens, Date.now() - start, null, ttfbMs, pinnedModelId);
return;
} catch (streamErr: any) {
@@ -1005,6 +1081,15 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {
const payload = { error: { message: `Provider error (${route.displayName}): stream interrupted`, type: 'stream_error' } };
try { res.write(`data: ${JSON.stringify(payload)}\n\n`); } catch { /* socket gone */ }
try { res.write('data: [DONE]\n\n'); res.end(); } catch { /* socket gone */ }
+ traceRouteEvent('Proxy', {
+ event: 'fail',
+ requestId: requestGroupId,
+ attempt,
+ platform: route.platform,
+ model: route.modelId,
+ latencyMs: Date.now() - start,
+ error: sanitizeProviderErrorMessage(streamErr.message),
+ });
logRequest(route.platform, route.modelId, route.keyId, 'error', estimatedInputTokens, totalOutputTokens, Date.now() - start, sanitizeProviderErrorMessage(streamErr.message), ttfbMs, pinnedModelId);
return;
}
@@ -1026,6 +1111,15 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {
const respMsg = result.choices?.[0]?.message;
const respText = contentToString(respMsg?.content ?? '');
if (!respText && (respMsg?.tool_calls?.length ?? 0) === 0) {
+ traceRouteEvent('Proxy', {
+ event: 'fail',
+ requestId: requestGroupId,
+ attempt,
+ platform: route.platform,
+ model: route.modelId,
+ latencyMs: Date.now() - start,
+ error: 'empty completion',
+ });
logRequest(route.platform, route.modelId, route.keyId, 'error', estimatedInputTokens, 0, Date.now() - start, 'empty completion (no content, no tool_calls)', null, pinnedModelId);
skipKeys.add(`${route.platform}:${route.modelId}:${route.keyId}`);
setCooldown(route.platform, route.modelId, route.keyId, getCooldownDurationForLimit(route.platform, route.modelId, route.keyId, { rpd: route.rpdLimit, tpd: route.tpdLimit }));
@@ -1082,17 +1176,31 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {
// Normalize array-shaped message.content to a string on the way out (#166).
res.json(normalizeOutboundContent(result));
- logRequest(
- route.platform, route.modelId, route.keyId, 'success',
- result.usage?.prompt_tokens ?? 0,
- result.usage?.completion_tokens ?? 0,
- Date.now() - start, null, null, pinnedModelId,
- );
+ traceRouteEvent('Proxy', {
+ event: 'ok',
+ requestId: requestGroupId,
+ attempt,
+ platform: route.platform,
+ model: route.modelId,
+ latencyMs: Date.now() - start,
+ inputTokens: result.usage?.prompt_tokens ?? 0,
+ outputTokens: result.usage?.completion_tokens ?? 0,
+ });
+ logRequest(route.platform, route.modelId, route.keyId, 'success', result.usage?.prompt_tokens ?? 0, result.usage?.completion_tokens ?? 0, Date.now() - start, null, null, pinnedModelId);
return;
}
} catch (err: any) {
const latency = Date.now() - start;
const safeError = sanitizeProviderErrorMessage(err.message);
+ traceRouteEvent('Proxy', {
+ event: 'fail',
+ requestId: requestGroupId,
+ attempt,
+ platform: route.platform,
+ model: route.modelId,
+ latencyMs: latency,
+ error: safeError,
+ });
logRequest(route.platform, route.modelId, route.keyId, 'error', estimatedInputTokens, 0, latency, safeError, null, pinnedModelId);
if (isRetryableError(err)) {
@@ -1126,7 +1234,6 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {
);
recordRateLimitHit(route.modelDbId);
lastError = err;
- console.log(`[Proxy] ${safeError.slice(0, 60)} from ${route.displayName}, falling back (attempt ${attempt + 1}/${MAX_RETRIES})`);
continue;
}
diff --git a/server/src/routes/responses.ts b/server/src/routes/responses.ts
index 59daf12c..baa05613 100644
--- a/server/src/routes/responses.ts
+++ b/server/src/routes/responses.ts
@@ -21,8 +21,10 @@ import {
isModelAccessForbiddenError,
timingSafeStringEqual,
extractApiToken,
+ getRequestGroupId,
getStickyModel,
setStickyModel,
+ traceRouteEvent,
logRequest,
} from './proxy.js';
import { sanitizeProviderErrorMessage } from '../lib/error-redaction.js';
@@ -264,6 +266,8 @@ export function buildResponseObject(opts: {
responsesRouter.post('/responses', async (req: Request, res: Response) => {
const start = Date.now();
+ const requestGroupId = getRequestGroupId(req);
+ res.setHeader('X-Request-ID', requestGroupId);
// Same unified-key auth as the proxy (accepts Bearer or x-api-key).
const token = extractApiToken(req);
@@ -324,6 +328,7 @@ responsesRouter.post('/responses', async (req: Request, res: Response) => {
const rawSessionId = req.headers['x-session-id'];
const sessionIdHeader = Array.isArray(rawSessionId) ? rawSessionId[0] : rawSessionId;
const preferredModel = getStickyModel(messages, sessionIdHeader);
+ const requestedModelLabel = reqData.model ?? 'auto';
// Tool-bearing requests (the normal case for Codex/agent clients on this
// endpoint) must stay on models that emit structured tool_calls — a model
@@ -374,6 +379,14 @@ responsesRouter.post('/responses', async (req: Request, res: Response) => {
}
try {
+ traceRouteEvent('Responses', {
+ event: attempt === 0 ? 'start' : 'next',
+ requestId: requestGroupId,
+ attempt,
+ platform: route.platform,
+ model: route.modelId,
+ requestedModel: attempt === 0 ? requestedModelLabel : undefined,
+ });
if (stream) {
let outputIndex = 0;
let msgItemId: string | null = null;
@@ -510,6 +523,15 @@ responsesRouter.post('/responses', async (req: Request, res: Response) => {
? rescueInlineToolCalls(heldText, new Set((tools ?? []).map(t => t.function.name)))
: { detected: false as const, calls: null, cleanText: heldText };
if (rescue.detected && !rescue.calls) {
+ traceRouteEvent('Responses', {
+ event: 'fail',
+ requestId: requestGroupId,
+ attempt,
+ platform: route.platform,
+ model: route.modelId,
+ latencyMs: Date.now() - start,
+ error: 'unparseable inline tool-call dialect',
+ });
logRequest(route.platform, route.modelId, route.keyId, 'error', estimatedInputTokens, 0, Date.now() - start, `unparseable inline tool-call dialect: ${heldText.slice(0, 120)}`);
skipKeys.add(`${route.platform}:${route.modelId}:${route.keyId}`);
setCooldown(route.platform, route.modelId, route.keyId, getCooldownDurationForLimit(route.platform, route.modelId, route.keyId, { rpd: route.rpdLimit, tpd: route.tpdLimit }));
@@ -551,6 +573,15 @@ responsesRouter.post('/responses', async (req: Request, res: Response) => {
// skeletons are out), so it's safe to fail over to the next model on
// the same SSE stream.
if (msgText.length === 0 && toolAcc.size === 0) {
+ traceRouteEvent('Responses', {
+ event: 'fail',
+ requestId: requestGroupId,
+ attempt,
+ platform: route.platform,
+ model: route.modelId,
+ latencyMs: Date.now() - start,
+ error: 'empty completion',
+ });
logRequest(route.platform, route.modelId, route.keyId, 'error', estimatedInputTokens, 0, Date.now() - start, 'empty completion (no content, no tool_calls)');
skipKeys.add(`${route.platform}:${route.modelId}:${route.keyId}`);
setCooldown(route.platform, route.modelId, route.keyId, getCooldownDurationForLimit(route.platform, route.modelId, route.keyId, { rpd: route.rpdLimit, tpd: route.tpdLimit }));
@@ -590,6 +621,16 @@ responsesRouter.post('/responses', async (req: Request, res: Response) => {
recordTokens(route.platform, route.modelId, route.keyId, estimatedInputTokens + totalOutputTokens);
recordSuccess(route.modelDbId);
setStickyModel(messages, route.modelDbId, sessionIdHeader);
+ traceRouteEvent('Responses', {
+ event: 'ok',
+ requestId: requestGroupId,
+ attempt,
+ platform: route.platform,
+ model: route.modelId,
+ latencyMs: Date.now() - start,
+ inputTokens: estimatedInputTokens,
+ outputTokens: totalOutputTokens,
+ });
logRequest(route.platform, route.modelId, route.keyId, 'success', estimatedInputTokens, totalOutputTokens, Date.now() - start, null);
return;
} else {
@@ -623,7 +664,16 @@ responsesRouter.post('/responses', async (req: Request, res: Response) => {
// Empty completion → fail over (see the streaming-path comment above).
if (!text && toolCalls.length === 0) {
- logRequest(route.platform, route.modelId, route.keyId, 'error', estimatedInputTokens, 0, Date.now() - start, 'empty completion (no content, no tool_calls)');
+ traceRouteEvent('Responses', {
+ event: 'fail',
+ requestId: requestGroupId,
+ attempt,
+ platform: route.platform,
+ model: route.modelId,
+ latencyMs: Date.now() - start,
+ error: 'empty completion',
+ });
+ logRequest(route.platform, route.modelId, route.keyId, 'error', promptTokens, 0, Date.now() - start, 'empty completion (no content, no tool_calls)');
skipKeys.add(`${route.platform}:${route.modelId}:${route.keyId}`);
setCooldown(route.platform, route.modelId, route.keyId, getCooldownDurationForLimit(route.platform, route.modelId, route.keyId, { rpd: route.rpdLimit, tpd: route.tpdLimit }));
recordRateLimitHit(route.modelDbId);
@@ -643,13 +693,31 @@ responsesRouter.post('/responses', async (req: Request, res: Response) => {
promptTokens, completionTokens,
}));
- logRequest(route.platform, route.modelId, route.keyId, 'success',
- promptTokens, completionTokens, Date.now() - start, null);
+ traceRouteEvent('Responses', {
+ event: 'ok',
+ requestId: requestGroupId,
+ attempt,
+ platform: route.platform,
+ model: route.modelId,
+ latencyMs: Date.now() - start,
+ inputTokens: promptTokens,
+ outputTokens: completionTokens,
+ });
+ logRequest(route.platform, route.modelId, route.keyId, 'success', promptTokens, completionTokens, Date.now() - start, null);
return;
}
} catch (err: any) {
const latency = Date.now() - start;
const safeError = sanitizeProviderErrorMessage(err.message);
+ traceRouteEvent('Responses', {
+ event: 'fail',
+ requestId: requestGroupId,
+ attempt,
+ platform: route.platform,
+ model: route.modelId,
+ latencyMs: latency,
+ error: safeError,
+ });
logRequest(route.platform, route.modelId, route.keyId, 'error', estimatedInputTokens, 0, latency, safeError);
// Mid-stream failures can't be retried (bytes already sent) — close cleanly.