Skip to content

7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.

TypeScript

fix(proxy,responses): record ttfb on the first reasoning token and count thinking tokens (#764) (#778)

suantea (asuan_suan)f5e267ca807d
diff --git a/server/src/__tests__/routes/proxy-stream-integrity.test.ts b/server/src/__tests__/routes/proxy-stream-integrity.test.ts
index 9716cfd4..53f17ed6 100644
--- a/server/src/__tests__/routes/proxy-stream-integrity.test.ts
+++ b/server/src/__tests__/routes/proxy-stream-integrity.test.ts
@@ -290,6 +290,44 @@ describe('proxy stream turn-integrity', () => {
expect(msg.content).toBeNull();
expect(r.body.choices[0].finish_reason).toBe('tool_calls');
});
+
+ it('records ttfb on the first reasoning_content token, not the first visible text (#764)', async () => {
+ // Reasoning models stream thinking (reasoning_content) long before the
+ // first answer token. The regression: ttfb was recorded at header flush,
+ // i.e. the END of the thinking phase, so the Analytics speed shown was the
+ // reasoning tail — or NULL on turns that never flushed. It must count the
+ // first token of ANY kind. We interleave a real delay between the
+ // reasoning frame and the first text frame to make the two moments
+ // measurable.
+ const origFetch = global.fetch;
+ const enc = new TextEncoder();
+ vi.spyOn(global, 'fetch').mockImplementation(async (url, init) => {
+ const urlStr = typeof url === 'string' ? url : url.toString();
+ if (!urlStr.includes('api.groq.com')) return origFetch(url as any, init);
+ const stream = new ReadableStream({
+ async start(controller) {
+ controller.enqueue(enc.encode(sse(
+ roleChunk,
+ { id: 'c1', object: 'chat.completion.chunk', created: 1, model: 'm', choices: [{ index: 0, delta: { reasoning_content: 'thinking hard…', content: null }, finish_reason: null }] },
+ )));
+ await new Promise(r => setTimeout(r, 400));
+ controller.enqueue(enc.encode(sse(textChunk('Hello world'), finishChunk('stop'), '[DONE]')));
+ controller.close();
+ },
+ });
+ return new Response(stream, { status: 200, headers: { 'Content-Type': 'text/event-stream' } });
+ });
+ const r = await request(app, '/v1/chat/completions', {
+ stream: true, messages: [{ role: 'user', content: 'ttfb reasoning test' }],
+ });
+ expect(r.status).toBe(200);
+ const rows = getDb().prepare("SELECT status, latency_ms, ttfb_ms FROM requests ORDER BY id").all() as any[];
+ expect(rows[0].status).toBe('success');
+ expect(rows[0].ttfb_ms).not.toBeNull();
+ // ttfb must be the reasoning-head moment (well before the text token that
+ // triggers header flush), not the flush time ≈ latency.
+ expect(rows[0].ttfb_ms).toBeLessThan(rows[0].latency_ms - 250);
+ });
});
describe('sticky session integrity', () => {
diff --git a/server/src/__tests__/routes/responses.test.ts b/server/src/__tests__/routes/responses.test.ts
index 2cf2e621..701beb27 100644
--- a/server/src/__tests__/routes/responses.test.ts
+++ b/server/src/__tests__/routes/responses.test.ts
@@ -129,6 +129,32 @@ describe('POST /v1/responses (#96)', () => {
expect(completed).toContain('"output_text":"Hello"');
});
+ it('stream: records ttfb on the first reasoning token, not the first visible text (#764)', async () => {
+ mockRouteRequest.mockReturnValue(fakeRoute({
+ async chatCompletion() { throw new Error('should not be called'); },
+ async *streamChatCompletion() {
+ // Reasoning models stream thinking before any visible text. ttfb must
+ // count the first token of ANY kind; a real delay between the
+ // reasoning frame and the first text frame makes the two moments
+ // measurable in the requests row.
+ yield { id: 'c', object: 'chat.completion.chunk', created: 0, model: 'fake-model', choices: [{ index: 0, delta: { reasoning_content: 'thinking hard…', content: null }, finish_reason: null }] };
+ await new Promise(r => setTimeout(r, 400));
+ yield { id: 'c', object: 'chat.completion.chunk', created: 0, model: 'fake-model', choices: [{ index: 0, delta: { content: 'Hello' }, finish_reason: null }] };
+ yield { id: 'c', object: 'chat.completion.chunk', created: 0, model: 'fake-model', choices: [{ index: 0, delta: {}, finish_reason: 'stop' }] };
+ },
+ }));
+
+ const { status, text } = await post(app, '/v1/responses', { input: 'hi', stream: true }, key);
+ expect(status).toBe(200);
+ expect(text).toContain('event: response.completed');
+ const rows = getDb().prepare("SELECT status, latency_ms, ttfb_ms FROM requests ORDER BY id DESC LIMIT 1").all() as any[];
+ expect(rows[0].status).toBe('success');
+ expect(rows[0].ttfb_ms).not.toBeNull();
+ // ttfb must be the reasoning-head moment (well before the text token that
+ // triggers the commit), not the flush time ≈ latency.
+ expect(rows[0].ttfb_ms).toBeLessThan(rows[0].latency_ms - 250);
+ });
+
it('stream: tool-call deltas produce function_call events with assembled arguments', async () => {
mockRouteRequest.mockReturnValue(fakeRoute({
async chatCompletion() { throw new Error('nope'); },
diff --git a/server/src/routes/proxy.ts b/server/src/routes/proxy.ts
index d4e12e53..a944a880 100644
--- a/server/src/routes/proxy.ts
+++ b/server/src/routes/proxy.ts
@@ -530,6 +530,18 @@ export function streamChunkText(chunk: any): string {
return chunk?.choices?.[0]?.delta?.content ?? '';
}
+// Pull the incremental reasoning text out of a streaming chunk. Reasoning
+// models stream thinking via `reasoning_content` (Z.ai, DeepSeek-style — the
+// <think> extractor in base.ts normalizes inline tags into the same field) or
+// `reasoning` (Ollama-style) before the first visible answer token; both
+// spellings must count for ttfb and output-token estimates. Same shape
+// tolerance as streamChunkText. (#764)
+export function streamReasoningText(chunk: any): string {
+ const delta = chunk?.choices?.[0]?.delta;
+ const r = delta?.reasoning_content ?? delta?.reasoning;
+ return typeof r === 'string' ? r : '';
+}
+
// OpenAI-compatible embeddings endpoint, routed through the embeddings family
// catalog: `model: "auto"` (or omitted) → the configured default family; a
// family name or provider model id → that family's provider chain. Failover
@@ -799,6 +811,16 @@ function completionTextFromChat(result: any): string {
return contentToString(result?.choices?.[0]?.message?.content ?? '');
}
+// Non-streaming counterpart of streamReasoningText: reasoning models attach
+// thinking to the completed message as `reasoning_content` or `reasoning`.
+// Included in the chars/4 output estimate so analytics and the rate-limit
+// ledger aren't undercounted for thinking models. (#764)
+export function completionReasoningText(result: any): string {
+ const msg = result?.choices?.[0]?.message;
+ const r = msg?.reasoning_content ?? msg?.reasoning;
+ return typeof r === 'string' ? r : '';
+}
+
function completionIdFromChat(id: string | undefined): string {
if (!id) return `cmpl-${Date.now()}`;
return id.startsWith('cmpl-') ? id : `cmpl-${id}`;
@@ -980,7 +1002,10 @@ proxyRouter.post('/completions', async (req: Request, res: Response) => {
const flushHeaders = () => {
if (headerSent) return;
- ttfbMs = Date.now() - start;
+ // #764: ttfb is recorded on the first token of ANY kind (content or
+ // reasoning) in the pump loop below; this call only backfills streams
+ // that reached the commit point without one.
+ if (ttfbMs === null) ttfbMs = Date.now() - start;
res.setHeader('Content-Type', 'text/event-stream');
res.setHeader('Cache-Control', 'no-cache');
res.setHeader('Connection', 'keep-alive');
@@ -1004,9 +1029,18 @@ proxyRouter.post('/completions', async (req: Request, res: Response) => {
if (clientGone) break; // client hung up: stop pulling; reader.cancel() aborts upstream
const text = streamChunkText(chunk);
if (text.length > 0) sawText = true;
+ // #764: reasoning models stream thinking before any visible text.
+ // ttfb must count the first token of ANY kind — otherwise the speed
+ // shown is the thinking tail, or NULL when headers never flush.
+ const reasoning = streamReasoningText(chunk);
+ if (ttfbMs === null && (text.length > 0 || reasoning.length > 0)) {
+ ttfbMs = Date.now() - start;
+ }
const finish = (chunk as any)?.choices?.[0]?.finish_reason;
if (finish) upstreamFinish = finish;
- totalOutputTokens += Math.ceil(text.length / 4);
+ // #764: reasoning tokens are real output consumption — count them
+ // so analytics and the rate-limit ledger aren't undercounted.
+ totalOutputTokens += Math.ceil((text.length + reasoning.length) / 4);
const frame = legacyCompletionChunk(route, chunk, text);
// Commit point: hold headers until the first real text, so a stream
// that dies before producing any fails over invisibly.
@@ -1103,9 +1137,11 @@ proxyRouter.post('/completions', async (req: Request, res: Response) => {
// Usage fallback: providers that omit `usage` used to be logged as 0
// tokens, silently undercounting analytics and the rate-limit ledger.
- // Fall back to the same chars/4 estimate the streaming path uses.
+ // Fall back to the same chars/4 estimate the streaming path uses,
+ // including reasoning tokens (thinking models). (#764)
const promptTokens = result.usage?.prompt_tokens ?? estimatedInputTokens;
- const completionTokens = result.usage?.completion_tokens ?? Math.ceil(text.length / 4);
+ const completionTokens = result.usage?.completion_tokens
+ ?? Math.ceil((text.length + completionReasoningText(result).length) / 4);
const totalTokens = result.usage?.total_tokens ?? (promptTokens + completionTokens);
recordUpstreamSuccess(route, totalTokens);
@@ -1819,7 +1855,9 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {
const flushHeaders = () => {
if (headerSent) return;
- ttfbMs = Date.now() - start;
+ // #764: backfill only — the pump loop already records ttfb on the
+ // first token (content or reasoning) it sees.
+ if (ttfbMs === null) ttfbMs = Date.now() - start;
res.setHeader('Content-Type', 'text/event-stream');
res.setHeader('Cache-Control', 'no-cache');
res.setHeader('Connection', 'keep-alive');
@@ -1894,13 +1932,13 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {
if (choice.finish_reason) upstreamFinish = choice.finish_reason;
- // #797: accumulate this turn's thinking trace (native
- // reasoning_content deltas; the <think> extractor in base.ts has
- // already normalized inline tags into the same field) so a
- // follow-up request whose client stripped it can have it restored
- // from session memory.
- const reasoningDelta = choice.delta?.reasoning_content;
- if (typeof reasoningDelta === 'string' && reasoningDelta.length > 0) streamReasoning += reasoningDelta;
+ // #797: accumulate this turn's thinking trace (native reasoning
+ // deltas; the <think> extractor in base.ts has already normalized
+ // inline tags into reasoning_content) so a follow-up request whose
+ // client stripped it can have it restored from session memory.
+ // Shared with the #764 ttfb/token accounting below.
+ const reasoning = streamReasoningText(anyChunk);
+ if (reasoning.length > 0) streamReasoning += reasoning;
// Buffer tool_call deltas — emitted complete + repaired at end.
for (const tc of choice.delta?.tool_calls ?? []) {
@@ -1916,12 +1954,22 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {
sanitizeResponse(anyChunk);
const text = typeof choice.delta?.content === 'string' ? choice.delta.content : '';
+ // #764: ttfb = first token of ANY kind, not just visible content —
+ // reasoning models stream thinking long before the first answer
+ // token, and the old code deferred ttfb until header flush (or left
+ // it NULL on long-thinking turns that never flushed).
+ if (ttfbMs === null && (text.length > 0 || reasoning.length > 0)) {
+ ttfbMs = Date.now() - start;
+ }
+
if (text.length === 0) {
// Role preamble / keep-alive: hold until first payload decides
// the mode, forward afterwards. tool_calls and finish_reason are
// stripped — both are re-emitted complete at the end (OpenRouter
// attaches tool_call deltas to chunks that also carry role/
// reasoning keys; forwarding them raw would duplicate the call).
+ // #764: thinking-only chunks still consumed tokens — count them.
+ if (reasoning.length > 0) totalOutputTokens += Math.ceil(reasoning.length / 4);
if (choice.delta && Object.keys(choice.delta).some(k => k !== 'content' && k !== 'tool_calls' && choice.delta[k] != null)) {
const cleaned = { ...anyChunk, choices: [{ ...choice, delta: { ...choice.delta, tool_calls: undefined }, finish_reason: null }] };
if (headerSent) writeChunk(cleaned); else preamble.push(cleaned);
@@ -1929,7 +1977,10 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {
continue;
}
- totalOutputTokens += Math.ceil(text.length / 4);
+ // #764: count reasoning tokens with the same chars/4 estimate so
+ // analytics and rate-limit reflect real consumption of thinking
+ // models (a chunk can carry both reasoning and text).
+ totalOutputTokens += Math.ceil((text.length + reasoning.length) / 4);
if (mode === 'passthrough') {
writeChunk({ ...anyChunk, choices: [{ ...choice, delta: { ...choice.delta, tool_calls: undefined }, finish_reason: null }] });
@@ -2173,11 +2224,12 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {
// Usage fallback: providers that omit `usage` used to be logged as 0
// tokens, silently undercounting analytics and the rate-limit ledger.
// Fall back to the same chars/4 estimate the streaming path uses (tool
- // arguments included, mirroring the stream accounting).
+ // arguments included, mirroring the stream accounting; reasoning tokens
+ // included too, so thinking models aren't undercounted — #764).
const respToolArgChars = (respMsg?.tool_calls ?? []).reduce((n, tc) => n + (tc?.function?.arguments?.length ?? 0), 0);
const promptTokens = result.usage?.prompt_tokens ?? estimatedInputTokens;
const completionTokens = result.usage?.completion_tokens
- ?? Math.ceil((contentToString(respMsg?.content ?? '').length + respToolArgChars) / 4);
+ ?? Math.ceil((contentToString(respMsg?.content ?? '').length + completionReasoningText(result).length + respToolArgChars) / 4);
const totalTokens = result.usage?.total_tokens ?? (promptTokens + completionTokens);
recordUpstreamSuccess(route, totalTokens);
// #797: remember this turn's thinking trace so the next request from
diff --git a/server/src/routes/responses.ts b/server/src/routes/responses.ts
index 76f97863..f1575407 100644
--- a/server/src/routes/responses.ts
+++ b/server/src/routes/responses.ts
@@ -21,6 +21,8 @@ import {
getRequestGroupId,
getStickyModel,
setStickyModel,
+ streamReasoningText,
+ completionReasoningText,
traceRouteEvent,
logRequest,
} from './proxy.js';
@@ -537,6 +539,11 @@ responsesRouter.post('/responses', async (req: Request, res: Response) => {
// tool-call accumulator keyed by the provider's tool_call index
const toolAcc = new Map<number, { outputIndex: number; itemId: string; callId: string; name: string; args: string }>();
let totalOutputTokens = 0;
+ // #764: ttfb = first token of ANY kind (content or reasoning), recorded
+ // in the pump loop; commit() only backfills streams that never produced
+ // one. This path previously logged no ttfb at all, so Analytics showed
+ // a null speed for every /v1/responses turn.
+ let ttfbMs: number | null = null;
// Inline-dialect hold window (#231): first text is held until it
// either matches a tool-call dialect marker (held to the end and
@@ -554,6 +561,7 @@ responsesRouter.post('/responses', async (req: Request, res: Response) => {
// on the same connection with no bytes on the wire. Idempotent.
const commit = () => {
if (streamStarted) return;
+ if (ttfbMs === null) ttfbMs = Date.now() - start;
res.setHeader('Content-Type', 'text/event-stream');
res.setHeader('Cache-Control', 'no-cache');
res.setHeader('Connection', 'keep-alive');
@@ -614,8 +622,16 @@ responsesRouter.post('/responses', async (req: Request, res: Response) => {
// Text deltas → output_text events on a single message item, after
// the dialect hold window has decided the text is real prose.
const text = delta.content ?? '';
+ // #764: reasoning models stream thinking before the first answer
+ // token — count that first token as ttfb so Analytics speed
+ // reflects the real head-of-stream, and count thinking tokens as
+ // real output consumption.
+ const reasoning = streamReasoningText(chunk);
+ if (ttfbMs === null && (text.length > 0 || reasoning.length > 0)) {
+ ttfbMs = Date.now() - start;
+ }
if (text) {
- totalOutputTokens += Math.ceil(text.length / 4);
+ totalOutputTokens += Math.ceil((text.length + reasoning.length) / 4);
if (dialectMode === 'passthrough') {
if (msgItemId === null) openTextItem('');
sse('response.output_text.delta', {
@@ -635,6 +651,9 @@ responsesRouter.post('/responses', async (req: Request, res: Response) => {
}
}
}
+ } else if (reasoning.length > 0) {
+ // #764: thinking-only chunk (no visible text yet) — count tokens.
+ totalOutputTokens += Math.ceil(reasoning.length / 4);
}
// Tool-call deltas → function_call item + argument deltas.
@@ -769,7 +788,7 @@ responsesRouter.post('/responses', async (req: Request, res: Response) => {
inputTokens: estimatedInputTokens,
outputTokens: totalOutputTokens,
});
- logRequest(route.platform, route.modelId, route.keyId, 'success', estimatedInputTokens, totalOutputTokens, Date.now() - start, null);
+ logRequest(route.platform, route.modelId, route.keyId, 'success', estimatedInputTokens, totalOutputTokens, Date.now() - start, null, ttfbMs);
return 'done';
} catch (streamErr: any) {
// Client abort mid-stream: the pump's own `if (clientGone) break`
@@ -792,7 +811,7 @@ responsesRouter.post('/responses', async (req: Request, res: Response) => {
latencyMs: Date.now() - start,
error: safe,
});
- logRequest(route.platform, route.modelId, route.keyId, 'error', estimatedInputTokens, 0, Date.now() - start, safe);
+ logRequest(route.platform, route.modelId, route.keyId, 'error', estimatedInputTokens, 0, Date.now() - start, safe, ttfbMs);
sse('response.failed', { response: { id: responseId, object: 'response', status: 'failed', error: { message: `Provider error (${route.displayName}): stream interrupted`, type: 'stream_error' } } });
res.end();
return 'committed';
@@ -833,7 +852,11 @@ responsesRouter.post('/responses', async (req: Request, res: Response) => {
}
}
const promptTokens = result.usage?.prompt_tokens ?? estimatedInputTokens;
- const completionTokens = result.usage?.completion_tokens ?? Math.ceil(text.length / 4);
+ // #764: include reasoning tokens (message.reasoning_content / reasoning)
+ // in the chars/4 estimate so thinking models aren't undercounted when the
+ // provider omits `usage`.
+ const completionTokens = result.usage?.completion_tokens
+ ?? Math.ceil((text.length + completionReasoningText(result).length) / 4);
// Empty completion → fail over via the shared loop (see the streaming
// path); finish_reason 'length' skips the cooldown/penalty.