7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.
TypeScript
fix(proxy,responses): record ttfb on the first reasoning token and count thinking tokens (#764) (#778)
suantea (asuan_suan)f5e267ca807d
diff --git a/server/src/__tests__/routes/proxy-stream-integrity.test.ts b/server/src/__tests__/routes/proxy-stream-integrity.test.tsindex 9716cfd4..53f17ed6 100644--- a/server/src/__tests__/routes/proxy-stream-integrity.test.ts+++ b/server/src/__tests__/routes/proxy-stream-integrity.test.ts@@ -290,6 +290,44 @@ describe('proxy stream turn-integrity', () => {expect(msg.content).toBeNull();expect(r.body.choices[0].finish_reason).toBe('tool_calls');});++ it('records ttfb on the first reasoning_content token, not the first visible text (#764)', async () => {+ // Reasoning models stream thinking (reasoning_content) long before the+ // first answer token. The regression: ttfb was recorded at header flush,+ // i.e. the END of the thinking phase, so the Analytics speed shown was the+ // reasoning tail — or NULL on turns that never flushed. It must count the+ // first token of ANY kind. We interleave a real delay between the+ // reasoning frame and the first text frame to make the two moments+ // measurable.+ const origFetch = global.fetch;+ const enc = new TextEncoder();+ vi.spyOn(global, 'fetch').mockImplementation(async (url, init) => {+ const urlStr = typeof url === 'string' ? url : url.toString();+ if (!urlStr.includes('api.groq.com')) return origFetch(url as any, init);+ const stream = new ReadableStream({+ async start(controller) {+ controller.enqueue(enc.encode(sse(+ roleChunk,+ { id: 'c1', object: 'chat.completion.chunk', created: 1, model: 'm', choices: [{ index: 0, delta: { reasoning_content: 'thinking hard…', content: null }, finish_reason: null }] },+ )));+ await new Promise(r => setTimeout(r, 400));+ controller.enqueue(enc.encode(sse(textChunk('Hello world'), finishChunk('stop'), '[DONE]')));+ controller.close();+ },+ });+ return new Response(stream, { status: 200, headers: { 'Content-Type': 'text/event-stream' } });+ });+ const r = await request(app, '/v1/chat/completions', {+ stream: true, messages: [{ role: 'user', content: 'ttfb reasoning test' }],+ });+ expect(r.status).toBe(200);+ const rows = getDb().prepare("SELECT status, latency_ms, ttfb_ms FROM requests ORDER BY id").all() as any[];+ expect(rows[0].status).toBe('success');+ expect(rows[0].ttfb_ms).not.toBeNull();+ // ttfb must be the reasoning-head moment (well before the text token that+ // triggers header flush), not the flush time ≈ latency.+ expect(rows[0].ttfb_ms).toBeLessThan(rows[0].latency_ms - 250);+ });});describe('sticky session integrity', () => {diff --git a/server/src/__tests__/routes/responses.test.ts b/server/src/__tests__/routes/responses.test.tsindex 2cf2e621..701beb27 100644--- a/server/src/__tests__/routes/responses.test.ts+++ b/server/src/__tests__/routes/responses.test.ts@@ -129,6 +129,32 @@ describe('POST /v1/responses (#96)', () => {expect(completed).toContain('"output_text":"Hello"');});+ it('stream: records ttfb on the first reasoning token, not the first visible text (#764)', async () => {+ mockRouteRequest.mockReturnValue(fakeRoute({+ async chatCompletion() { throw new Error('should not be called'); },+ async *streamChatCompletion() {+ // Reasoning models stream thinking before any visible text. ttfb must+ // count the first token of ANY kind; a real delay between the+ // reasoning frame and the first text frame makes the two moments+ // measurable in the requests row.+ yield { id: 'c', object: 'chat.completion.chunk', created: 0, model: 'fake-model', choices: [{ index: 0, delta: { reasoning_content: 'thinking hard…', content: null }, finish_reason: null }] };+ await new Promise(r => setTimeout(r, 400));+ yield { id: 'c', object: 'chat.completion.chunk', created: 0, model: 'fake-model', choices: [{ index: 0, delta: { content: 'Hello' }, finish_reason: null }] };+ yield { id: 'c', object: 'chat.completion.chunk', created: 0, model: 'fake-model', choices: [{ index: 0, delta: {}, finish_reason: 'stop' }] };+ },+ }));++ const { status, text } = await post(app, '/v1/responses', { input: 'hi', stream: true }, key);+ expect(status).toBe(200);+ expect(text).toContain('event: response.completed');+ const rows = getDb().prepare("SELECT status, latency_ms, ttfb_ms FROM requests ORDER BY id DESC LIMIT 1").all() as any[];+ expect(rows[0].status).toBe('success');+ expect(rows[0].ttfb_ms).not.toBeNull();+ // ttfb must be the reasoning-head moment (well before the text token that+ // triggers the commit), not the flush time ≈ latency.+ expect(rows[0].ttfb_ms).toBeLessThan(rows[0].latency_ms - 250);+ });+it('stream: tool-call deltas produce function_call events with assembled arguments', async () => {mockRouteRequest.mockReturnValue(fakeRoute({async chatCompletion() { throw new Error('nope'); },diff --git a/server/src/routes/proxy.ts b/server/src/routes/proxy.tsindex d4e12e53..a944a880 100644--- a/server/src/routes/proxy.ts+++ b/server/src/routes/proxy.ts@@ -530,6 +530,18 @@ export function streamChunkText(chunk: any): string {return chunk?.choices?.[0]?.delta?.content ?? '';}+// Pull the incremental reasoning text out of a streaming chunk. Reasoning+// models stream thinking via `reasoning_content` (Z.ai, DeepSeek-style — the+// <think> extractor in base.ts normalizes inline tags into the same field) or+// `reasoning` (Ollama-style) before the first visible answer token; both+// spellings must count for ttfb and output-token estimates. Same shape+// tolerance as streamChunkText. (#764)+export function streamReasoningText(chunk: any): string {+ const delta = chunk?.choices?.[0]?.delta;+ const r = delta?.reasoning_content ?? delta?.reasoning;+ return typeof r === 'string' ? r : '';+}+// OpenAI-compatible embeddings endpoint, routed through the embeddings family// catalog: `model: "auto"` (or omitted) → the configured default family; a// family name or provider model id → that family's provider chain. Failover@@ -799,6 +811,16 @@ function completionTextFromChat(result: any): string {return contentToString(result?.choices?.[0]?.message?.content ?? '');}+// Non-streaming counterpart of streamReasoningText: reasoning models attach+// thinking to the completed message as `reasoning_content` or `reasoning`.+// Included in the chars/4 output estimate so analytics and the rate-limit+// ledger aren't undercounted for thinking models. (#764)+export function completionReasoningText(result: any): string {+ const msg = result?.choices?.[0]?.message;+ const r = msg?.reasoning_content ?? msg?.reasoning;+ return typeof r === 'string' ? r : '';+}+function completionIdFromChat(id: string | undefined): string {if (!id) return `cmpl-${Date.now()}`;return id.startsWith('cmpl-') ? id : `cmpl-${id}`;@@ -980,7 +1002,10 @@ proxyRouter.post('/completions', async (req: Request, res: Response) => {const flushHeaders = () => {if (headerSent) return;- ttfbMs = Date.now() - start;+ // #764: ttfb is recorded on the first token of ANY kind (content or+ // reasoning) in the pump loop below; this call only backfills streams+ // that reached the commit point without one.+ if (ttfbMs === null) ttfbMs = Date.now() - start;res.setHeader('Content-Type', 'text/event-stream');res.setHeader('Cache-Control', 'no-cache');res.setHeader('Connection', 'keep-alive');@@ -1004,9 +1029,18 @@ proxyRouter.post('/completions', async (req: Request, res: Response) => {if (clientGone) break; // client hung up: stop pulling; reader.cancel() aborts upstreamconst text = streamChunkText(chunk);if (text.length > 0) sawText = true;+ // #764: reasoning models stream thinking before any visible text.+ // ttfb must count the first token of ANY kind — otherwise the speed+ // shown is the thinking tail, or NULL when headers never flush.+ const reasoning = streamReasoningText(chunk);+ if (ttfbMs === null && (text.length > 0 || reasoning.length > 0)) {+ ttfbMs = Date.now() - start;+ }const finish = (chunk as any)?.choices?.[0]?.finish_reason;if (finish) upstreamFinish = finish;- totalOutputTokens += Math.ceil(text.length / 4);+ // #764: reasoning tokens are real output consumption — count them+ // so analytics and the rate-limit ledger aren't undercounted.+ totalOutputTokens += Math.ceil((text.length + reasoning.length) / 4);const frame = legacyCompletionChunk(route, chunk, text);// Commit point: hold headers until the first real text, so a stream// that dies before producing any fails over invisibly.@@ -1103,9 +1137,11 @@ proxyRouter.post('/completions', async (req: Request, res: Response) => {// Usage fallback: providers that omit `usage` used to be logged as 0// tokens, silently undercounting analytics and the rate-limit ledger.- // Fall back to the same chars/4 estimate the streaming path uses.+ // Fall back to the same chars/4 estimate the streaming path uses,+ // including reasoning tokens (thinking models). (#764)const promptTokens = result.usage?.prompt_tokens ?? estimatedInputTokens;- const completionTokens = result.usage?.completion_tokens ?? Math.ceil(text.length / 4);+ const completionTokens = result.usage?.completion_tokens+ ?? Math.ceil((text.length + completionReasoningText(result).length) / 4);const totalTokens = result.usage?.total_tokens ?? (promptTokens + completionTokens);recordUpstreamSuccess(route, totalTokens);@@ -1819,7 +1855,9 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {const flushHeaders = () => {if (headerSent) return;- ttfbMs = Date.now() - start;+ // #764: backfill only — the pump loop already records ttfb on the+ // first token (content or reasoning) it sees.+ if (ttfbMs === null) ttfbMs = Date.now() - start;res.setHeader('Content-Type', 'text/event-stream');res.setHeader('Cache-Control', 'no-cache');res.setHeader('Connection', 'keep-alive');@@ -1894,13 +1932,13 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {if (choice.finish_reason) upstreamFinish = choice.finish_reason;- // #797: accumulate this turn's thinking trace (native- // reasoning_content deltas; the <think> extractor in base.ts has- // already normalized inline tags into the same field) so a- // follow-up request whose client stripped it can have it restored- // from session memory.- const reasoningDelta = choice.delta?.reasoning_content;- if (typeof reasoningDelta === 'string' && reasoningDelta.length > 0) streamReasoning += reasoningDelta;+ // #797: accumulate this turn's thinking trace (native reasoning+ // deltas; the <think> extractor in base.ts has already normalized+ // inline tags into reasoning_content) so a follow-up request whose+ // client stripped it can have it restored from session memory.+ // Shared with the #764 ttfb/token accounting below.+ const reasoning = streamReasoningText(anyChunk);+ if (reasoning.length > 0) streamReasoning += reasoning;// Buffer tool_call deltas — emitted complete + repaired at end.for (const tc of choice.delta?.tool_calls ?? []) {@@ -1916,12 +1954,22 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {sanitizeResponse(anyChunk);const text = typeof choice.delta?.content === 'string' ? choice.delta.content : '';+ // #764: ttfb = first token of ANY kind, not just visible content —+ // reasoning models stream thinking long before the first answer+ // token, and the old code deferred ttfb until header flush (or left+ // it NULL on long-thinking turns that never flushed).+ if (ttfbMs === null && (text.length > 0 || reasoning.length > 0)) {+ ttfbMs = Date.now() - start;+ }+if (text.length === 0) {// Role preamble / keep-alive: hold until first payload decides// the mode, forward afterwards. tool_calls and finish_reason are// stripped — both are re-emitted complete at the end (OpenRouter// attaches tool_call deltas to chunks that also carry role/// reasoning keys; forwarding them raw would duplicate the call).+ // #764: thinking-only chunks still consumed tokens — count them.+ if (reasoning.length > 0) totalOutputTokens += Math.ceil(reasoning.length / 4);if (choice.delta && Object.keys(choice.delta).some(k => k !== 'content' && k !== 'tool_calls' && choice.delta[k] != null)) {const cleaned = { ...anyChunk, choices: [{ ...choice, delta: { ...choice.delta, tool_calls: undefined }, finish_reason: null }] };if (headerSent) writeChunk(cleaned); else preamble.push(cleaned);@@ -1929,7 +1977,10 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {continue;}- totalOutputTokens += Math.ceil(text.length / 4);+ // #764: count reasoning tokens with the same chars/4 estimate so+ // analytics and rate-limit reflect real consumption of thinking+ // models (a chunk can carry both reasoning and text).+ totalOutputTokens += Math.ceil((text.length + reasoning.length) / 4);if (mode === 'passthrough') {writeChunk({ ...anyChunk, choices: [{ ...choice, delta: { ...choice.delta, tool_calls: undefined }, finish_reason: null }] });@@ -2173,11 +2224,12 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {// Usage fallback: providers that omit `usage` used to be logged as 0// tokens, silently undercounting analytics and the rate-limit ledger.// Fall back to the same chars/4 estimate the streaming path uses (tool- // arguments included, mirroring the stream accounting).+ // arguments included, mirroring the stream accounting; reasoning tokens+ // included too, so thinking models aren't undercounted — #764).const respToolArgChars = (respMsg?.tool_calls ?? []).reduce((n, tc) => n + (tc?.function?.arguments?.length ?? 0), 0);const promptTokens = result.usage?.prompt_tokens ?? estimatedInputTokens;const completionTokens = result.usage?.completion_tokens- ?? Math.ceil((contentToString(respMsg?.content ?? '').length + respToolArgChars) / 4);+ ?? Math.ceil((contentToString(respMsg?.content ?? '').length + completionReasoningText(result).length + respToolArgChars) / 4);const totalTokens = result.usage?.total_tokens ?? (promptTokens + completionTokens);recordUpstreamSuccess(route, totalTokens);// #797: remember this turn's thinking trace so the next request fromdiff --git a/server/src/routes/responses.ts b/server/src/routes/responses.tsindex 76f97863..f1575407 100644--- a/server/src/routes/responses.ts+++ b/server/src/routes/responses.ts@@ -21,6 +21,8 @@ import {getRequestGroupId,getStickyModel,setStickyModel,+ streamReasoningText,+ completionReasoningText,traceRouteEvent,logRequest,} from './proxy.js';@@ -537,6 +539,11 @@ responsesRouter.post('/responses', async (req: Request, res: Response) => {// tool-call accumulator keyed by the provider's tool_call indexconst toolAcc = new Map<number, { outputIndex: number; itemId: string; callId: string; name: string; args: string }>();let totalOutputTokens = 0;+ // #764: ttfb = first token of ANY kind (content or reasoning), recorded+ // in the pump loop; commit() only backfills streams that never produced+ // one. This path previously logged no ttfb at all, so Analytics showed+ // a null speed for every /v1/responses turn.+ let ttfbMs: number | null = null;// Inline-dialect hold window (#231): first text is held until it// either matches a tool-call dialect marker (held to the end and@@ -554,6 +561,7 @@ responsesRouter.post('/responses', async (req: Request, res: Response) => {// on the same connection with no bytes on the wire. Idempotent.const commit = () => {if (streamStarted) return;+ if (ttfbMs === null) ttfbMs = Date.now() - start;res.setHeader('Content-Type', 'text/event-stream');res.setHeader('Cache-Control', 'no-cache');res.setHeader('Connection', 'keep-alive');@@ -614,8 +622,16 @@ responsesRouter.post('/responses', async (req: Request, res: Response) => {// Text deltas → output_text events on a single message item, after// the dialect hold window has decided the text is real prose.const text = delta.content ?? '';+ // #764: reasoning models stream thinking before the first answer+ // token — count that first token as ttfb so Analytics speed+ // reflects the real head-of-stream, and count thinking tokens as+ // real output consumption.+ const reasoning = streamReasoningText(chunk);+ if (ttfbMs === null && (text.length > 0 || reasoning.length > 0)) {+ ttfbMs = Date.now() - start;+ }if (text) {- totalOutputTokens += Math.ceil(text.length / 4);+ totalOutputTokens += Math.ceil((text.length + reasoning.length) / 4);if (dialectMode === 'passthrough') {if (msgItemId === null) openTextItem('');sse('response.output_text.delta', {@@ -635,6 +651,9 @@ responsesRouter.post('/responses', async (req: Request, res: Response) => {}}}+ } else if (reasoning.length > 0) {+ // #764: thinking-only chunk (no visible text yet) — count tokens.+ totalOutputTokens += Math.ceil(reasoning.length / 4);}// Tool-call deltas → function_call item + argument deltas.@@ -769,7 +788,7 @@ responsesRouter.post('/responses', async (req: Request, res: Response) => {inputTokens: estimatedInputTokens,outputTokens: totalOutputTokens,});- logRequest(route.platform, route.modelId, route.keyId, 'success', estimatedInputTokens, totalOutputTokens, Date.now() - start, null);+ logRequest(route.platform, route.modelId, route.keyId, 'success', estimatedInputTokens, totalOutputTokens, Date.now() - start, null, ttfbMs);return 'done';} catch (streamErr: any) {// Client abort mid-stream: the pump's own `if (clientGone) break`@@ -792,7 +811,7 @@ responsesRouter.post('/responses', async (req: Request, res: Response) => {latencyMs: Date.now() - start,error: safe,});- logRequest(route.platform, route.modelId, route.keyId, 'error', estimatedInputTokens, 0, Date.now() - start, safe);+ logRequest(route.platform, route.modelId, route.keyId, 'error', estimatedInputTokens, 0, Date.now() - start, safe, ttfbMs);sse('response.failed', { response: { id: responseId, object: 'response', status: 'failed', error: { message: `Provider error (${route.displayName}): stream interrupted`, type: 'stream_error' } } });res.end();return 'committed';@@ -833,7 +852,11 @@ responsesRouter.post('/responses', async (req: Request, res: Response) => {}}const promptTokens = result.usage?.prompt_tokens ?? estimatedInputTokens;- const completionTokens = result.usage?.completion_tokens ?? Math.ceil(text.length / 4);+ // #764: include reasoning tokens (message.reasoning_content / reasoning)+ // in the chars/4 estimate so thinking models aren't undercounted when the+ // provider omits `usage`.+ const completionTokens = result.usage?.completion_tokens+ ?? Math.ceil((text.length + completionReasoningText(result).length) / 4);// Empty completion → fail over via the shared loop (see the streaming// path); finish_reason 'length' skips the cooldown/penalty.