7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.
TypeScript
fix(openai-compat): record real token usage from streaming responses (#845)
xyblue13573453b5271db
diff --git a/server/src/__tests__/routes/proxy-stream-integrity.test.ts b/server/src/__tests__/routes/proxy-stream-integrity.test.tsindex 53f17ed6..95e700ec 100644--- a/server/src/__tests__/routes/proxy-stream-integrity.test.ts+++ b/server/src/__tests__/routes/proxy-stream-integrity.test.ts@@ -49,7 +49,7 @@ const TOOLS = [{ type: 'function', function: { name: 'Read', description: 'readfunction mockUpstream(script: Array<{ body: string; status?: number }>) {const origFetch = global.fetch;let call = 0;- const seen: Array<{ model: string }> = [];+ const seen: Array<{ model: string; streamOptions?: { include_usage?: boolean } }> = [];vi.spyOn(global, 'fetch').mockImplementation(async (url, init) => {const urlStr = typeof url === 'string' ? url : url.toString();// Only intercept provider upstreams; the test's own localhost request@@ -58,7 +58,7 @@ function mockUpstream(script: Array<{ body: string; status?: number }>) {return origFetch(url as any, init);}const reqBody = JSON.parse(String((init as RequestInit).body));- seen.push({ model: reqBody.model });+ seen.push({ model: reqBody.model, streamOptions: reqBody.stream_options });const step = script[Math.min(call++, script.length - 1)];return new Response(step.body, {status: step.status ?? 200,@@ -268,6 +268,39 @@ describe('proxy stream turn-integrity', () => {expect(r.text.trim().endsWith('data: [DONE]')).toBe(true);});+ it('forwards stream_options.include_usage and records upstream token usage', async () => {+ const usage = { prompt_tokens: 3263, completion_tokens: 15, total_tokens: 3278 };+ const up = mockUpstream([{+ body: sse(+ roleChunk,+ textChunk('Hello'),+ finishChunk('stop'),+ { id: 'c1', object: 'chat.completion.chunk', created: 1, model: 'm', choices: [], usage },+ '[DONE]',+ ),+ }]);++ const r = await request(app, '/v1/chat/completions', {+ stream: true,+ stream_options: { include_usage: true },+ messages: [{ role: 'user', content: 'real streaming usage test' }],+ });++ expect(r.status).toBe(200);+ expect(up.seen[0].streamOptions).toEqual({ include_usage: true });+ expect(frames(r.text).find(f => f.usage)?.usage).toEqual(usage);++ const row = getDb().prepare(+ "SELECT input_tokens, output_tokens FROM requests WHERE status = 'success' ORDER BY id DESC LIMIT 1",+ ).get() as { input_tokens: number; output_tokens: number };+ expect(row).toEqual({ input_tokens: 3263, output_tokens: 15 });++ const ledger = getDb().prepare(+ "SELECT COALESCE(SUM(tokens), 0) AS total FROM rate_limit_usage WHERE kind = 'tokens'",+ ).get() as { total: number };+ expect(ledger.total).toBe(3278);+ });+it('rescues a non-streaming inline dialect answer into structured tool_calls', async () => {const origFetch = global.fetch;vi.spyOn(global, 'fetch').mockImplementation(async (url, init) => {diff --git a/server/src/providers/base.ts b/server/src/providers/base.tsindex d27f60f9..bb471ebd 100644--- a/server/src/providers/base.ts+++ b/server/src/providers/base.ts@@ -160,6 +160,9 @@ export interface CompletionOptions extends ExtendedSamplingOptions {tools?: ChatToolDefinition[];tool_choice?: ChatToolChoice;parallel_tool_calls?: boolean;+ stream_options?: {+ include_usage?: boolean;+ };/** Per-call HTTP timeout override. Not part of the OpenAI wire format (it is* stripped before the request body is built); used by the probe script so* NVIDIA's 15-60s serverless cold starts don't read as failures. */diff --git a/server/src/providers/openai-compat.ts b/server/src/providers/openai-compat.tsindex 08b1c173..7747096f 100644--- a/server/src/providers/openai-compat.ts+++ b/server/src/providers/openai-compat.ts@@ -325,6 +325,7 @@ export class OpenAICompatProvider extends BaseProvider {parallel_tool_calls: this.resolveParallelToolCalls(options),...extendedBodyParams(this.platform, options),stream: true,+ stream_options: options?.stream_options,}),// Default 'headers' bounds: the deadline dies at response headers, and// the client signal + stall watchdog own the stream from there.diff --git a/server/src/routes/proxy.ts b/server/src/routes/proxy.tsindex 94c35bcd..0817dfc3 100644--- a/server/src/routes/proxy.ts+++ b/server/src/routes/proxy.ts@@ -2,7 +2,7 @@ import crypto from 'crypto';import { Router } from 'express';import type { Request, Response } from 'express';import { z } from 'zod';-import type { ChatMessage, ChatToolCall, ModelListRow } from '@freellmapi/shared/types.js';+import type { ChatMessage, ChatToolCall, ModelListRow, TokenUsage } from '@freellmapi/shared/types.js';import { routeRequest, resolveRoutingChain, resolveModelGroupCandidates, resolveStickyPreference, recordRateLimitHit, recordSuccess, hasEnabledVisionModel, hasEnabledToolsModel, hasOtherUsableKey, routingReserveTokens, type RouteResult, type ResolvedChain, type ChainRow } from '../services/router.js';import { recordRequest, recordTokens, setCooldown, getCooldownDurationForLimit, PAYMENT_REQUIRED_COOLDOWN_MS, MODEL_FORBIDDEN_COOLDOWN_MS, learnLimitFromError } from '../services/ratelimit.js';import { runEmbeddings, EmbeddingsError } from '../services/embeddings.js';@@ -500,6 +500,9 @@ const chatCompletionSchema = z.object({top_p: z.number().min(0).max(1).optional(),stop: stopSchema.optional(),stream: z.boolean().optional(),+ stream_options: z.object({+ include_usage: z.boolean().optional(),+ }).optional(),// Top-level tool knobs may arrive as explicit nulls from clients that// serialize every field of their request struct; all treated as absent// and never forwarded as null. (#200)@@ -1388,9 +1391,9 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {// Token estimation is intentionally a heuristic (~4 chars per token). Used// for routing decisions (skip a model whose budget is too small) and for// streaming bookkeeping where the provider doesn't echo a final usage count.- // Non-streaming requests reconcile against the provider's real `usage` block- // (see line ~340). Streaming will drift from real consumption — accepted- // tradeoff because per-request usage isn't always returned mid-stream.+ // Non-streaming requests reconcile against the provider's real `usage` block;+ // streaming does the same when stream_options.include_usage produces a final+ // usage frame, and otherwise falls back to this estimate.const estimatedInputTokens = messages.reduce((sum, m) => {const text = contentToString(m.content);return sum + Math.ceil(text.length / 4);@@ -1880,7 +1883,7 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {try {const gen = route.provider.streamChatCompletion(route.apiKey, outboundMessages, route.modelId,- { temperature, max_tokens, top_p, stop, tools, tool_choice, parallel_tool_calls, ...samplingParams, signal: clientAbort.signal },+ { temperature, max_tokens, top_p, stop, tools, tool_choice, parallel_tool_calls, stream_options: parsed.data.stream_options, ...samplingParams, signal: clientAbort.signal },quotaContextForRoute(route, 'chat/completions'),);@@ -2094,7 +2097,11 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {res.write('data: [DONE]\n\n');res.end();- recordUpstreamSuccess(route, estimatedInputTokens + injectedHandoffTokens + totalOutputTokens);+ const upstreamUsage = (usageChunk as { usage?: TokenUsage } | null)?.usage;+ const inputTokens = upstreamUsage?.prompt_tokens ?? (estimatedInputTokens + injectedHandoffTokens);+ const outputTokens = upstreamUsage?.completion_tokens ?? totalOutputTokens;+ const totalTokens = upstreamUsage?.total_tokens ?? (inputTokens + outputTokens);+ recordUpstreamSuccess(route, totalTokens);setStickyModel(messages, route.modelDbId, sessionIdHeader, stickyStrategyKey);if (handoffMode !== 'off' && sessionKey) recordSuccessfulModel({ sessionKey, modelKey });// #797: remember this turn's thinking trace so the next request from@@ -2107,10 +2114,10 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {platform: route.platform,model: route.modelId,latencyMs: Date.now() - start,- inputTokens: estimatedInputTokens + injectedHandoffTokens,- outputTokens: totalOutputTokens,+ inputTokens,+ outputTokens,});- logRequest(route.platform, route.modelId, route.keyId, 'success', estimatedInputTokens + injectedHandoffTokens, totalOutputTokens, Date.now() - start, null, ttfbMs, pinnedModelId,+ logRequest(route.platform, route.modelId, route.keyId, 'success', inputTokens, outputTokens, Date.now() - start, null, ttfbMs, pinnedModelId,observeServedModel({ platform: route.platform, requestedModel: route.modelId, servedModel: upstreamModel }));return 'done';} catch (streamErr: any) {diff --git a/shared/types.ts b/shared/types.tsindex 0c36908e..1bc05f37 100644--- a/shared/types.ts+++ b/shared/types.ts@@ -339,6 +339,9 @@ export interface ChatCompletionRequest {temperature?: number;max_tokens?: number;stream?: boolean;+ stream_options?: {+ include_usage?: boolean;+ };top_p?: number;stop?: string | string[];tools?: ChatToolDefinition[];@@ -389,6 +392,7 @@ export interface ChatCompletionChunk {};finish_reason: string | null;}[];+ usage?: TokenUsage;}// ---- Analytics Types ----