Skip to content

7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.

TypeScript

fix(openai-compat): record real token usage from streaming responses (#845)

xyblue13573453b5271db
diff --git a/server/src/__tests__/routes/proxy-stream-integrity.test.ts b/server/src/__tests__/routes/proxy-stream-integrity.test.ts
index 53f17ed6..95e700ec 100644
--- a/server/src/__tests__/routes/proxy-stream-integrity.test.ts
+++ b/server/src/__tests__/routes/proxy-stream-integrity.test.ts
@@ -49,7 +49,7 @@ const TOOLS = [{ type: 'function', function: { name: 'Read', description: 'read
function mockUpstream(script: Array<{ body: string; status?: number }>) {
const origFetch = global.fetch;
let call = 0;
- const seen: Array<{ model: string }> = [];
+ const seen: Array<{ model: string; streamOptions?: { include_usage?: boolean } }> = [];
vi.spyOn(global, 'fetch').mockImplementation(async (url, init) => {
const urlStr = typeof url === 'string' ? url : url.toString();
// Only intercept provider upstreams; the test's own localhost request
@@ -58,7 +58,7 @@ function mockUpstream(script: Array<{ body: string; status?: number }>) {
return origFetch(url as any, init);
}
const reqBody = JSON.parse(String((init as RequestInit).body));
- seen.push({ model: reqBody.model });
+ seen.push({ model: reqBody.model, streamOptions: reqBody.stream_options });
const step = script[Math.min(call++, script.length - 1)];
return new Response(step.body, {
status: step.status ?? 200,
@@ -268,6 +268,39 @@ describe('proxy stream turn-integrity', () => {
expect(r.text.trim().endsWith('data: [DONE]')).toBe(true);
});
+ it('forwards stream_options.include_usage and records upstream token usage', async () => {
+ const usage = { prompt_tokens: 3263, completion_tokens: 15, total_tokens: 3278 };
+ const up = mockUpstream([{
+ body: sse(
+ roleChunk,
+ textChunk('Hello'),
+ finishChunk('stop'),
+ { id: 'c1', object: 'chat.completion.chunk', created: 1, model: 'm', choices: [], usage },
+ '[DONE]',
+ ),
+ }]);
+
+ const r = await request(app, '/v1/chat/completions', {
+ stream: true,
+ stream_options: { include_usage: true },
+ messages: [{ role: 'user', content: 'real streaming usage test' }],
+ });
+
+ expect(r.status).toBe(200);
+ expect(up.seen[0].streamOptions).toEqual({ include_usage: true });
+ expect(frames(r.text).find(f => f.usage)?.usage).toEqual(usage);
+
+ const row = getDb().prepare(
+ "SELECT input_tokens, output_tokens FROM requests WHERE status = 'success' ORDER BY id DESC LIMIT 1",
+ ).get() as { input_tokens: number; output_tokens: number };
+ expect(row).toEqual({ input_tokens: 3263, output_tokens: 15 });
+
+ const ledger = getDb().prepare(
+ "SELECT COALESCE(SUM(tokens), 0) AS total FROM rate_limit_usage WHERE kind = 'tokens'",
+ ).get() as { total: number };
+ expect(ledger.total).toBe(3278);
+ });
+
it('rescues a non-streaming inline dialect answer into structured tool_calls', async () => {
const origFetch = global.fetch;
vi.spyOn(global, 'fetch').mockImplementation(async (url, init) => {
diff --git a/server/src/providers/base.ts b/server/src/providers/base.ts
index d27f60f9..bb471ebd 100644
--- a/server/src/providers/base.ts
+++ b/server/src/providers/base.ts
@@ -160,6 +160,9 @@ export interface CompletionOptions extends ExtendedSamplingOptions {
tools?: ChatToolDefinition[];
tool_choice?: ChatToolChoice;
parallel_tool_calls?: boolean;
+ stream_options?: {
+ include_usage?: boolean;
+ };
/** Per-call HTTP timeout override. Not part of the OpenAI wire format (it is
* stripped before the request body is built); used by the probe script so
* NVIDIA's 15-60s serverless cold starts don't read as failures. */
diff --git a/server/src/providers/openai-compat.ts b/server/src/providers/openai-compat.ts
index 08b1c173..7747096f 100644
--- a/server/src/providers/openai-compat.ts
+++ b/server/src/providers/openai-compat.ts
@@ -325,6 +325,7 @@ export class OpenAICompatProvider extends BaseProvider {
parallel_tool_calls: this.resolveParallelToolCalls(options),
...extendedBodyParams(this.platform, options),
stream: true,
+ stream_options: options?.stream_options,
}),
// Default 'headers' bounds: the deadline dies at response headers, and
// the client signal + stall watchdog own the stream from there.
diff --git a/server/src/routes/proxy.ts b/server/src/routes/proxy.ts
index 94c35bcd..0817dfc3 100644
--- a/server/src/routes/proxy.ts
+++ b/server/src/routes/proxy.ts
@@ -2,7 +2,7 @@ import crypto from 'crypto';
import { Router } from 'express';
import type { Request, Response } from 'express';
import { z } from 'zod';
-import type { ChatMessage, ChatToolCall, ModelListRow } from '@freellmapi/shared/types.js';
+import type { ChatMessage, ChatToolCall, ModelListRow, TokenUsage } from '@freellmapi/shared/types.js';
import { routeRequest, resolveRoutingChain, resolveModelGroupCandidates, resolveStickyPreference, recordRateLimitHit, recordSuccess, hasEnabledVisionModel, hasEnabledToolsModel, hasOtherUsableKey, routingReserveTokens, type RouteResult, type ResolvedChain, type ChainRow } from '../services/router.js';
import { recordRequest, recordTokens, setCooldown, getCooldownDurationForLimit, PAYMENT_REQUIRED_COOLDOWN_MS, MODEL_FORBIDDEN_COOLDOWN_MS, learnLimitFromError } from '../services/ratelimit.js';
import { runEmbeddings, EmbeddingsError } from '../services/embeddings.js';
@@ -500,6 +500,9 @@ const chatCompletionSchema = z.object({
top_p: z.number().min(0).max(1).optional(),
stop: stopSchema.optional(),
stream: z.boolean().optional(),
+ stream_options: z.object({
+ include_usage: z.boolean().optional(),
+ }).optional(),
// Top-level tool knobs may arrive as explicit nulls from clients that
// serialize every field of their request struct; all treated as absent
// and never forwarded as null. (#200)
@@ -1388,9 +1391,9 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {
// Token estimation is intentionally a heuristic (~4 chars per token). Used
// for routing decisions (skip a model whose budget is too small) and for
// streaming bookkeeping where the provider doesn't echo a final usage count.
- // Non-streaming requests reconcile against the provider's real `usage` block
- // (see line ~340). Streaming will drift from real consumption — accepted
- // tradeoff because per-request usage isn't always returned mid-stream.
+ // Non-streaming requests reconcile against the provider's real `usage` block;
+ // streaming does the same when stream_options.include_usage produces a final
+ // usage frame, and otherwise falls back to this estimate.
const estimatedInputTokens = messages.reduce((sum, m) => {
const text = contentToString(m.content);
return sum + Math.ceil(text.length / 4);
@@ -1880,7 +1883,7 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {
try {
const gen = route.provider.streamChatCompletion(
route.apiKey, outboundMessages, route.modelId,
- { temperature, max_tokens, top_p, stop, tools, tool_choice, parallel_tool_calls, ...samplingParams, signal: clientAbort.signal },
+ { temperature, max_tokens, top_p, stop, tools, tool_choice, parallel_tool_calls, stream_options: parsed.data.stream_options, ...samplingParams, signal: clientAbort.signal },
quotaContextForRoute(route, 'chat/completions'),
);
@@ -2094,7 +2097,11 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {
res.write('data: [DONE]\n\n');
res.end();
- recordUpstreamSuccess(route, estimatedInputTokens + injectedHandoffTokens + totalOutputTokens);
+ const upstreamUsage = (usageChunk as { usage?: TokenUsage } | null)?.usage;
+ const inputTokens = upstreamUsage?.prompt_tokens ?? (estimatedInputTokens + injectedHandoffTokens);
+ const outputTokens = upstreamUsage?.completion_tokens ?? totalOutputTokens;
+ const totalTokens = upstreamUsage?.total_tokens ?? (inputTokens + outputTokens);
+ recordUpstreamSuccess(route, totalTokens);
setStickyModel(messages, route.modelDbId, sessionIdHeader, stickyStrategyKey);
if (handoffMode !== 'off' && sessionKey) recordSuccessfulModel({ sessionKey, modelKey });
// #797: remember this turn's thinking trace so the next request from
@@ -2107,10 +2114,10 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {
platform: route.platform,
model: route.modelId,
latencyMs: Date.now() - start,
- inputTokens: estimatedInputTokens + injectedHandoffTokens,
- outputTokens: totalOutputTokens,
+ inputTokens,
+ outputTokens,
});
- logRequest(route.platform, route.modelId, route.keyId, 'success', estimatedInputTokens + injectedHandoffTokens, totalOutputTokens, Date.now() - start, null, ttfbMs, pinnedModelId,
+ logRequest(route.platform, route.modelId, route.keyId, 'success', inputTokens, outputTokens, Date.now() - start, null, ttfbMs, pinnedModelId,
observeServedModel({ platform: route.platform, requestedModel: route.modelId, servedModel: upstreamModel }));
return 'done';
} catch (streamErr: any) {
diff --git a/shared/types.ts b/shared/types.ts
index 0c36908e..1bc05f37 100644
--- a/shared/types.ts
+++ b/shared/types.ts
@@ -339,6 +339,9 @@ export interface ChatCompletionRequest {
temperature?: number;
max_tokens?: number;
stream?: boolean;
+ stream_options?: {
+ include_usage?: boolean;
+ };
top_p?: number;
stop?: string | string[];
tools?: ChatToolDefinition[];
@@ -389,6 +392,7 @@ export interface ChatCompletionChunk {
};
finish_reason: string | null;
}[];
+ usage?: TokenUsage;
}
// ---- Analytics Types ----