Skip to content

7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.

TypeScript

Inject an estimated usage frame when a streaming upstream omits usage (#686)

OPPIH.X0a0bd24e757c
diff --git a/server/src/__tests__/routes/proxy-stream-usage.test.ts b/server/src/__tests__/routes/proxy-stream-usage.test.ts
new file mode 100644
index 00000000..5f2aed41
--- /dev/null
+++ b/server/src/__tests__/routes/proxy-stream-usage.test.ts
@@ -0,0 +1,310 @@
+import { describe, it, expect, beforeAll, beforeEach, afterEach, vi } from 'vitest';
+import type { Express } from 'express';
+import { createApp } from '../../app.js';
+import { initDb, getDb, getUnifiedApiKey } from '../../db/index.js';
+import { mintDashboardToken, isGatedApiPath } from '../helpers/auth.js';
+
+let dashToken = '';
+
+async function request(
+ app: Express,
+ method: string,
+ path: string,
+ body?: any,
+ headers: Record<string, string> = {},
+) {
+ const server = app.listen(0);
+ const addr = server.address() as any;
+ const url = `http://127.0.0.1:${addr.port}${path}`;
+
+ const res = await fetch(url, {
+ method,
+ headers: {
+ ...(body ? { 'Content-Type': 'application/json' } : {}),
+ ...(isGatedApiPath(path) && !('Authorization' in headers)
+ ? { Authorization: `Bearer ${dashToken}` }
+ : {}),
+ ...headers,
+ },
+ body: body ? JSON.stringify(body) : undefined,
+ });
+
+ const raw = await res.text();
+ server.close();
+
+ let json: any = null;
+ try {
+ json = JSON.parse(raw);
+ } catch {}
+
+ return { status: res.status, body: json, raw };
+}
+
+function authHeaders() {
+ return { Authorization: `Bearer ${getUnifiedApiKey()}` };
+}
+
+/** Build an SSE Response whose frames deliberately contain NO usage block. */
+function sseResponseNoUsage(text = 'hi') {
+ const encoder = new TextEncoder();
+ const frames = [
+ `data: {"id":"c1","object":"chat.completion.chunk","created":1,"model":"m","choices":[{"index":0,"delta":{"content":"${text}"},"finish_reason":null}]}\n\n`,
+ `data: {"id":"c1","object":"chat.completion.chunk","created":1,"model":"m","choices":[{"index":0,"delta":{},"finish_reason":"stop"}]}\n\n`,
+ 'data: [DONE]\n\n',
+ ];
+ return {
+ ok: true,
+ body: new ReadableStream<Uint8Array>({
+ start(controller) {
+ for (const f of frames) controller.enqueue(encoder.encode(f));
+ controller.close();
+ },
+ }),
+ headers: new Headers(),
+ };
+}
+
+/** SSE Response whose final frame carries an upstream usage block. */
+function sseResponseWithUsage() {
+ const encoder = new TextEncoder();
+ const frames = [
+ `data: {"id":"c1","object":"chat.completion.chunk","created":1,"model":"m","choices":[{"index":0,"delta":{"content":"hi"},"finish_reason":null}]}\n\n`,
+ `data: {"id":"c1","object":"chat.completion.chunk","created":1,"model":"m","choices":[{"index":0,"delta":{},"finish_reason":"stop"}]}\n\n`,
+ `data: {"id":"c1","object":"chat.completion.chunk","created":1,"model":"m","choices":[],"usage":{"prompt_tokens":11,"completion_tokens":2,"total_tokens":13}}\n\n`,
+ 'data: [DONE]\n\n',
+ ];
+ return {
+ ok: true,
+ body: new ReadableStream<Uint8Array>({
+ start(controller) {
+ for (const f of frames) controller.enqueue(encoder.encode(f));
+ controller.close();
+ },
+ }),
+ headers: new Headers(),
+ };
+}
+
+/** SSE Response with usage-only frame emitted MID-STREAM (before the finish chunk). */
+function sseResponseUsageMidStream() {
+ const encoder = new TextEncoder();
+ const frames = [
+ `data: {"id":"c1","object":"chat.completion.chunk","created":1,"model":"m","choices":[{"index":0,"delta":{"content":"hel"},"finish_reason":null}]}\n\n`,
+ `data: {"id":"c1","object":"chat.completion.chunk","created":1,"model":"m","choices":[],"usage":{"prompt_tokens":11,"completion_tokens":2,"total_tokens":13}}\n\n`,
+ `data: {"id":"c1","object":"chat.completion.chunk","created":1,"model":"m","choices":[{"index":0,"delta":{"content":"lo"},"finish_reason":null}]}\n\n`,
+ `data: {"id":"c1","object":"chat.completion.chunk","created":1,"model":"m","choices":[{"index":0,"delta":{},"finish_reason":"stop"}]}\n\n`,
+ 'data: [DONE]\n\n',
+ ];
+ return {
+ ok: true,
+ body: new ReadableStream<Uint8Array>({
+ start(controller) {
+ for (const f of frames) controller.enqueue(encoder.encode(f));
+ controller.close();
+ },
+ }),
+ headers: new Headers(),
+ };
+}
+
+/** SSE Response with usage attached to a CONTENT frame (non-final position). */
+function sseResponseUsageOnContentFrame() {
+ const encoder = new TextEncoder();
+ const frames = [
+ `data: {"id":"c1","object":"chat.completion.chunk","created":1,"model":"m","choices":[{"index":0,"delta":{"content":"hi"},"finish_reason":null}],"usage":{"prompt_tokens":11,"completion_tokens":2,"total_tokens":13}}\n\n`,
+ `data: {"id":"c1","object":"chat.completion.chunk","created":1,"model":"m","choices":[{"index":0,"delta":{},"finish_reason":"stop"}]}\n\n`,
+ 'data: [DONE]\n\n',
+ ];
+ return {
+ ok: true,
+ body: new ReadableStream<Uint8Array>({
+ start(controller) {
+ for (const f of frames) controller.enqueue(encoder.encode(f));
+ controller.close();
+ },
+ }),
+ headers: new Headers(),
+ };
+}
+
+function usageFrames(raw: string) {
+ return raw
+ .split('\n\n')
+ .filter((l) => l.startsWith('data:'))
+ .map((l) => l.slice(5).trim())
+ .filter((p) => p !== '[DONE]')
+ .map((p) => JSON.parse(p))
+ .filter((f) => f && f.usage);
+}
+
+describe('Streaming usage-frame fallback injection', () => {
+ let app: Express;
+
+ beforeAll(() => {
+ process.env.ENCRYPTION_KEY = '0'.repeat(64);
+ initDb(':memory:');
+ app = createApp();
+ dashToken = mintDashboardToken();
+ });
+
+ beforeEach(async () => {
+ const db = getDb();
+ db.prepare('DELETE FROM api_keys').run();
+ db.prepare('DELETE FROM requests').run();
+
+ const addKey = await request(app, 'POST', '/api/keys', {
+ platform: 'groq',
+ key: 'gsk_stream_usage_test',
+ label: 'stream-usage',
+ });
+ expect(addKey.status).toBe(201);
+ });
+
+ afterEach(() => {
+ vi.restoreAllMocks();
+ });
+
+ it('injects an estimated usage frame when the upstream never echoes one and include_usage is requested', async () => {
+ const origFetch = global.fetch;
+ vi.spyOn(global, 'fetch').mockImplementation(async (url, init) => {
+ const urlStr = typeof url === 'string' ? url : url.toString();
+ if (urlStr.includes('api.groq.com/openai/v1/chat/completions')) {
+ return sseResponseNoUsage() as any;
+ }
+ return origFetch(url, init);
+ });
+
+ const { status, raw } = await request(app, 'POST', '/v1/chat/completions', {
+ messages: [{ role: 'user', content: 'hi' }],
+ stream: true,
+ stream_options: { include_usage: true },
+ }, authHeaders());
+
+ expect(status).toBe(200);
+ const frames = usageFrames(raw);
+ expect(frames).toHaveLength(1);
+ // Usage-only frame: empty choices, estimated positive token counts.
+ expect(frames[0].choices).toEqual([]);
+ expect(frames[0].usage.prompt_tokens).toBeGreaterThan(0);
+ expect(frames[0].usage.completion_tokens).toBeGreaterThan(0);
+ expect(frames[0].usage.total_tokens).toBe(
+ frames[0].usage.prompt_tokens + frames[0].usage.completion_tokens,
+ );
+ // Synthetic counts are flagged: a client doing cost accounting can tell
+ // this block apart from an upstream's real one.
+ expect(frames[0].usage.estimated).toBe(true);
+ });
+
+ it('does NOT inject a usage frame when include_usage is not requested', async () => {
+ const origFetch = global.fetch;
+ vi.spyOn(global, 'fetch').mockImplementation(async (url, init) => {
+ const urlStr = typeof url === 'string' ? url : url.toString();
+ if (urlStr.includes('api.groq.com/openai/v1/chat/completions')) {
+ return sseResponseNoUsage() as any;
+ }
+ return origFetch(url, init);
+ });
+
+ const { status, raw } = await request(app, 'POST', '/v1/chat/completions', {
+ messages: [{ role: 'user', content: 'hi' }],
+ stream: true,
+ }, authHeaders());
+
+ expect(status).toBe(200);
+ expect(usageFrames(raw)).toHaveLength(0);
+ });
+
+ it('passes through the upstream usage frame unchanged when one is present', async () => {
+ const origFetch = global.fetch;
+ vi.spyOn(global, 'fetch').mockImplementation(async (url, init) => {
+ const urlStr = typeof url === 'string' ? url : url.toString();
+ if (urlStr.includes('api.groq.com/openai/v1/chat/completions')) {
+ return sseResponseWithUsage() as any;
+ }
+ return origFetch(url, init);
+ });
+
+ const { status, raw } = await request(app, 'POST', '/v1/chat/completions', {
+ messages: [{ role: 'user', content: 'hi' }],
+ stream: true,
+ stream_options: { include_usage: true },
+ }, authHeaders());
+
+ expect(status).toBe(200);
+ const frames = usageFrames(raw);
+ expect(frames).toHaveLength(1);
+ expect(frames[0].usage).toEqual({
+ prompt_tokens: 11,
+ completion_tokens: 2,
+ total_tokens: 13,
+ });
+ // Real upstream counts are never relabelled as estimates.
+ expect(frames[0].usage).not.toHaveProperty('estimated');
+ });
+
+ it('does NOT inject an estimate when the upstream emits a usage frame mid-stream (non-final position)', async () => {
+ const origFetch = global.fetch;
+ vi.spyOn(global, 'fetch').mockImplementation(async (url, init) => {
+ const urlStr = typeof url === 'string' ? url : url.toString();
+ if (urlStr.includes('api.groq.com/openai/v1/chat/completions')) {
+ return sseResponseUsageMidStream() as any;
+ }
+ return origFetch(url, init);
+ });
+
+ const { status, raw } = await request(app, 'POST', '/v1/chat/completions', {
+ messages: [{ role: 'user', content: 'hi' }],
+ stream: true,
+ stream_options: { include_usage: true },
+ }, authHeaders());
+
+ expect(status).toBe(200);
+ // The upstream's real mid-stream frame must be the ONLY usage frame —
+ // no estimate injected on top of it.
+ const frames = usageFrames(raw);
+ expect(frames).toHaveLength(1);
+ expect(frames[0].usage).toEqual({
+ prompt_tokens: 11,
+ completion_tokens: 2,
+ total_tokens: 13,
+ });
+ // Stream must still carry the full content ("hel" + "lo").
+ const allFrames = raw
+ .split('\n\n')
+ .filter((l) => l.startsWith('data:'))
+ .map((l) => l.slice(5).trim())
+ .filter((p) => p !== '[DONE]')
+ .map((p) => JSON.parse(p));
+ const content = allFrames
+ .flatMap((f) => f.choices?.map((c: any) => c.delta?.content ?? '') ?? [])
+ .join('');
+ expect(content).toBe('hello');
+ });
+
+ it('does NOT inject an estimate when usage rides on a content frame (non-final position)', async () => {
+ const origFetch = global.fetch;
+ vi.spyOn(global, 'fetch').mockImplementation(async (url, init) => {
+ const urlStr = typeof url === 'string' ? url : url.toString();
+ if (urlStr.includes('api.groq.com/openai/v1/chat/completions')) {
+ return sseResponseUsageOnContentFrame() as any;
+ }
+ return origFetch(url, init);
+ });
+
+ const { status, raw } = await request(app, 'POST', '/v1/chat/completions', {
+ messages: [{ role: 'user', content: 'hi' }],
+ stream: true,
+ stream_options: { include_usage: true },
+ }, authHeaders());
+
+ expect(status).toBe(200);
+ // Real usage must be preserved and the estimate must NOT fire.
+ const frames = usageFrames(raw);
+ expect(frames).toHaveLength(1);
+ expect(frames[0].usage).toEqual({
+ prompt_tokens: 11,
+ completion_tokens: 2,
+ total_tokens: 13,
+ });
+ });
+});
diff --git a/server/src/routes/proxy.ts b/server/src/routes/proxy.ts
index 29af8596..ae4d3a41 100644
--- a/server/src/routes/proxy.ts
+++ b/server/src/routes/proxy.ts
@@ -2294,12 +2294,47 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {
? 'tool_calls'
: (upstreamFinish && upstreamFinish !== 'tool_calls' ? upstreamFinish : 'stop');
writeChunk(mkChunk({}, finish));
- if (usageChunk) writeChunk(usageChunk);
+ // One prompt-token estimate for both the injected usage frame below
+ // and the accounting fallback after it, so a client that reads the
+ // frame and the row this request writes can never disagree. Images
+ // are billed at the same flat per-image estimate the routing budget
+ // uses (the chars/4 pass sees text only).
+ const estimatedPromptTokens = estimatedInputTokens + injectedHandoffTokens + imageCount * IMAGE_TOKEN_ESTIMATE;
+ if (usageChunk) {
+ writeChunk(usageChunk);
+ } else if (parsed.data.stream_options?.include_usage) {
+ // Some OpenAI-compatible upstreams (e.g. OpenCode Zen) never echo
+ // a final usage frame even when stream_options.include_usage is
+ // requested. Strict clients (Hermes, Cline, Continue) treat a
+ // missing usage block as "no accounting happened" and skip
+ // per-call token/cost/billing_provider writes entirely.
+ //
+ // Injected ONLY when the client asked for usage via
+ // stream_options.include_usage AND the upstream never sent one;
+ // the numbers are this gateway's own chars/4 estimate (the same
+ // total the accounting below records), never the upstream's
+ // accounting, so the block is flagged `estimated: true` rather
+ // than passed off as real counts.
+ const completionTokens = totalOutputTokens;
+ writeChunk({
+ id: lastMeta.id ?? `chatcmpl-${Date.now()}`,
+ object: 'chat.completion.chunk',
+ created: lastMeta.created ?? Math.floor(Date.now() / 1000),
+ model: lastMeta.model ?? route.modelId,
+ choices: [],
+ usage: {
+ prompt_tokens: estimatedPromptTokens,
+ completion_tokens: completionTokens,
+ total_tokens: estimatedPromptTokens + completionTokens,
+ estimated: true,
+ },
+ });
+ }
res.write('data: [DONE]\n\n');
res.end();
const upstreamUsage = (usageChunk as { usage?: TokenUsage } | null)?.usage;
- const inputTokens = upstreamUsage?.prompt_tokens ?? (estimatedInputTokens + injectedHandoffTokens);
+ const inputTokens = upstreamUsage?.prompt_tokens ?? estimatedPromptTokens;
const outputTokens = upstreamUsage?.completion_tokens ?? totalOutputTokens;
const totalTokens = upstreamUsage?.total_tokens ?? (inputTokens + outputTokens);
recordUpstreamSuccess(route, totalTokens);