7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.
TypeScript
Inject an estimated usage frame when a streaming upstream omits usage (#686)
OPPIH.X0a0bd24e757c
diff --git a/server/src/__tests__/routes/proxy-stream-usage.test.ts b/server/src/__tests__/routes/proxy-stream-usage.test.tsnew file mode 100644index 00000000..5f2aed41--- /dev/null+++ b/server/src/__tests__/routes/proxy-stream-usage.test.ts@@ -0,0 +1,310 @@+import { describe, it, expect, beforeAll, beforeEach, afterEach, vi } from 'vitest';+import type { Express } from 'express';+import { createApp } from '../../app.js';+import { initDb, getDb, getUnifiedApiKey } from '../../db/index.js';+import { mintDashboardToken, isGatedApiPath } from '../helpers/auth.js';++let dashToken = '';++async function request(+ app: Express,+ method: string,+ path: string,+ body?: any,+ headers: Record<string, string> = {},+) {+ const server = app.listen(0);+ const addr = server.address() as any;+ const url = `http://127.0.0.1:${addr.port}${path}`;++ const res = await fetch(url, {+ method,+ headers: {+ ...(body ? { 'Content-Type': 'application/json' } : {}),+ ...(isGatedApiPath(path) && !('Authorization' in headers)+ ? { Authorization: `Bearer ${dashToken}` }+ : {}),+ ...headers,+ },+ body: body ? JSON.stringify(body) : undefined,+ });++ const raw = await res.text();+ server.close();++ let json: any = null;+ try {+ json = JSON.parse(raw);+ } catch {}++ return { status: res.status, body: json, raw };+}++function authHeaders() {+ return { Authorization: `Bearer ${getUnifiedApiKey()}` };+}++/** Build an SSE Response whose frames deliberately contain NO usage block. */+function sseResponseNoUsage(text = 'hi') {+ const encoder = new TextEncoder();+ const frames = [+ `data: {"id":"c1","object":"chat.completion.chunk","created":1,"model":"m","choices":[{"index":0,"delta":{"content":"${text}"},"finish_reason":null}]}\n\n`,+ `data: {"id":"c1","object":"chat.completion.chunk","created":1,"model":"m","choices":[{"index":0,"delta":{},"finish_reason":"stop"}]}\n\n`,+ 'data: [DONE]\n\n',+ ];+ return {+ ok: true,+ body: new ReadableStream<Uint8Array>({+ start(controller) {+ for (const f of frames) controller.enqueue(encoder.encode(f));+ controller.close();+ },+ }),+ headers: new Headers(),+ };+}++/** SSE Response whose final frame carries an upstream usage block. */+function sseResponseWithUsage() {+ const encoder = new TextEncoder();+ const frames = [+ `data: {"id":"c1","object":"chat.completion.chunk","created":1,"model":"m","choices":[{"index":0,"delta":{"content":"hi"},"finish_reason":null}]}\n\n`,+ `data: {"id":"c1","object":"chat.completion.chunk","created":1,"model":"m","choices":[{"index":0,"delta":{},"finish_reason":"stop"}]}\n\n`,+ `data: {"id":"c1","object":"chat.completion.chunk","created":1,"model":"m","choices":[],"usage":{"prompt_tokens":11,"completion_tokens":2,"total_tokens":13}}\n\n`,+ 'data: [DONE]\n\n',+ ];+ return {+ ok: true,+ body: new ReadableStream<Uint8Array>({+ start(controller) {+ for (const f of frames) controller.enqueue(encoder.encode(f));+ controller.close();+ },+ }),+ headers: new Headers(),+ };+}++/** SSE Response with usage-only frame emitted MID-STREAM (before the finish chunk). */+function sseResponseUsageMidStream() {+ const encoder = new TextEncoder();+ const frames = [+ `data: {"id":"c1","object":"chat.completion.chunk","created":1,"model":"m","choices":[{"index":0,"delta":{"content":"hel"},"finish_reason":null}]}\n\n`,+ `data: {"id":"c1","object":"chat.completion.chunk","created":1,"model":"m","choices":[],"usage":{"prompt_tokens":11,"completion_tokens":2,"total_tokens":13}}\n\n`,+ `data: {"id":"c1","object":"chat.completion.chunk","created":1,"model":"m","choices":[{"index":0,"delta":{"content":"lo"},"finish_reason":null}]}\n\n`,+ `data: {"id":"c1","object":"chat.completion.chunk","created":1,"model":"m","choices":[{"index":0,"delta":{},"finish_reason":"stop"}]}\n\n`,+ 'data: [DONE]\n\n',+ ];+ return {+ ok: true,+ body: new ReadableStream<Uint8Array>({+ start(controller) {+ for (const f of frames) controller.enqueue(encoder.encode(f));+ controller.close();+ },+ }),+ headers: new Headers(),+ };+}++/** SSE Response with usage attached to a CONTENT frame (non-final position). */+function sseResponseUsageOnContentFrame() {+ const encoder = new TextEncoder();+ const frames = [+ `data: {"id":"c1","object":"chat.completion.chunk","created":1,"model":"m","choices":[{"index":0,"delta":{"content":"hi"},"finish_reason":null}],"usage":{"prompt_tokens":11,"completion_tokens":2,"total_tokens":13}}\n\n`,+ `data: {"id":"c1","object":"chat.completion.chunk","created":1,"model":"m","choices":[{"index":0,"delta":{},"finish_reason":"stop"}]}\n\n`,+ 'data: [DONE]\n\n',+ ];+ return {+ ok: true,+ body: new ReadableStream<Uint8Array>({+ start(controller) {+ for (const f of frames) controller.enqueue(encoder.encode(f));+ controller.close();+ },+ }),+ headers: new Headers(),+ };+}++function usageFrames(raw: string) {+ return raw+ .split('\n\n')+ .filter((l) => l.startsWith('data:'))+ .map((l) => l.slice(5).trim())+ .filter((p) => p !== '[DONE]')+ .map((p) => JSON.parse(p))+ .filter((f) => f && f.usage);+}++describe('Streaming usage-frame fallback injection', () => {+ let app: Express;++ beforeAll(() => {+ process.env.ENCRYPTION_KEY = '0'.repeat(64);+ initDb(':memory:');+ app = createApp();+ dashToken = mintDashboardToken();+ });++ beforeEach(async () => {+ const db = getDb();+ db.prepare('DELETE FROM api_keys').run();+ db.prepare('DELETE FROM requests').run();++ const addKey = await request(app, 'POST', '/api/keys', {+ platform: 'groq',+ key: 'gsk_stream_usage_test',+ label: 'stream-usage',+ });+ expect(addKey.status).toBe(201);+ });++ afterEach(() => {+ vi.restoreAllMocks();+ });++ it('injects an estimated usage frame when the upstream never echoes one and include_usage is requested', async () => {+ const origFetch = global.fetch;+ vi.spyOn(global, 'fetch').mockImplementation(async (url, init) => {+ const urlStr = typeof url === 'string' ? url : url.toString();+ if (urlStr.includes('api.groq.com/openai/v1/chat/completions')) {+ return sseResponseNoUsage() as any;+ }+ return origFetch(url, init);+ });++ const { status, raw } = await request(app, 'POST', '/v1/chat/completions', {+ messages: [{ role: 'user', content: 'hi' }],+ stream: true,+ stream_options: { include_usage: true },+ }, authHeaders());++ expect(status).toBe(200);+ const frames = usageFrames(raw);+ expect(frames).toHaveLength(1);+ // Usage-only frame: empty choices, estimated positive token counts.+ expect(frames[0].choices).toEqual([]);+ expect(frames[0].usage.prompt_tokens).toBeGreaterThan(0);+ expect(frames[0].usage.completion_tokens).toBeGreaterThan(0);+ expect(frames[0].usage.total_tokens).toBe(+ frames[0].usage.prompt_tokens + frames[0].usage.completion_tokens,+ );+ // Synthetic counts are flagged: a client doing cost accounting can tell+ // this block apart from an upstream's real one.+ expect(frames[0].usage.estimated).toBe(true);+ });++ it('does NOT inject a usage frame when include_usage is not requested', async () => {+ const origFetch = global.fetch;+ vi.spyOn(global, 'fetch').mockImplementation(async (url, init) => {+ const urlStr = typeof url === 'string' ? url : url.toString();+ if (urlStr.includes('api.groq.com/openai/v1/chat/completions')) {+ return sseResponseNoUsage() as any;+ }+ return origFetch(url, init);+ });++ const { status, raw } = await request(app, 'POST', '/v1/chat/completions', {+ messages: [{ role: 'user', content: 'hi' }],+ stream: true,+ }, authHeaders());++ expect(status).toBe(200);+ expect(usageFrames(raw)).toHaveLength(0);+ });++ it('passes through the upstream usage frame unchanged when one is present', async () => {+ const origFetch = global.fetch;+ vi.spyOn(global, 'fetch').mockImplementation(async (url, init) => {+ const urlStr = typeof url === 'string' ? url : url.toString();+ if (urlStr.includes('api.groq.com/openai/v1/chat/completions')) {+ return sseResponseWithUsage() as any;+ }+ return origFetch(url, init);+ });++ const { status, raw } = await request(app, 'POST', '/v1/chat/completions', {+ messages: [{ role: 'user', content: 'hi' }],+ stream: true,+ stream_options: { include_usage: true },+ }, authHeaders());++ expect(status).toBe(200);+ const frames = usageFrames(raw);+ expect(frames).toHaveLength(1);+ expect(frames[0].usage).toEqual({+ prompt_tokens: 11,+ completion_tokens: 2,+ total_tokens: 13,+ });+ // Real upstream counts are never relabelled as estimates.+ expect(frames[0].usage).not.toHaveProperty('estimated');+ });++ it('does NOT inject an estimate when the upstream emits a usage frame mid-stream (non-final position)', async () => {+ const origFetch = global.fetch;+ vi.spyOn(global, 'fetch').mockImplementation(async (url, init) => {+ const urlStr = typeof url === 'string' ? url : url.toString();+ if (urlStr.includes('api.groq.com/openai/v1/chat/completions')) {+ return sseResponseUsageMidStream() as any;+ }+ return origFetch(url, init);+ });++ const { status, raw } = await request(app, 'POST', '/v1/chat/completions', {+ messages: [{ role: 'user', content: 'hi' }],+ stream: true,+ stream_options: { include_usage: true },+ }, authHeaders());++ expect(status).toBe(200);+ // The upstream's real mid-stream frame must be the ONLY usage frame —+ // no estimate injected on top of it.+ const frames = usageFrames(raw);+ expect(frames).toHaveLength(1);+ expect(frames[0].usage).toEqual({+ prompt_tokens: 11,+ completion_tokens: 2,+ total_tokens: 13,+ });+ // Stream must still carry the full content ("hel" + "lo").+ const allFrames = raw+ .split('\n\n')+ .filter((l) => l.startsWith('data:'))+ .map((l) => l.slice(5).trim())+ .filter((p) => p !== '[DONE]')+ .map((p) => JSON.parse(p));+ const content = allFrames+ .flatMap((f) => f.choices?.map((c: any) => c.delta?.content ?? '') ?? [])+ .join('');+ expect(content).toBe('hello');+ });++ it('does NOT inject an estimate when usage rides on a content frame (non-final position)', async () => {+ const origFetch = global.fetch;+ vi.spyOn(global, 'fetch').mockImplementation(async (url, init) => {+ const urlStr = typeof url === 'string' ? url : url.toString();+ if (urlStr.includes('api.groq.com/openai/v1/chat/completions')) {+ return sseResponseUsageOnContentFrame() as any;+ }+ return origFetch(url, init);+ });++ const { status, raw } = await request(app, 'POST', '/v1/chat/completions', {+ messages: [{ role: 'user', content: 'hi' }],+ stream: true,+ stream_options: { include_usage: true },+ }, authHeaders());++ expect(status).toBe(200);+ // Real usage must be preserved and the estimate must NOT fire.+ const frames = usageFrames(raw);+ expect(frames).toHaveLength(1);+ expect(frames[0].usage).toEqual({+ prompt_tokens: 11,+ completion_tokens: 2,+ total_tokens: 13,+ });+ });+});diff --git a/server/src/routes/proxy.ts b/server/src/routes/proxy.tsindex 29af8596..ae4d3a41 100644--- a/server/src/routes/proxy.ts+++ b/server/src/routes/proxy.ts@@ -2294,12 +2294,47 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {? 'tool_calls': (upstreamFinish && upstreamFinish !== 'tool_calls' ? upstreamFinish : 'stop');writeChunk(mkChunk({}, finish));- if (usageChunk) writeChunk(usageChunk);+ // One prompt-token estimate for both the injected usage frame below+ // and the accounting fallback after it, so a client that reads the+ // frame and the row this request writes can never disagree. Images+ // are billed at the same flat per-image estimate the routing budget+ // uses (the chars/4 pass sees text only).+ const estimatedPromptTokens = estimatedInputTokens + injectedHandoffTokens + imageCount * IMAGE_TOKEN_ESTIMATE;+ if (usageChunk) {+ writeChunk(usageChunk);+ } else if (parsed.data.stream_options?.include_usage) {+ // Some OpenAI-compatible upstreams (e.g. OpenCode Zen) never echo+ // a final usage frame even when stream_options.include_usage is+ // requested. Strict clients (Hermes, Cline, Continue) treat a+ // missing usage block as "no accounting happened" and skip+ // per-call token/cost/billing_provider writes entirely.+ //+ // Injected ONLY when the client asked for usage via+ // stream_options.include_usage AND the upstream never sent one;+ // the numbers are this gateway's own chars/4 estimate (the same+ // total the accounting below records), never the upstream's+ // accounting, so the block is flagged `estimated: true` rather+ // than passed off as real counts.+ const completionTokens = totalOutputTokens;+ writeChunk({+ id: lastMeta.id ?? `chatcmpl-${Date.now()}`,+ object: 'chat.completion.chunk',+ created: lastMeta.created ?? Math.floor(Date.now() / 1000),+ model: lastMeta.model ?? route.modelId,+ choices: [],+ usage: {+ prompt_tokens: estimatedPromptTokens,+ completion_tokens: completionTokens,+ total_tokens: estimatedPromptTokens + completionTokens,+ estimated: true,+ },+ });+ }res.write('data: [DONE]\n\n');res.end();const upstreamUsage = (usageChunk as { usage?: TokenUsage } | null)?.usage;- const inputTokens = upstreamUsage?.prompt_tokens ?? (estimatedInputTokens + injectedHandoffTokens);+ const inputTokens = upstreamUsage?.prompt_tokens ?? estimatedPromptTokens;const outputTokens = upstreamUsage?.completion_tokens ?? totalOutputTokens;const totalTokens = upstreamUsage?.total_tokens ?? (inputTokens + outputTokens);recordUpstreamSuccess(route, totalTokens);