7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.
TypeScript
fix(proxy): always surface usage so agents show context-window counts (#1084) (#1120)
suantea (asuan_suan)8c8a69072cf2
diff --git a/server/src/__tests__/routes/proxy-stream-usage.test.ts b/server/src/__tests__/routes/proxy-stream-usage.test.tsindex 5f2aed41..830c5f85 100644--- a/server/src/__tests__/routes/proxy-stream-usage.test.ts+++ b/server/src/__tests__/routes/proxy-stream-usage.test.ts@@ -195,7 +195,7 @@ describe('Streaming usage-frame fallback injection', () => {expect(frames[0].usage.estimated).toBe(true);});- it('does NOT inject a usage frame when include_usage is not requested', async () => {+ it('injects an estimated usage frame when the upstream never echoes one, even without include_usage', async () => {const origFetch = global.fetch;vi.spyOn(global, 'fetch').mockImplementation(async (url, init) => {const urlStr = typeof url === 'string' ? url : url.toString();@@ -211,7 +211,53 @@ describe('Streaming usage-frame fallback injection', () => {}, authHeaders());expect(status).toBe(200);- expect(usageFrames(raw)).toHaveLength(0);+ // #1084: agents read `usage` for context-window display even when they+ // never requested include_usage; a missing frame used to show 0. The+ // estimate is injected regardless of the client's stream_options.+ const frames = usageFrames(raw);+ expect(frames).toHaveLength(1);+ expect(frames[0].choices).toEqual([]);+ expect(frames[0].usage.prompt_tokens).toBeGreaterThan(0);+ expect(frames[0].usage.completion_tokens).toBeGreaterThan(0);+ expect(frames[0].usage.estimated).toBe(true);+ });++ it('injects estimated usage into a NON-streaming response when the upstream omits usage', async () => {+ const origFetch = global.fetch;+ vi.spyOn(global, 'fetch').mockImplementation(async (url, init) => {+ const urlStr = typeof url === 'string' ? url : url.toString();+ if (urlStr.includes('api.groq.com/openai/v1/chat/completions')) {+ return {+ ok: true,+ json: () => Promise.resolve({+ id: 'chatcmpl-nousage',+ object: 'chat.completion',+ created: 123,+ model: 'openai/gpt-oss-120b',+ choices: [{+ index: 0,+ message: { role: 'assistant', content: 'hi there' },+ finish_reason: 'stop',+ }],+ // No `usage` block — the #1084 scenario.+ }),+ } as any;+ }+ return origFetch(url, init);+ });++ const { status, body } = await request(app, 'POST', '/v1/chat/completions', {+ messages: [{ role: 'user', content: 'hi' }],+ }, authHeaders());++ expect(status).toBe(200);+ expect(body.usage).toBeDefined();+ expect(body.usage.prompt_tokens).toBeGreaterThan(0);+ expect(body.usage.completion_tokens).toBeGreaterThan(0);+ expect(body.usage.total_tokens).toBe(+ body.usage.prompt_tokens + body.usage.completion_tokens,+ );+ expect(body.usage.estimated).toBe(true);});it('passes through the upstream usage frame unchanged when one is present', async () => {diff --git a/server/src/routes/proxy.ts b/server/src/routes/proxy.tsindex 38e2dac7..7fb77b26 100644--- a/server/src/routes/proxy.ts+++ b/server/src/routes/proxy.ts@@ -2392,19 +2392,20 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {const estimatedPromptTokens = estimatedInputTokens + injectedHandoffTokens + imageCount * IMAGE_TOKEN_ESTIMATE;if (usageChunk) {writeChunk(usageChunk);- } else if (parsed.data.stream_options?.include_usage) {- // Some OpenAI-compatible upstreams (e.g. OpenCode Zen) never echo- // a final usage frame even when stream_options.include_usage is- // requested. Strict clients (Hermes, Cline, Continue) treat a+ } else {+ // Some OpenAI-compatible upstreams never echo a final usage+ // frame — neither when stream_options.include_usage is requested+ // nor otherwise. Strict clients (Hermes, Cline, Continue) treat a// missing usage block as "no accounting happened" and skip- // per-call token/cost/billing_provider writes entirely.+ // per-call token/cost/billing_provider writes entirely; agents+ // that read usage for context-window display (e.g. #1084) show 0.//- // Injected ONLY when the client asked for usage via- // stream_options.include_usage AND the upstream never sent one;- // the numbers are this gateway's own chars/4 estimate (the same- // total the accounting below records), never the upstream's- // accounting, so the block is flagged `estimated: true` rather- // than passed off as real counts.+ // So inject the estimate whenever the upstream never sent one —+ // regardless of whether the client asked for include_usage. The+ // numbers are this gateway's own chars/4 estimate (the same total+ // the accounting below records), never the upstream's accounting,+ // so the block is flagged `estimated: true` rather than passed+ // off as real counts.const completionTokens = totalOutputTokens;writeChunk({id: lastMeta.id ?? `chatcmpl-${Date.now()}`,@@ -2637,6 +2638,20 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {const outboundBody = sanitizeResponse(normalizeOutboundContent(result));res.setHeader('X-FreeLLM-Cache', cacheKey ? 'MISS' : 'OFF');+ // #1084: agents show zero context usage when the upstream omits+ // `usage` (many free-tier providers do). Fall back to the same+ // chars/4 estimate used for accounting above, flagged `estimated:+ // true` so a cost-accounting client can tell it apart from the+ // upstream's real counts.+ if (!outboundBody.usage) {+ outboundBody.usage = {+ prompt_tokens: promptTokens,+ completion_tokens: completionTokens,+ total_tokens: totalTokens,+ estimated: true,+ };+ }+// Persist the completed response for Idempotency-Key replays. Only// non-streaming requests with a valid key reach here; a truncated turn// (finish_reason 'length') is NOT stored — replaying a cut-off answerdiff --git a/shared/types.ts b/shared/types.tsindex 06dfacc3..bfcd7309 100644--- a/shared/types.ts+++ b/shared/types.ts@@ -436,6 +436,10 @@ export interface TokenUsage {// to its chars/4 estimate when absent. (#764)completion_tokens_details?: { reasoning_tokens?: number };prompt_tokens_details?: { cached_tokens?: number };+ // Gateway-synthesized block (upstream never sent usage): flagged so a+ // cost-accounting client can tell it apart from the upstream's real+ // counts. (#1084)+ estimated?: boolean;}export interface ChatCompletionResponse {