Skip to content

7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.

TypeScript

fix(proxy): always surface usage so agents show context-window counts (#1084) (#1120)

suantea (asuan_suan)8c8a69072cf2
diff --git a/server/src/__tests__/routes/proxy-stream-usage.test.ts b/server/src/__tests__/routes/proxy-stream-usage.test.ts
index 5f2aed41..830c5f85 100644
--- a/server/src/__tests__/routes/proxy-stream-usage.test.ts
+++ b/server/src/__tests__/routes/proxy-stream-usage.test.ts
@@ -195,7 +195,7 @@ describe('Streaming usage-frame fallback injection', () => {
expect(frames[0].usage.estimated).toBe(true);
});
- it('does NOT inject a usage frame when include_usage is not requested', async () => {
+ it('injects an estimated usage frame when the upstream never echoes one, even without include_usage', async () => {
const origFetch = global.fetch;
vi.spyOn(global, 'fetch').mockImplementation(async (url, init) => {
const urlStr = typeof url === 'string' ? url : url.toString();
@@ -211,7 +211,53 @@ describe('Streaming usage-frame fallback injection', () => {
}, authHeaders());
expect(status).toBe(200);
- expect(usageFrames(raw)).toHaveLength(0);
+ // #1084: agents read `usage` for context-window display even when they
+ // never requested include_usage; a missing frame used to show 0. The
+ // estimate is injected regardless of the client's stream_options.
+ const frames = usageFrames(raw);
+ expect(frames).toHaveLength(1);
+ expect(frames[0].choices).toEqual([]);
+ expect(frames[0].usage.prompt_tokens).toBeGreaterThan(0);
+ expect(frames[0].usage.completion_tokens).toBeGreaterThan(0);
+ expect(frames[0].usage.estimated).toBe(true);
+ });
+
+ it('injects estimated usage into a NON-streaming response when the upstream omits usage', async () => {
+ const origFetch = global.fetch;
+ vi.spyOn(global, 'fetch').mockImplementation(async (url, init) => {
+ const urlStr = typeof url === 'string' ? url : url.toString();
+ if (urlStr.includes('api.groq.com/openai/v1/chat/completions')) {
+ return {
+ ok: true,
+ json: () => Promise.resolve({
+ id: 'chatcmpl-nousage',
+ object: 'chat.completion',
+ created: 123,
+ model: 'openai/gpt-oss-120b',
+ choices: [{
+ index: 0,
+ message: { role: 'assistant', content: 'hi there' },
+ finish_reason: 'stop',
+ }],
+ // No `usage` block — the #1084 scenario.
+ }),
+ } as any;
+ }
+ return origFetch(url, init);
+ });
+
+ const { status, body } = await request(app, 'POST', '/v1/chat/completions', {
+ messages: [{ role: 'user', content: 'hi' }],
+ }, authHeaders());
+
+ expect(status).toBe(200);
+ expect(body.usage).toBeDefined();
+ expect(body.usage.prompt_tokens).toBeGreaterThan(0);
+ expect(body.usage.completion_tokens).toBeGreaterThan(0);
+ expect(body.usage.total_tokens).toBe(
+ body.usage.prompt_tokens + body.usage.completion_tokens,
+ );
+ expect(body.usage.estimated).toBe(true);
});
it('passes through the upstream usage frame unchanged when one is present', async () => {
diff --git a/server/src/routes/proxy.ts b/server/src/routes/proxy.ts
index 38e2dac7..7fb77b26 100644
--- a/server/src/routes/proxy.ts
+++ b/server/src/routes/proxy.ts
@@ -2392,19 +2392,20 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {
const estimatedPromptTokens = estimatedInputTokens + injectedHandoffTokens + imageCount * IMAGE_TOKEN_ESTIMATE;
if (usageChunk) {
writeChunk(usageChunk);
- } else if (parsed.data.stream_options?.include_usage) {
- // Some OpenAI-compatible upstreams (e.g. OpenCode Zen) never echo
- // a final usage frame even when stream_options.include_usage is
- // requested. Strict clients (Hermes, Cline, Continue) treat a
+ } else {
+ // Some OpenAI-compatible upstreams never echo a final usage
+ // frame — neither when stream_options.include_usage is requested
+ // nor otherwise. Strict clients (Hermes, Cline, Continue) treat a
// missing usage block as "no accounting happened" and skip
- // per-call token/cost/billing_provider writes entirely.
+ // per-call token/cost/billing_provider writes entirely; agents
+ // that read usage for context-window display (e.g. #1084) show 0.
//
- // Injected ONLY when the client asked for usage via
- // stream_options.include_usage AND the upstream never sent one;
- // the numbers are this gateway's own chars/4 estimate (the same
- // total the accounting below records), never the upstream's
- // accounting, so the block is flagged `estimated: true` rather
- // than passed off as real counts.
+ // So inject the estimate whenever the upstream never sent one —
+ // regardless of whether the client asked for include_usage. The
+ // numbers are this gateway's own chars/4 estimate (the same total
+ // the accounting below records), never the upstream's accounting,
+ // so the block is flagged `estimated: true` rather than passed
+ // off as real counts.
const completionTokens = totalOutputTokens;
writeChunk({
id: lastMeta.id ?? `chatcmpl-${Date.now()}`,
@@ -2637,6 +2638,20 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {
const outboundBody = sanitizeResponse(normalizeOutboundContent(result));
res.setHeader('X-FreeLLM-Cache', cacheKey ? 'MISS' : 'OFF');
+ // #1084: agents show zero context usage when the upstream omits
+ // `usage` (many free-tier providers do). Fall back to the same
+ // chars/4 estimate used for accounting above, flagged `estimated:
+ // true` so a cost-accounting client can tell it apart from the
+ // upstream's real counts.
+ if (!outboundBody.usage) {
+ outboundBody.usage = {
+ prompt_tokens: promptTokens,
+ completion_tokens: completionTokens,
+ total_tokens: totalTokens,
+ estimated: true,
+ };
+ }
+
// Persist the completed response for Idempotency-Key replays. Only
// non-streaming requests with a valid key reach here; a truncated turn
// (finish_reason 'length') is NOT stored — replaying a cut-off answer
diff --git a/shared/types.ts b/shared/types.ts
index 06dfacc3..bfcd7309 100644
--- a/shared/types.ts
+++ b/shared/types.ts
@@ -436,6 +436,10 @@ export interface TokenUsage {
// to its chars/4 estimate when absent. (#764)
completion_tokens_details?: { reasoning_tokens?: number };
prompt_tokens_details?: { cached_tokens?: number };
+ // Gateway-synthesized block (upstream never sent usage): flagged so a
+ // cost-accounting client can tell it apart from the upstream's real
+ // counts. (#1084)
+ estimated?: boolean;
}
export interface ChatCompletionResponse {