Skip to content

7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.

TypeScript

fix(completions): respond with the estimated usage block when the provider omits usage (#1336)

tashdroid3b9c351a77b8
diff --git a/server/src/__tests__/routes/proxy-completions.test.ts b/server/src/__tests__/routes/proxy-completions.test.ts
index 98e1631b..731f514f 100644
--- a/server/src/__tests__/routes/proxy-completions.test.ts
+++ b/server/src/__tests__/routes/proxy-completions.test.ts
@@ -114,6 +114,48 @@ describe('POST /v1/completions', () => {
expect(capturedBody.messages[1].content).toContain('console.log(answer);');
});
+ it('answers with an estimated usage block when the provider omits usage', async () => {
+ // The OpenAI completions shape requires `usage`; free-tier providers that
+ // drop it used to surface as `usage: undefined` on /v1/completions even
+ // though the chars/4 fallback (#764) was already computed.
+ const origFetch = global.fetch;
+
+ vi.spyOn(global, 'fetch').mockImplementation(async (url, init) => {
+ const urlStr = typeof url === 'string' ? url : url.toString();
+ if (urlStr.includes('api.groq.com/openai/v1/chat/completions')) {
+ return {
+ ok: true,
+ headers: new Headers(),
+ json: () => Promise.resolve({
+ id: 'chatcmpl-nousage',
+ object: 'chat.completion',
+ created: 123,
+ model: JSON.parse(String(init?.body)).model,
+ choices: [{
+ index: 0,
+ message: { role: 'assistant', content: ' = 42;' },
+ finish_reason: 'stop',
+ }],
+ }),
+ } as any;
+ }
+ return origFetch(url, init);
+ });
+
+ const { status, body } = await request(app, 'POST', '/v1/completions', {
+ model: 'auto',
+ prompt: 'const answer',
+ max_tokens: 12,
+ }, authHeaders());
+
+ expect(status).toBe(200);
+ expect(body.usage).not.toBeNull();
+ expect(body.usage).toMatchObject({ estimated: true });
+ expect(body.usage.prompt_tokens).toBeGreaterThan(0);
+ expect(body.usage.completion_tokens).toBe(2); // ceil(' = 42;'.length / 4)
+ expect(body.usage.total_tokens).toBe(body.usage.prompt_tokens + body.usage.completion_tokens);
+ });
+
it('accepts autocomplete clients that send many stop sequences and forwards a provider-safe subset', async () => {
let capturedBody: any = null;
const origFetch = global.fetch;
diff --git a/server/src/routes/proxy.ts b/server/src/routes/proxy.ts
index a361343b..0383cfcf 100644
--- a/server/src/routes/proxy.ts
+++ b/server/src/routes/proxy.ts
@@ -1386,7 +1386,16 @@ proxyRouter.post('/completions', async (req: Request, res: Response) => {
logprobs: null,
finish_reason: result.choices?.[0]?.finish_reason ?? 'stop',
}],
- usage: result.usage,
+ // `usage` is required by the OpenAI completions spec. The fallback
+ // counts computed just above (chars/4 when the provider omits usage,
+ // #764) existed but were dropped here — clients like editor
+ // ghost-text plugins read usage to throttle and saw `undefined`.
+ usage: result.usage ?? {
+ prompt_tokens: promptTokens,
+ completion_tokens: completionTokens,
+ total_tokens: totalTokens,
+ estimated: true,
+ },
execution_id: requestGroupId,
});