7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.
TypeScript
fix(completions): respond with the estimated usage block when the provider omits usage (#1336)
tashdroid3b9c351a77b8
diff --git a/server/src/__tests__/routes/proxy-completions.test.ts b/server/src/__tests__/routes/proxy-completions.test.tsindex 98e1631b..731f514f 100644--- a/server/src/__tests__/routes/proxy-completions.test.ts+++ b/server/src/__tests__/routes/proxy-completions.test.ts@@ -114,6 +114,48 @@ describe('POST /v1/completions', () => {expect(capturedBody.messages[1].content).toContain('console.log(answer);');});+ it('answers with an estimated usage block when the provider omits usage', async () => {+ // The OpenAI completions shape requires `usage`; free-tier providers that+ // drop it used to surface as `usage: undefined` on /v1/completions even+ // though the chars/4 fallback (#764) was already computed.+ const origFetch = global.fetch;++ vi.spyOn(global, 'fetch').mockImplementation(async (url, init) => {+ const urlStr = typeof url === 'string' ? url : url.toString();+ if (urlStr.includes('api.groq.com/openai/v1/chat/completions')) {+ return {+ ok: true,+ headers: new Headers(),+ json: () => Promise.resolve({+ id: 'chatcmpl-nousage',+ object: 'chat.completion',+ created: 123,+ model: JSON.parse(String(init?.body)).model,+ choices: [{+ index: 0,+ message: { role: 'assistant', content: ' = 42;' },+ finish_reason: 'stop',+ }],+ }),+ } as any;+ }+ return origFetch(url, init);+ });++ const { status, body } = await request(app, 'POST', '/v1/completions', {+ model: 'auto',+ prompt: 'const answer',+ max_tokens: 12,+ }, authHeaders());++ expect(status).toBe(200);+ expect(body.usage).not.toBeNull();+ expect(body.usage).toMatchObject({ estimated: true });+ expect(body.usage.prompt_tokens).toBeGreaterThan(0);+ expect(body.usage.completion_tokens).toBe(2); // ceil(' = 42;'.length / 4)+ expect(body.usage.total_tokens).toBe(body.usage.prompt_tokens + body.usage.completion_tokens);+ });+it('accepts autocomplete clients that send many stop sequences and forwards a provider-safe subset', async () => {let capturedBody: any = null;const origFetch = global.fetch;diff --git a/server/src/routes/proxy.ts b/server/src/routes/proxy.tsindex a361343b..0383cfcf 100644--- a/server/src/routes/proxy.ts+++ b/server/src/routes/proxy.ts@@ -1386,7 +1386,16 @@ proxyRouter.post('/completions', async (req: Request, res: Response) => {logprobs: null,finish_reason: result.choices?.[0]?.finish_reason ?? 'stop',}],- usage: result.usage,+ // `usage` is required by the OpenAI completions spec. The fallback+ // counts computed just above (chars/4 when the provider omits usage,+ // #764) existed but were dropped here — clients like editor+ // ghost-text plugins read usage to throttle and saw `undefined`.+ usage: result.usage ?? {+ prompt_tokens: promptTokens,+ completion_tokens: completionTokens,+ total_tokens: totalTokens,+ estimated: true,+ },execution_id: requestGroupId,});