7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.
TypeScript
fix(proxy): fail over on empty completions instead of returning success (#191)
Tashfeen6eadf9eca41d
diff --git a/server/src/__tests__/routes/proxy-empty-completion.test.ts b/server/src/__tests__/routes/proxy-empty-completion.test.tsnew file mode 100644index 00000000..8a0ac928--- /dev/null+++ b/server/src/__tests__/routes/proxy-empty-completion.test.ts@@ -0,0 +1,176 @@+import { describe, it, expect, beforeAll, beforeEach, vi } from 'vitest';+import type { Express } from 'express';++// Script the provider per-test: every platform resolves to this fake, so the+// first chain model gets emptyResult/emptyStream and the failover target gets+// a real completion. Mock BEFORE importing anything that pulls in the router.+const chatCompletion = vi.fn();+const streamChatCompletion = vi.fn();+const fakeProvider = { name: 'fake', chatCompletion, streamChatCompletion } as any;++vi.mock('../../providers/index.js', async (importOriginal) => {+ const actual = await importOriginal() as any;+ return {+ ...actual,+ getProvider: () => fakeProvider,+ resolveProvider: () => fakeProvider,+ };+});++const { createApp } = await import('../../app.js');+const { initDb, getDb, getUnifiedApiKey } = await import('../../db/index.js');+const { encrypt } = await import('../../lib/crypto.js');+const { setRoutingStrategy } = await import('../../services/router.js');++async function post(app: Express, path: string, body: any, key: string) {+ const server = app.listen(0);+ const addr = server.address() as any;+ const res = await fetch(`http://127.0.0.1:${addr.port}${path}`, {+ method: 'POST',+ headers: { 'Content-Type': 'application/json', Authorization: `Bearer ${key}` },+ body: JSON.stringify(body),+ });+ const raw = await res.text();+ server.close();+ let json: any = null;+ try { json = JSON.parse(raw); } catch {}+ return { status: res.status, body: json, raw, headers: res.headers };+}++const EMPTY_RESULT = {+ choices: [{ message: { role: 'assistant', content: '' } }],+ usage: { prompt_tokens: 10, completion_tokens: 0, total_tokens: 10 },+};+const GOOD_RESULT = {+ choices: [{ message: { role: 'assistant', content: 'a real answer' } }],+ usage: { prompt_tokens: 10, completion_tokens: 3, total_tokens: 13 },+};++async function* emptyStream() { /* zero chunks */ }+async function* goodStream() {+ yield { choices: [{ delta: { content: 'streamed ' } }] };+ yield { choices: [{ delta: { content: 'answer' } }] };+}++describe('Empty-completion failover', () => {+ let app: Express;+ let key: string;++ beforeAll(() => {+ process.env.ENCRYPTION_KEY = '0'.repeat(64);+ initDb(':memory:');+ app = createApp();+ key = getUnifiedApiKey();++ const db = getDb();+ setRoutingStrategy('priority');+ const { encrypted, iv, authTag } = encrypt('test-key');+ db.prepare(`+ INSERT INTO api_keys (platform, label, encrypted_key, iv, auth_tag, status, enabled)+ VALUES ('groq', 'test', ?, ?, ?, 'healthy', 1)+ `).run(encrypted, iv, authTag);+ });++ beforeEach(() => {+ chatCompletion.mockReset();+ streamChatCompletion.mockReset();+ // Clear cooldowns set by previous failovers so each test routes fresh.+ getDb().prepare('DELETE FROM rate_limit_cooldowns').run();+ });++ it('/v1/chat/completions (non-stream): empty completion fails over to the next model', async () => {+ chatCompletion+ .mockResolvedValueOnce(EMPTY_RESULT)+ .mockResolvedValueOnce(GOOD_RESULT);++ const { status, body, headers } = await post(app, '/v1/chat/completions', {+ messages: [{ role: 'user', content: 'hi' }],+ }, key);++ expect(status).toBe(200);+ expect(body.choices[0].message.content).toBe('a real answer');+ expect(headers.get('x-fallback-attempts')).toBe('1');+ expect(chatCompletion).toHaveBeenCalledTimes(2);+ // First model must differ from the one that answered.+ expect(chatCompletion.mock.calls[0][2]).not.toBe(chatCompletion.mock.calls[1][2]);+ });++ it('/v1/chat/completions (stream): zero-chunk stream fails over instead of emitting an empty stream', async () => {+ streamChatCompletion+ .mockReturnValueOnce(emptyStream())+ .mockReturnValueOnce(goodStream());++ const { status, raw } = await post(app, '/v1/chat/completions', {+ messages: [{ role: 'user', content: 'hi' }],+ stream: true,+ }, key);++ expect(status).toBe(200);+ expect(raw).toContain('streamed ');+ expect(raw).toContain('[DONE]');+ expect(streamChatCompletion).toHaveBeenCalledTimes(2);+ });++ it('/v1/responses (non-stream): empty completion fails over', async () => {+ chatCompletion+ .mockResolvedValueOnce(EMPTY_RESULT)+ .mockResolvedValueOnce(GOOD_RESULT);++ const { status, body } = await post(app, '/v1/responses', {+ input: 'hi',+ }, key);++ expect(status).toBe(200);+ expect(body.output_text).toBe('a real answer');+ expect(chatCompletion).toHaveBeenCalledTimes(2);+ });++ it('/v1/responses (stream): empty stream fails over on the same SSE connection', async () => {+ streamChatCompletion+ .mockReturnValueOnce(emptyStream())+ .mockReturnValueOnce(goodStream());++ const { status, raw } = await post(app, '/v1/responses', {+ input: 'hi',+ stream: true,+ }, key);++ expect(status).toBe(200);+ expect(raw).toContain('response.completed');+ expect(raw).toContain('streamed answer');+ expect(raw).not.toContain('response.failed');+ expect(streamChatCompletion).toHaveBeenCalledTimes(2);+ });++ it('logs the empty attempt as an error and the failover as success', async () => {+ chatCompletion+ .mockResolvedValueOnce(EMPTY_RESULT)+ .mockResolvedValueOnce(GOOD_RESULT);++ const db = getDb();+ db.prepare('DELETE FROM requests').run();+ await post(app, '/v1/chat/completions', { messages: [{ role: 'user', content: 'hi' }] }, key);++ const rows = db.prepare('SELECT status, error FROM requests ORDER BY id').all() as Array<{ status: string; error: string | null }>;+ expect(rows.length).toBe(2);+ expect(rows[0].status).toBe('error');+ expect(rows[0].error).toContain('empty completion');+ expect(rows[1].status).toBe('success');+ });++ it('a tool-calls-only completion (no text) is NOT treated as empty', async () => {+ chatCompletion.mockResolvedValueOnce({+ choices: [{ message: { role: 'assistant', content: null, tool_calls: [{ id: 'c1', type: 'function', function: { name: 'f', arguments: '{}' } }] } }],+ usage: { prompt_tokens: 10, completion_tokens: 5, total_tokens: 15 },+ });++ const { status, body } = await post(app, '/v1/chat/completions', {+ messages: [{ role: 'user', content: 'hi' }],+ tools: [{ type: 'function', function: { name: 'f', parameters: { type: 'object', properties: {} } } }],+ }, key);++ expect(status).toBe(200);+ expect(body.choices[0].message.tool_calls).toHaveLength(1);+ expect(chatCompletion).toHaveBeenCalledTimes(1);+ });+});diff --git a/server/src/routes/proxy.ts b/server/src/routes/proxy.tsindex 6b286b3b..5b89eda7 100644--- a/server/src/routes/proxy.ts+++ b/server/src/routes/proxy.ts@@ -523,9 +523,17 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {}if (!streamStarted) {- // Upstream returned no chunks — emit minimal successful stream.- res.setHeader('Content-Type', 'text/event-stream');- res.setHeader('X-Routed-Via', `${route.platform}/${route.modelId}`);+ // Upstream returned zero chunks — an empty completion in stream+ // clothing. No headers are out yet, so fail over to the next model+ // instead of handing the client a valid-looking empty stream+ // (production case: nemotron-3-super returning nothing on large+ // contexts while the request logs as success).+ logRequest(route.platform, route.modelId, route.keyId, 'error', estimatedInputTokens, 0, Date.now() - start, 'empty completion (stream produced no chunks)');+ skipKeys.add(`${route.platform}:${route.modelId}:${route.keyId}`);+ setCooldown(route.platform, route.modelId, route.keyId, getCooldownDurationForLimit(route.platform, route.modelId, route.keyId, { rpd: route.rpdLimit, tpd: route.tpdLimit }));+ recordRateLimitHit(route.modelDbId);+ lastError = new Error(`empty completion from ${route.displayName}`);+ continue;}res.write('data: [DONE]\n\n');res.end();@@ -557,6 +565,20 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {{ temperature, max_tokens, top_p, tools, tool_choice, parallel_tool_calls },);+ // Empty completion (no text, no tool calls) → fail over rather than+ // return a transport-level "success" the caller can't act on. Mirrors+ // the zero-chunk streaming case above.+ const respMsg = result.choices?.[0]?.message;+ const respText = contentToString(respMsg?.content ?? '');+ if (!respText && (respMsg?.tool_calls?.length ?? 0) === 0) {+ logRequest(route.platform, route.modelId, route.keyId, 'error', estimatedInputTokens, 0, Date.now() - start, 'empty completion (no content, no tool_calls)');+ skipKeys.add(`${route.platform}:${route.modelId}:${route.keyId}`);+ setCooldown(route.platform, route.modelId, route.keyId, getCooldownDurationForLimit(route.platform, route.modelId, route.keyId, { rpd: route.rpdLimit, tpd: route.tpdLimit }));+ recordRateLimitHit(route.modelDbId);+ lastError = new Error(`empty completion from ${route.displayName}`);+ continue;+ }+const totalTokens = result.usage?.total_tokens ?? 0;recordTokens(route.platform, route.modelId, route.keyId, totalTokens);recordSuccess(route.modelDbId);diff --git a/server/src/routes/responses.ts b/server/src/routes/responses.tsindex 38776953..4a54c866 100644--- a/server/src/routes/responses.ts+++ b/server/src/routes/responses.ts@@ -443,6 +443,23 @@ responsesRouter.post('/responses', async (req: Request, res: Response) => {}}+ // Empty completion — the provider returned 200 with no text AND no+ // tool calls. Seen in production from nemotron-3-super on ~65k-token+ // contexts: transport-level "success", zero usable output, so the+ // agent client records a successful run it can't act on and its issue+ // dead-ends. Nothing substantive has been emitted yet (output_item+ // events only fire on the first delta; only the created/in_progress+ // skeletons are out), so it's safe to fail over to the next model on+ // the same SSE stream.+ if (msgText.length === 0 && toolAcc.size === 0) {+ logRequest(route.platform, route.modelId, route.keyId, 'error', estimatedInputTokens, 0, Date.now() - start, 'empty completion (no content, no tool_calls)');+ skipKeys.add(`${route.platform}:${route.modelId}:${route.keyId}`);+ setCooldown(route.platform, route.modelId, route.keyId, getCooldownDurationForLimit(route.platform, route.modelId, route.keyId, { rpd: route.rpdLimit, tpd: route.tpdLimit }));+ recordRateLimitHit(route.modelDbId);+ lastError = new Error(`empty completion from ${route.displayName}`);+ continue;+ }+// Finalize any open text item.if (msgItemId !== null) {sse('response.output_text.done', { item_id: msgItemId, output_index: 0, content_index: 0, text: msgText });@@ -478,6 +495,16 @@ responsesRouter.post('/responses', async (req: Request, res: Response) => {const promptTokens = result.usage?.prompt_tokens ?? estimatedInputTokens;const completionTokens = result.usage?.completion_tokens ?? Math.ceil(text.length / 4);+ // Empty completion → fail over (see the streaming-path comment above).+ if (!text && toolCalls.length === 0) {+ logRequest(route.platform, route.modelId, route.keyId, 'error', estimatedInputTokens, 0, Date.now() - start, 'empty completion (no content, no tool_calls)');+ skipKeys.add(`${route.platform}:${route.modelId}:${route.keyId}`);+ setCooldown(route.platform, route.modelId, route.keyId, getCooldownDurationForLimit(route.platform, route.modelId, route.keyId, { rpd: route.rpdLimit, tpd: route.tpdLimit }));+ recordRateLimitHit(route.modelDbId);+ lastError = new Error(`empty completion from ${route.displayName}`);+ continue;+ }+recordTokens(route.platform, route.modelId, route.keyId, result.usage?.total_tokens ?? 0);recordSuccess(route.modelDbId);setStickyModel(messages, route.modelDbId);@@ -517,7 +544,17 @@ responsesRouter.post('/responses', async (req: Request, res: Response) => {}}+ // Exhausted all retries. The streaming skeleton may already be on the wire+ // (reachable since empty-completion failover can burn every attempt after+ // streamStarted) — close the SSE stream with a failed event instead of+ // writing JSON onto a committed event-stream response.+ const exhaustedMsg = `All models rate-limited after ${MAX_RETRIES} attempts. Last: ${lastError?.message}`;+ if (streamStarted) {+ sse('response.failed', { response: { id: responseId, object: 'response', status: 'failed', error: { message: exhaustedMsg, type: 'rate_limit_error' } } });+ res.end();+ return;+ }res.status(429).json({- error: { message: `All models rate-limited after ${MAX_RETRIES} attempts. Last: ${lastError?.message}`, type: 'rate_limit_error' },+ error: { message: exhaustedMsg, type: 'rate_limit_error' },});});