Skip to content

7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.

TypeScript

fix(proxy): fail over on empty completions instead of returning success (#191)

Tashfeen6eadf9eca41d
diff --git a/server/src/__tests__/routes/proxy-empty-completion.test.ts b/server/src/__tests__/routes/proxy-empty-completion.test.ts
new file mode 100644
index 00000000..8a0ac928
--- /dev/null
+++ b/server/src/__tests__/routes/proxy-empty-completion.test.ts
@@ -0,0 +1,176 @@
+import { describe, it, expect, beforeAll, beforeEach, vi } from 'vitest';
+import type { Express } from 'express';
+
+// Script the provider per-test: every platform resolves to this fake, so the
+// first chain model gets emptyResult/emptyStream and the failover target gets
+// a real completion. Mock BEFORE importing anything that pulls in the router.
+const chatCompletion = vi.fn();
+const streamChatCompletion = vi.fn();
+const fakeProvider = { name: 'fake', chatCompletion, streamChatCompletion } as any;
+
+vi.mock('../../providers/index.js', async (importOriginal) => {
+ const actual = await importOriginal() as any;
+ return {
+ ...actual,
+ getProvider: () => fakeProvider,
+ resolveProvider: () => fakeProvider,
+ };
+});
+
+const { createApp } = await import('../../app.js');
+const { initDb, getDb, getUnifiedApiKey } = await import('../../db/index.js');
+const { encrypt } = await import('../../lib/crypto.js');
+const { setRoutingStrategy } = await import('../../services/router.js');
+
+async function post(app: Express, path: string, body: any, key: string) {
+ const server = app.listen(0);
+ const addr = server.address() as any;
+ const res = await fetch(`http://127.0.0.1:${addr.port}${path}`, {
+ method: 'POST',
+ headers: { 'Content-Type': 'application/json', Authorization: `Bearer ${key}` },
+ body: JSON.stringify(body),
+ });
+ const raw = await res.text();
+ server.close();
+ let json: any = null;
+ try { json = JSON.parse(raw); } catch {}
+ return { status: res.status, body: json, raw, headers: res.headers };
+}
+
+const EMPTY_RESULT = {
+ choices: [{ message: { role: 'assistant', content: '' } }],
+ usage: { prompt_tokens: 10, completion_tokens: 0, total_tokens: 10 },
+};
+const GOOD_RESULT = {
+ choices: [{ message: { role: 'assistant', content: 'a real answer' } }],
+ usage: { prompt_tokens: 10, completion_tokens: 3, total_tokens: 13 },
+};
+
+async function* emptyStream() { /* zero chunks */ }
+async function* goodStream() {
+ yield { choices: [{ delta: { content: 'streamed ' } }] };
+ yield { choices: [{ delta: { content: 'answer' } }] };
+}
+
+describe('Empty-completion failover', () => {
+ let app: Express;
+ let key: string;
+
+ beforeAll(() => {
+ process.env.ENCRYPTION_KEY = '0'.repeat(64);
+ initDb(':memory:');
+ app = createApp();
+ key = getUnifiedApiKey();
+
+ const db = getDb();
+ setRoutingStrategy('priority');
+ const { encrypted, iv, authTag } = encrypt('test-key');
+ db.prepare(`
+ INSERT INTO api_keys (platform, label, encrypted_key, iv, auth_tag, status, enabled)
+ VALUES ('groq', 'test', ?, ?, ?, 'healthy', 1)
+ `).run(encrypted, iv, authTag);
+ });
+
+ beforeEach(() => {
+ chatCompletion.mockReset();
+ streamChatCompletion.mockReset();
+ // Clear cooldowns set by previous failovers so each test routes fresh.
+ getDb().prepare('DELETE FROM rate_limit_cooldowns').run();
+ });
+
+ it('/v1/chat/completions (non-stream): empty completion fails over to the next model', async () => {
+ chatCompletion
+ .mockResolvedValueOnce(EMPTY_RESULT)
+ .mockResolvedValueOnce(GOOD_RESULT);
+
+ const { status, body, headers } = await post(app, '/v1/chat/completions', {
+ messages: [{ role: 'user', content: 'hi' }],
+ }, key);
+
+ expect(status).toBe(200);
+ expect(body.choices[0].message.content).toBe('a real answer');
+ expect(headers.get('x-fallback-attempts')).toBe('1');
+ expect(chatCompletion).toHaveBeenCalledTimes(2);
+ // First model must differ from the one that answered.
+ expect(chatCompletion.mock.calls[0][2]).not.toBe(chatCompletion.mock.calls[1][2]);
+ });
+
+ it('/v1/chat/completions (stream): zero-chunk stream fails over instead of emitting an empty stream', async () => {
+ streamChatCompletion
+ .mockReturnValueOnce(emptyStream())
+ .mockReturnValueOnce(goodStream());
+
+ const { status, raw } = await post(app, '/v1/chat/completions', {
+ messages: [{ role: 'user', content: 'hi' }],
+ stream: true,
+ }, key);
+
+ expect(status).toBe(200);
+ expect(raw).toContain('streamed ');
+ expect(raw).toContain('[DONE]');
+ expect(streamChatCompletion).toHaveBeenCalledTimes(2);
+ });
+
+ it('/v1/responses (non-stream): empty completion fails over', async () => {
+ chatCompletion
+ .mockResolvedValueOnce(EMPTY_RESULT)
+ .mockResolvedValueOnce(GOOD_RESULT);
+
+ const { status, body } = await post(app, '/v1/responses', {
+ input: 'hi',
+ }, key);
+
+ expect(status).toBe(200);
+ expect(body.output_text).toBe('a real answer');
+ expect(chatCompletion).toHaveBeenCalledTimes(2);
+ });
+
+ it('/v1/responses (stream): empty stream fails over on the same SSE connection', async () => {
+ streamChatCompletion
+ .mockReturnValueOnce(emptyStream())
+ .mockReturnValueOnce(goodStream());
+
+ const { status, raw } = await post(app, '/v1/responses', {
+ input: 'hi',
+ stream: true,
+ }, key);
+
+ expect(status).toBe(200);
+ expect(raw).toContain('response.completed');
+ expect(raw).toContain('streamed answer');
+ expect(raw).not.toContain('response.failed');
+ expect(streamChatCompletion).toHaveBeenCalledTimes(2);
+ });
+
+ it('logs the empty attempt as an error and the failover as success', async () => {
+ chatCompletion
+ .mockResolvedValueOnce(EMPTY_RESULT)
+ .mockResolvedValueOnce(GOOD_RESULT);
+
+ const db = getDb();
+ db.prepare('DELETE FROM requests').run();
+ await post(app, '/v1/chat/completions', { messages: [{ role: 'user', content: 'hi' }] }, key);
+
+ const rows = db.prepare('SELECT status, error FROM requests ORDER BY id').all() as Array<{ status: string; error: string | null }>;
+ expect(rows.length).toBe(2);
+ expect(rows[0].status).toBe('error');
+ expect(rows[0].error).toContain('empty completion');
+ expect(rows[1].status).toBe('success');
+ });
+
+ it('a tool-calls-only completion (no text) is NOT treated as empty', async () => {
+ chatCompletion.mockResolvedValueOnce({
+ choices: [{ message: { role: 'assistant', content: null, tool_calls: [{ id: 'c1', type: 'function', function: { name: 'f', arguments: '{}' } }] } }],
+ usage: { prompt_tokens: 10, completion_tokens: 5, total_tokens: 15 },
+ });
+
+ const { status, body } = await post(app, '/v1/chat/completions', {
+ messages: [{ role: 'user', content: 'hi' }],
+ tools: [{ type: 'function', function: { name: 'f', parameters: { type: 'object', properties: {} } } }],
+ }, key);
+
+ expect(status).toBe(200);
+ expect(body.choices[0].message.tool_calls).toHaveLength(1);
+ expect(chatCompletion).toHaveBeenCalledTimes(1);
+ });
+});
diff --git a/server/src/routes/proxy.ts b/server/src/routes/proxy.ts
index 6b286b3b..5b89eda7 100644
--- a/server/src/routes/proxy.ts
+++ b/server/src/routes/proxy.ts
@@ -523,9 +523,17 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {
}
if (!streamStarted) {
- // Upstream returned no chunks — emit minimal successful stream.
- res.setHeader('Content-Type', 'text/event-stream');
- res.setHeader('X-Routed-Via', `${route.platform}/${route.modelId}`);
+ // Upstream returned zero chunks — an empty completion in stream
+ // clothing. No headers are out yet, so fail over to the next model
+ // instead of handing the client a valid-looking empty stream
+ // (production case: nemotron-3-super returning nothing on large
+ // contexts while the request logs as success).
+ logRequest(route.platform, route.modelId, route.keyId, 'error', estimatedInputTokens, 0, Date.now() - start, 'empty completion (stream produced no chunks)');
+ skipKeys.add(`${route.platform}:${route.modelId}:${route.keyId}`);
+ setCooldown(route.platform, route.modelId, route.keyId, getCooldownDurationForLimit(route.platform, route.modelId, route.keyId, { rpd: route.rpdLimit, tpd: route.tpdLimit }));
+ recordRateLimitHit(route.modelDbId);
+ lastError = new Error(`empty completion from ${route.displayName}`);
+ continue;
}
res.write('data: [DONE]\n\n');
res.end();
@@ -557,6 +565,20 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {
{ temperature, max_tokens, top_p, tools, tool_choice, parallel_tool_calls },
);
+ // Empty completion (no text, no tool calls) → fail over rather than
+ // return a transport-level "success" the caller can't act on. Mirrors
+ // the zero-chunk streaming case above.
+ const respMsg = result.choices?.[0]?.message;
+ const respText = contentToString(respMsg?.content ?? '');
+ if (!respText && (respMsg?.tool_calls?.length ?? 0) === 0) {
+ logRequest(route.platform, route.modelId, route.keyId, 'error', estimatedInputTokens, 0, Date.now() - start, 'empty completion (no content, no tool_calls)');
+ skipKeys.add(`${route.platform}:${route.modelId}:${route.keyId}`);
+ setCooldown(route.platform, route.modelId, route.keyId, getCooldownDurationForLimit(route.platform, route.modelId, route.keyId, { rpd: route.rpdLimit, tpd: route.tpdLimit }));
+ recordRateLimitHit(route.modelDbId);
+ lastError = new Error(`empty completion from ${route.displayName}`);
+ continue;
+ }
+
const totalTokens = result.usage?.total_tokens ?? 0;
recordTokens(route.platform, route.modelId, route.keyId, totalTokens);
recordSuccess(route.modelDbId);
diff --git a/server/src/routes/responses.ts b/server/src/routes/responses.ts
index 38776953..4a54c866 100644
--- a/server/src/routes/responses.ts
+++ b/server/src/routes/responses.ts
@@ -443,6 +443,23 @@ responsesRouter.post('/responses', async (req: Request, res: Response) => {
}
}
+ // Empty completion — the provider returned 200 with no text AND no
+ // tool calls. Seen in production from nemotron-3-super on ~65k-token
+ // contexts: transport-level "success", zero usable output, so the
+ // agent client records a successful run it can't act on and its issue
+ // dead-ends. Nothing substantive has been emitted yet (output_item
+ // events only fire on the first delta; only the created/in_progress
+ // skeletons are out), so it's safe to fail over to the next model on
+ // the same SSE stream.
+ if (msgText.length === 0 && toolAcc.size === 0) {
+ logRequest(route.platform, route.modelId, route.keyId, 'error', estimatedInputTokens, 0, Date.now() - start, 'empty completion (no content, no tool_calls)');
+ skipKeys.add(`${route.platform}:${route.modelId}:${route.keyId}`);
+ setCooldown(route.platform, route.modelId, route.keyId, getCooldownDurationForLimit(route.platform, route.modelId, route.keyId, { rpd: route.rpdLimit, tpd: route.tpdLimit }));
+ recordRateLimitHit(route.modelDbId);
+ lastError = new Error(`empty completion from ${route.displayName}`);
+ continue;
+ }
+
// Finalize any open text item.
if (msgItemId !== null) {
sse('response.output_text.done', { item_id: msgItemId, output_index: 0, content_index: 0, text: msgText });
@@ -478,6 +495,16 @@ responsesRouter.post('/responses', async (req: Request, res: Response) => {
const promptTokens = result.usage?.prompt_tokens ?? estimatedInputTokens;
const completionTokens = result.usage?.completion_tokens ?? Math.ceil(text.length / 4);
+ // Empty completion → fail over (see the streaming-path comment above).
+ if (!text && toolCalls.length === 0) {
+ logRequest(route.platform, route.modelId, route.keyId, 'error', estimatedInputTokens, 0, Date.now() - start, 'empty completion (no content, no tool_calls)');
+ skipKeys.add(`${route.platform}:${route.modelId}:${route.keyId}`);
+ setCooldown(route.platform, route.modelId, route.keyId, getCooldownDurationForLimit(route.platform, route.modelId, route.keyId, { rpd: route.rpdLimit, tpd: route.tpdLimit }));
+ recordRateLimitHit(route.modelDbId);
+ lastError = new Error(`empty completion from ${route.displayName}`);
+ continue;
+ }
+
recordTokens(route.platform, route.modelId, route.keyId, result.usage?.total_tokens ?? 0);
recordSuccess(route.modelDbId);
setStickyModel(messages, route.modelDbId);
@@ -517,7 +544,17 @@ responsesRouter.post('/responses', async (req: Request, res: Response) => {
}
}
+ // Exhausted all retries. The streaming skeleton may already be on the wire
+ // (reachable since empty-completion failover can burn every attempt after
+ // streamStarted) — close the SSE stream with a failed event instead of
+ // writing JSON onto a committed event-stream response.
+ const exhaustedMsg = `All models rate-limited after ${MAX_RETRIES} attempts. Last: ${lastError?.message}`;
+ if (streamStarted) {
+ sse('response.failed', { response: { id: responseId, object: 'response', status: 'failed', error: { message: exhaustedMsg, type: 'rate_limit_error' } } });
+ res.end();
+ return;
+ }
res.status(429).json({
- error: { message: `All models rate-limited after ${MAX_RETRIES} attempts. Last: ${lastError?.message}`, type: 'rate_limit_error' },
+ error: { message: exhaustedMsg, type: 'rate_limit_error' },
});
});