Skip to content

7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.

TypeScript

Fail over on reasoning-only responses (#1184) (#1191)

nicyoongfe0f1c4ec3fe
diff --git a/server/src/__tests__/routes/reasoning-only-failover.test.ts b/server/src/__tests__/routes/reasoning-only-failover.test.ts
new file mode 100644
index 00000000..3a8c98aa
--- /dev/null
+++ b/server/src/__tests__/routes/reasoning-only-failover.test.ts
@@ -0,0 +1,167 @@
+import { describe, it, expect, beforeAll, beforeEach, vi } from 'vitest';
+import type { Express } from 'express';
+
+// Reasoning-only responses used to pass as success on the shared inbound-chat
+// lane (#1184): the empty-completion check required the turn to have no text,
+// no reasoning AND no tool calls, so a stream that spent the whole budget on a
+// thinking trace — then dropped or truncated the answer — was delivered as a
+// blank success and the failover ladder never moved. Coding agents (the main
+// consumers) treat a reasoning-only turn as an empty assistant message and
+// stall.
+//
+// The fix mirrors the Anthropic surface, which already fails over on
+// reasoning-only turns: the empty check drops `!reasoning`, and streaming
+// holds thinking deltas until something commit-worthy arrives, so a
+// thinking-only stream stays uncommitted through to stream end and the next
+// attempt happens invisibly.
+
+const chatCompletion = vi.fn();
+const streamChatCompletion = vi.fn();
+const fakeProvider = { name: 'fake', chatCompletion, streamChatCompletion } as any;
+
+vi.mock('../../providers/index.js', async (importOriginal) => {
+ const actual = await importOriginal() as any;
+ return {
+ ...actual,
+ getProvider: () => fakeProvider,
+ resolveProvider: () => fakeProvider,
+ };
+});
+
+const { createApp } = await import('../../app.js');
+const { initDb, getDb, setSetting } = await import('../../db/index.js');
+const { encrypt } = await import('../../lib/crypto.js');
+const { setRoutingStrategy } = await import('../../services/router.js');
+
+async function post(app: Express, body: Record<string, unknown>) {
+ const server = app.listen(0, '127.0.0.1');
+ if (!server.listening) await new Promise<void>(resolve => server.once('listening', () => resolve()));
+ const addr = server.address() as any;
+ const res = await fetch(`http://127.0.0.1:${addr.port}/api/chat`, {
+ method: 'POST',
+ headers: { 'Content-Type': 'application/json' },
+ body: JSON.stringify(body),
+ });
+ const raw = await res.text();
+ server.close();
+ let json: any = null;
+ try { json = JSON.parse(raw); } catch { /* NDJSON stream */ }
+ return { status: res.status, body: json, raw };
+}
+
+/** Collect content / thinking / tool_calls out of the Ollama NDJSON stream. */
+function readStream(raw: string): { content: string; thinking: string } {
+ let content = '';
+ let thinking = '';
+ for (const line of raw.split('\n')) {
+ if (!line.trim()) continue;
+ let frame: any;
+ try { frame = JSON.parse(line); } catch { continue; }
+ if (frame.message?.content) content += frame.message.content;
+ if (frame.message?.thinking) thinking += frame.message.thinking;
+ }
+ return { content, thinking };
+}
+
+/** Attempt 1: a pure thinking trace, then the stream just ends. */
+function* reasoningOnlyTurn() {
+ yield { choices: [{ delta: { reasoning_content: 'pondering the void ' } }] };
+ yield { choices: [{ delta: { reasoning_content: 'still pondering' } }] };
+ yield { choices: [{ delta: {}, finish_reason: 'stop' }] };
+}
+
+function* textTurn(text: string) {
+ yield { choices: [{ delta: { content: text } }] };
+ yield { choices: [{ delta: {}, finish_reason: 'stop' }] };
+}
+
+function* reasoningThenTextTurn() {
+ yield { choices: [{ delta: { reasoning_content: 'working it out' } }] };
+ yield { choices: [{ delta: { content: 'the answer' } }] };
+ yield { choices: [{ delta: {}, finish_reason: 'stop' }] };
+}
+
+function insertKey(label: string): void {
+ const db = getDb();
+ const { encrypted, iv, authTag } = encrypt('test-key');
+ db.prepare(`
+ INSERT INTO api_keys (platform, label, encrypted_key, iv, auth_tag, status, enabled)
+ VALUES ('groq', ?, ?, ?, ?, 'healthy', 1)
+ `).run(label, encrypted, iv, authTag);
+}
+
+describe('reasoning-only turns fail over on the shared inbound-chat lane (#1184)', () => {
+ let app: Express;
+
+ beforeAll(() => {
+ process.env.ENCRYPTION_KEY = '0'.repeat(64);
+ initDb(':memory:');
+ setSetting('ollama_emulation', 'open-loopback');
+ app = createApp();
+ setRoutingStrategy('priority');
+ insertKey('first');
+ insertKey('second');
+ });
+
+ beforeEach(() => {
+ chatCompletion.mockReset();
+ streamChatCompletion.mockReset();
+ getDb().prepare('DELETE FROM rate_limit_cooldowns').run();
+ });
+
+ it('non-streaming: a reasoning-only answer fails over to the next key', async () => {
+ chatCompletion
+ .mockResolvedValueOnce({ choices: [{ message: { reasoning_content: 'all thinking, no answer' }, finish_reason: 'stop' }] })
+ .mockResolvedValueOnce({ choices: [{ message: { content: 'the real answer' }, finish_reason: 'stop' }] });
+
+ const { status, body } = await post(app, {
+ model: 'auto',
+ messages: [{ role: 'user', content: 'hi' }],
+ stream: false,
+ });
+
+ expect(status).toBe(200);
+ expect(chatCompletion).toHaveBeenCalledTimes(2);
+ expect(body.message?.content).toBe('the real answer');
+ // The failed attempt's thinking trace must not leak into the served turn.
+ expect(body.message?.thinking ?? '').not.toContain('all thinking');
+ });
+
+ it('streaming: a thinking-only stream fails over and the client never sees attempt one', async () => {
+ streamChatCompletion
+ .mockImplementationOnce(() => reasoningOnlyTurn())
+ .mockImplementationOnce(() => textTurn('the real answer'));
+
+ const { status, raw } = await post(app, {
+ model: 'auto',
+ messages: [{ role: 'user', content: 'hi' }],
+ stream: true,
+ });
+
+ expect(status).toBe(200);
+ expect(streamChatCompletion).toHaveBeenCalledTimes(2);
+ const { content, thinking } = readStream(raw);
+ expect(content).toBe('the real answer');
+ expect(thinking).not.toContain('pondering');
+ });
+
+ it('streaming: thinking that precedes real content is delivered, in order', async () => {
+ streamChatCompletion.mockImplementation(() => reasoningThenTextTurn());
+
+ const { status, raw } = await post(app, {
+ model: 'auto',
+ messages: [{ role: 'user', content: 'hi' }],
+ stream: true,
+ });
+
+ expect(status).toBe(200);
+ expect(streamChatCompletion).toHaveBeenCalledTimes(1);
+ const { content, thinking } = readStream(raw);
+ expect(content).toBe('the answer');
+ expect(thinking).toContain('working it out');
+ const thinkingIdx = raw.indexOf('working it out');
+ const contentIdx = raw.indexOf('the answer');
+ expect(thinkingIdx).toBeGreaterThan(-1);
+ expect(thinkingIdx).toBeLessThan(contentIdx);
+ });
+});
diff --git a/server/src/lib/inbound-chat.ts b/server/src/lib/inbound-chat.ts
index 54487a8b..df32166c 100644
--- a/server/src/lib/inbound-chat.ts
+++ b/server/src/lib/inbound-chat.ts
@@ -259,7 +259,10 @@ export async function runInboundChat(
let text = contentToString(message?.content ?? '');
const reasoning = message?.reasoning_content ?? '';
let toolCalls = message?.tool_calls ?? [];
- if (!text && !reasoning && toolCalls.length === 0) {
+ // Reasoning-only is an empty turn to the caller — coding agents stall
+ // on a blank reply — so it fails over like an empty completion,
+ // mirroring the Anthropic surface's non-streaming check (#1184).
+ if (!text && toolCalls.length === 0) {
throw Object.assign(
new Error(`empty completion from ${route.displayName}`),
result.choices?.[0]?.finish_reason === 'length' ? { skipBench: true } : {},
@@ -354,12 +357,22 @@ export async function runInboundChat(
const toolAcc = new Map<number, { id?: string; name: string; args: string; thoughtSignature?: string }>();
let dialectMode: 'undecided' | 'passthrough' | 'dialect' = 'undecided';
let heldText = '';
+ // Thinking deltas held until something commit-worthy arrives (#1184) —
+ // the same buffer the Anthropic surface keeps: a stream that produces
+ // ONLY reasoning stays uncommitted through to stream end and fails over
+ // invisibly, instead of locking the ladder on a blank reply. Flushed in
+ // order (before the text/tool output) the moment the stream commits.
+ let heldReasoning = '';
const commit = () => {
if (committed) return;
res.setHeader('X-Routed-Via', routedViaValue(route.platform, route.modelId));
setFallbackHeaders(res, attempt, attemptLog);
wire.startStream(res, route);
committed = true;
+ if (heldReasoning) {
+ wire.sendReasoningDelta?.(res, route, heldReasoning);
+ heldReasoning = '';
+ }
};
try {
@@ -400,10 +413,15 @@ export async function runInboundChat(
}
}
if (typeof deltaReasoning === 'string' && deltaReasoning) {
- commit();
reasoning += deltaReasoning;
outputTokens += Math.ceil(deltaReasoning.length / 4);
- wire.sendReasoningDelta?.(res, route, deltaReasoning);
+ if (committed) {
+ wire.sendReasoningDelta?.(res, route, deltaReasoning);
+ } else {
+ // Hold rather than commit: thinking alone is not proof the turn
+ // will produce a usable answer (#1184).
+ heldReasoning += deltaReasoning;
+ }
}
for (const call of choice.delta?.tool_calls ?? []) {
const index = call.index ?? 0;
@@ -474,7 +492,10 @@ export async function runInboundChat(
0,
) / 4);
}
- if (!text && !reasoning && toolCalls.length === 0) {
+ // Reasoning-only is an empty turn to the caller (#1184) — same rule as
+ // the non-streaming path above; nothing was committed, so the failover
+ // hop is invisible to the client.
+ if (!text && toolCalls.length === 0) {
if (clientGone) return 'committed';
throw Object.assign(
new Error(`empty completion from ${route.displayName}`),