Skip to content

7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.

TypeScript

fix(stream): commit long-reasoning streams early so idle clients stay connected (#1642) (#1645)

Fangore48367346e21d
diff --git a/docs/en/architecture/03-streaming-pipeline.md b/docs/en/architecture/03-streaming-pipeline.md
index 140323cd..863a20cb 100644
--- a/docs/en/architecture/03-streaming-pipeline.md
+++ b/docs/en/architecture/03-streaming-pipeline.md
@@ -66,7 +66,7 @@ headerSent = false
- `undecided` + `!wantsTools` OR `!couldBecomeDialectMarker` OR `heldText > 256` → `passthrough`, flush headers, stream
- `passthrough` → forward immediately (strip tool_calls from delta, re-emit complete at end)
- `dialect` → keep buffering
-9. **Empty chunks** (role/keep-alive/reasoning-only): held in `preamble` until flush.
+9. **Empty chunks** (role/keep-alive/reasoning-only): held in `preamble` until flush. Reasoning that runs past `REASONING_HOLD_MAX_MS` (15s) flushes headers early and then streams live, so idle-watchdog clients stay connected (#1642); a failure after that is a mid-stream error.
**Stream end (clean):**
diff --git a/server/src/__tests__/routes/reasoning-only-failover.test.ts b/server/src/__tests__/routes/reasoning-only-failover.test.ts
index 3a8c98aa..7ccaf968 100644
--- a/server/src/__tests__/routes/reasoning-only-failover.test.ts
+++ b/server/src/__tests__/routes/reasoning-only-failover.test.ts
@@ -1,4 +1,4 @@
-import { describe, it, expect, beforeAll, beforeEach, vi } from 'vitest';
+import { describe, it, expect, beforeAll, beforeEach, afterEach, vi } from 'vitest';
import type { Express } from 'express';
// Reasoning-only responses used to pass as success on the shared inbound-chat
@@ -29,24 +29,25 @@ vi.mock('../../providers/index.js', async (importOriginal) => {
});
const { createApp } = await import('../../app.js');
-const { initDb, getDb, setSetting } = await import('../../db/index.js');
+const { initDb, getDb, setSetting, getUnifiedApiKey } = await import('../../db/index.js');
const { encrypt } = await import('../../lib/crypto.js');
const { setRoutingStrategy } = await import('../../services/router.js');
+const { TOOL_ARGUMENT_VALIDATION_SETTING } = await import('../../lib/tool-validate.js');
-async function post(app: Express, body: Record<string, unknown>) {
+async function post(app: Express, body: Record<string, unknown>, path = '/api/chat', headers: Record<string, string> = {}) {
const server = app.listen(0, '127.0.0.1');
if (!server.listening) await new Promise<void>(resolve => server.once('listening', () => resolve()));
const addr = server.address() as any;
- const res = await fetch(`http://127.0.0.1:${addr.port}/api/chat`, {
+ const res = await fetch(`http://127.0.0.1:${addr.port}${path}`, {
method: 'POST',
- headers: { 'Content-Type': 'application/json' },
+ headers: { 'Content-Type': 'application/json', ...headers },
body: JSON.stringify(body),
});
const raw = await res.text();
server.close();
let json: any = null;
try { json = JSON.parse(raw); } catch { /* NDJSON stream */ }
- return { status: res.status, body: json, raw };
+ return { status: res.status, body: json, raw, routedVia: res.headers.get('x-routed-via') };
}
/** Collect content / thinking / tool_calls out of the Ollama NDJSON stream. */
@@ -90,14 +91,16 @@ function insertKey(label: string): void {
`).run(label, encrypted, iv, authTag);
}
+let builtApp: Express;
+const app = () => builtApp;
+
describe('reasoning-only turns fail over on the shared inbound-chat lane (#1184)', () => {
- let app: Express;
beforeAll(() => {
process.env.ENCRYPTION_KEY = '0'.repeat(64);
initDb(':memory:');
setSetting('ollama_emulation', 'open-loopback');
- app = createApp();
+ builtApp = createApp();
setRoutingStrategy('priority');
insertKey('first');
insertKey('second');
@@ -114,7 +117,7 @@ describe('reasoning-only turns fail over on the shared inbound-chat lane (#1184)
.mockResolvedValueOnce({ choices: [{ message: { reasoning_content: 'all thinking, no answer' }, finish_reason: 'stop' }] })
.mockResolvedValueOnce({ choices: [{ message: { content: 'the real answer' }, finish_reason: 'stop' }] });
- const { status, body } = await post(app, {
+ const { status, body } = await post(app(), {
model: 'auto',
messages: [{ role: 'user', content: 'hi' }],
stream: false,
@@ -132,7 +135,7 @@ describe('reasoning-only turns fail over on the shared inbound-chat lane (#1184)
.mockImplementationOnce(() => reasoningOnlyTurn())
.mockImplementationOnce(() => textTurn('the real answer'));
- const { status, raw } = await post(app, {
+ const { status, raw } = await post(app(), {
model: 'auto',
messages: [{ role: 'user', content: 'hi' }],
stream: true,
@@ -148,7 +151,7 @@ describe('reasoning-only turns fail over on the shared inbound-chat lane (#1184)
it('streaming: thinking that precedes real content is delivered, in order', async () => {
streamChatCompletion.mockImplementation(() => reasoningThenTextTurn());
- const { status, raw } = await post(app, {
+ const { status, raw } = await post(app(), {
model: 'auto',
messages: [{ role: 'user', content: 'hi' }],
stream: true,
@@ -165,3 +168,166 @@ describe('reasoning-only turns fail over on the shared inbound-chat lane (#1184)
expect(thinkingIdx).toBeLessThan(contentIdx);
});
});
+
+// #1642: holding reasoning uncommitted for minutes made idle-watchdog clients
+// (Claude Code) hang up. Past REASONING_HOLD_MAX_MS the attempt commits with
+// its real status + routing headers and keeps the stream alive; dying after
+// that goes through the surface's normal mid-stream error path.
+describe('long reasoning commits and keeps the stream alive (#1642)', () => {
+ let skew = 0;
+ const realNow = Date.now.bind(Date);
+ const auth = () => ({ Authorization: `Bearer ${getUnifiedApiKey()}`, 'anthropic-version': '2023-06-01' });
+ const req = { model: 'auto', max_tokens: 100, stream: true, messages: [{ role: 'user', content: 'hi' }] };
+
+ /** Reasoning that outlasts the hold window, then optionally an answer. */
+ function* longReasoning(answer: string | null) {
+ yield { choices: [{ delta: { reasoning_content: 'deep thought ' } }] };
+ skew += 16_000;
+ yield { choices: [{ delta: { reasoning_content: 'still thinking' } }] };
+ if (answer) yield { choices: [{ delta: { content: answer } }] };
+ yield { choices: [{ delta: {}, finish_reason: 'stop' }] };
+ }
+
+ beforeEach(() => {
+ skew = 0;
+ vi.spyOn(Date, 'now').mockImplementation(() => realNow() + skew);
+ streamChatCompletion.mockReset();
+ getDb().prepare('DELETE FROM rate_limit_cooldowns').run();
+ });
+ afterEach(() => { vi.restoreAllMocks(); });
+
+ it('/v1/messages: pings once committed, then delivers thinking and the answer', async () => {
+ streamChatCompletion.mockImplementation(() => longReasoning('the answer'));
+ const { status, raw, routedVia } = await post(app(), req, '/v1/messages', auth());
+ expect(status).toBe(200);
+ expect(routedVia).toBeTruthy();
+ const start = raw.indexOf('event: message_start');
+ const ping = raw.indexOf('event: ping');
+ expect(start).toBeGreaterThan(-1);
+ expect(ping).toBeGreaterThan(start);
+ expect(raw.indexOf('deep thought still thinking')).toBeGreaterThan(ping);
+ expect(raw).toContain('the answer');
+ expect(raw).toContain('event: message_stop');
+ });
+
+ it('/v1/messages: a committed reasoning-only stream ends with an error event, not a blank success', async () => {
+ streamChatCompletion.mockImplementation(() => longReasoning(null));
+ const { status, raw } = await post(app(), req, '/v1/messages', auth());
+ expect(status).toBe(200);
+ expect(streamChatCompletion).toHaveBeenCalledTimes(1);
+ expect(raw).toContain('event: error');
+ expect(raw).not.toContain('event: message_stop');
+ });
+
+ it('/v1/chat/completions: streams the reasoning live once committed, answer after it', async () => {
+ streamChatCompletion.mockImplementation(() => longReasoning('the answer'));
+ const { status, raw, routedVia } = await post(app(), req, '/v1/chat/completions', auth());
+ expect(status).toBe(200);
+ expect(routedVia).toBeTruthy();
+ const thought = raw.indexOf('still thinking');
+ expect(thought).toBeGreaterThan(-1);
+ expect(thought).toBeLessThan(raw.indexOf('the answer'));
+ });
+
+ it('/v1/chat/completions: a committed reasoning-only stream ends with stream_error', async () => {
+ streamChatCompletion.mockImplementation(() => longReasoning(null));
+ const { status, raw } = await post(app(), req, '/v1/chat/completions', auth());
+ expect(status).toBe(200);
+ expect(streamChatCompletion).toHaveBeenCalledTimes(1);
+ expect(raw).toContain('stream_error');
+ expect(raw).toContain('data: [DONE]');
+ });
+
+ describe('after an early commit', () => {
+ const schema = { type: 'object', properties: { city: { type: 'string' } }, required: ['city'] };
+ function* longReasoningThenBadToolCall(prefix = '') {
+ yield { choices: [{ delta: { reasoning_content: 'deep thought ' } }] };
+ skew += 16_000;
+ yield { choices: [{ delta: { reasoning_content: 'still thinking' } }] };
+ if (prefix) yield { choices: [{ delta: { content: prefix } }] };
+ yield { choices: [{ delta: { tool_calls: [{ index: 0, id: 'call_1', function: { name: 'lookup', arguments: '{}' } }] } }] };
+ yield { choices: [{ delta: {}, finish_reason: 'tool_calls' }] };
+ }
+ afterEach(() => { getDb().prepare('DELETE FROM settings WHERE key = ?').run(TOOL_ARGUMENT_VALIDATION_SETTING); });
+
+ it('/v1/messages: still validates tool arguments and ends with an error event', async () => {
+ setSetting(TOOL_ARGUMENT_VALIDATION_SETTING, '1');
+ streamChatCompletion.mockImplementation(() => longReasoningThenBadToolCall());
+ const { raw } = await post(app(), { ...req, tools: [{ name: 'lookup', description: 'find', input_schema: schema }] }, '/v1/messages', auth());
+ expect(raw).toContain('event: ping');
+ expect(raw).toContain('event: error');
+ expect(raw).not.toContain('"tool_use"');
+ });
+
+ it('/v1/chat/completions: still validates tool arguments and ends with stream_error', async () => {
+ setSetting(TOOL_ARGUMENT_VALIDATION_SETTING, '1');
+ streamChatCompletion.mockImplementation(() => longReasoningThenBadToolCall());
+ const tools = [{ type: 'function', function: { name: 'lookup', description: 'find', parameters: schema } }];
+ const { raw } = await post(app(), { ...req, tools }, '/v1/chat/completions', auth());
+ expect(raw).toContain('still thinking');
+ expect(raw).toContain('stream_error');
+ expect(raw).not.toContain('"tool_calls":[');
+ });
+
+ it.each([
+ ['no text', ''],
+ ['whitespace-only text', '\n\n'],
+ ])('/api/chat (Ollama lane): still validates tool arguments after %s, ending with an error frame', async (_label, prefix) => {
+ setSetting(TOOL_ARGUMENT_VALIDATION_SETTING, '1');
+ streamChatCompletion.mockImplementation(() => longReasoningThenBadToolCall(prefix));
+ const tools = [{ type: 'function', function: { name: 'lookup', description: 'find', parameters: schema } }];
+ const { raw } = await post(app(), { ...req, tools });
+ const frames = raw.split('\n').filter(l => l.trim()).map(l => JSON.parse(l));
+ expect(readStream(raw).thinking).toContain('still thinking');
+ expect(raw).not.toContain('"lookup"');
+ expect(frames.at(-1)).toMatchObject({ error: expect.stringMatching(/argument/i), done: true });
+ });
+
+ it('/v1/chat/completions: json mode never commits early, so prose still fails over invisibly', async () => {
+ streamChatCompletion
+ .mockImplementationOnce(() => longReasoning('just some prose'))
+ .mockImplementationOnce(() => textTurn('{"ok":true}'));
+ const { status, raw } = await post(app(), { ...req, response_format: { type: 'json_object' } }, '/v1/chat/completions', auth());
+ expect(status).toBe(200);
+ expect(streamChatCompletion).toHaveBeenCalledTimes(2);
+ expect(raw).toContain('{\\"ok\\":true}');
+ expect(raw).not.toContain('deep thought');
+ expect(raw).not.toContain('stream_error');
+ });
+
+ it.each(['/v1/messages', '/v1/chat/completions'])('%s: a client that leaves after the early commit is not logged as a provider failure', async (path) => {
+ // The upstream stalls until the server has seen the hang-up, then ends empty.
+ let hungUp!: () => void;
+ const clientLeft = new Promise<void>(r => { hungUp = r; });
+ let finished!: () => void;
+ const attemptOver = new Promise<void>(r => { finished = r; });
+ async function* reasoningThenStall() {
+ try {
+ yield { choices: [{ delta: { reasoning_content: 'deep thought ' } }] };
+ skew += 16_000;
+ yield { choices: [{ delta: { reasoning_content: 'still thinking' } }] };
+ await clientLeft;
+ yield { choices: [{ delta: {}, finish_reason: 'stop' }] };
+ } finally { finished(); }
+ }
+ streamChatCompletion.mockImplementation(() => reasoningThenStall());
+ getDb().prepare('DELETE FROM requests').run();
+ const server = app().listen(0, '127.0.0.1');
+ if (!server.listening) await new Promise<void>(resolve => server.once('listening', () => resolve()));
+ server.on('connection', socket => socket.on('close', () => setImmediate(hungUp)));
+ const abort = new AbortController();
+ const res = await fetch(`http://127.0.0.1:${(server.address() as any).port}${path}`, {
+ method: 'POST', signal: abort.signal,
+ headers: { 'Content-Type': 'application/json', ...auth() },
+ body: JSON.stringify(req),
+ });
+ expect(res.status).toBe(200); // committed early: headers are out
+ abort.abort();
+ await attemptOver;
+ await new Promise(r => setImmediate(r));
+ server.close();
+ expect(streamChatCompletion).toHaveBeenCalledTimes(1);
+ expect(getDb().prepare("SELECT COUNT(*) AS n FROM requests WHERE status = 'error'").get()).toEqual({ n: 0 });
+ });
+ });
+});
diff --git a/server/src/lib/fallback-loop.ts b/server/src/lib/fallback-loop.ts
index dbbfc4bd..9e2f32e5 100644
--- a/server/src/lib/fallback-loop.ts
+++ b/server/src/lib/fallback-loop.ts
@@ -170,6 +170,14 @@ function benchKeyAcrossPlatform(
// Precedence mirrors the response cache: the settings-table value wins when
// present (runtime-tunable), then the env var, then the default.
export const DEFAULT_FALLBACK_TIME_BUDGET_MS = 45_000;
+
+// #1642: streaming surfaces hold reasoning-only output uncommitted so a dead
+// attempt fails over invisibly, but the client hears nothing meanwhile and
+// idle-watchdog clients (Claude Code) hang up. Once an attempt has been
+// reasoning this long it has proven alive, so it commits (headers, status and
+// X-Routed-Via for THIS attempt) and keeps the stream audibly alive; a later
+// failure takes the surface's normal mid-stream error path.
+export const REASONING_HOLD_MAX_MS = 15_000;
// Share of the wall-clock budget an aborted attempt must have been silent for
// before the hedge abort counts as provider health rather than bad luck. An
// attempt late in a ladder inherits only the scraps of the budget its
diff --git a/server/src/lib/inbound-chat.ts b/server/src/lib/inbound-chat.ts
index d5e70823..62b85e34 100644
--- a/server/src/lib/inbound-chat.ts
+++ b/server/src/lib/inbound-chat.ts
@@ -26,6 +26,7 @@ import {
runFallbackLoop,
setExhaustionHeaders,
setFallbackHeaders,
+ REASONING_HOLD_MAX_MS,
type AttemptRecord,
type ExhaustionBody,
} from './fallback-loop.js';
@@ -365,6 +366,7 @@ export async function runInboundChat(
// invisibly, instead of locking the ladder on a blank reply. Flushed in
// order (before the text/tool output) the moment the stream commits.
let heldReasoning = '';
+ const attemptStart = Date.now();
const commit = () => {
if (committed) return;
res.setHeader('X-Routed-Via', routedViaValue(route.platform, route.modelId));
@@ -417,6 +419,8 @@ export async function runInboundChat(
if (typeof deltaReasoning === 'string' && deltaReasoning) {
reasoning += deltaReasoning;
outputTokens += Math.ceil(deltaReasoning.length / 4);
+ // #1642: long reasoning with nothing sent yet: commit, then stream it live.
+ if (!committed && wire.sendReasoningDelta && Date.now() - attemptStart >= REASONING_HOLD_MAX_MS) commit();
if (committed) {
wire.sendReasoningDelta?.(res, route, deltaReasoning);
} else {
@@ -480,9 +484,10 @@ export async function runInboundChat(
// Opt-in schema verdict, before the tool calls are committed. Tool
// calls are buffered to the end of the stream on this path, so a
// tool-only turn has sent nothing yet and can still fail over. A turn
- // that already streamed text is past its commit point — leave it
+ // that already streamed real text is past its commit point — leave it
// alone rather than tearing down a stream the client is reading.
- if (isToolArgumentValidationEnabled() && toolCalls.length > 0 && !committed) {
+ // Whitespace or an early reasoning commit (#1642) is not real text.
+ if (isToolArgumentValidationEnabled() && toolCalls.length > 0 && (!committed || !text.trim())) {
const invalid = invalidToolCallReasons(toolCalls, schemas);
if (invalid.length > 0) throw invalidToolArgumentsError(route.displayName, invalid);
}
diff --git a/server/src/routes/anthropic.ts b/server/src/routes/anthropic.ts
index 423d7cb5..9c3302a5 100644
--- a/server/src/routes/anthropic.ts
+++ b/server/src/routes/anthropic.ts
@@ -22,7 +22,7 @@ import { convertDocumentBlock, documentRejectionMessage } from '../lib/anthropic
import { isClientAbortError, newClientAbortError, newHedgeAbortError, isUpstreamClassificationOutput } from '../lib/error-classify.js';
import { logRequest } from '../lib/request-log.js';
import { extractApiToken, timingSafeStringEqual, getStickyModel, setStickyModel } from './proxy.js';
-import { runFallbackLoop, newFallbackState, fallbackRoutingTokens, recordUpstreamSuccess, type ExhaustionBody, setFallbackHeaders, setExhaustionHeaders, type AttemptRecord, type FallbackState } from '../lib/fallback-loop.js';
+import { runFallbackLoop, newFallbackState, fallbackRoutingTokens, recordUpstreamSuccess, type ExhaustionBody, setFallbackHeaders, setExhaustionHeaders, type AttemptRecord, type FallbackState, REASONING_HOLD_MAX_MS } from '../lib/fallback-loop.js';
import { routedViaValue } from '../lib/header-value.js';
import { applyTokenBudget, tokenBudgetMessage } from '../lib/guardrails.js';
import { resolveAnthropicModel, claudeFamilyDiscoveryEntries } from '../services/anthropic-map.js';
@@ -906,6 +906,7 @@ async function streamCompletion(
// point where it was: a stream that produces ONLY reasoning and dies still
// fails over invisibly, exactly like the OpenAI surface's preamble hold.
let reasoningBuf = '';
+ let quietSince = Date.now(); // last time the client heard from this attempt
const flushThinking = () => {
if (reasoningBuf.length === 0) return;
const idx = nextIndex++;
@@ -969,6 +970,12 @@ async function streamCompletion(
// `reasoning`); the shared <think> extractor emits reasoning_content.
const reasoningDelta = choice.delta?.reasoning_content ?? choice.delta?.reasoning;
if (typeof reasoningDelta === 'string' && reasoningDelta.length > 0) reasoningBuf += reasoningDelta;
+ // #1642: long reasoning with nothing streamed yet: commit and ping.
+ if (reasoningBuf.length > 0 && !textBlockOpen && Date.now() - quietSince >= REASONING_HOLD_MAX_MS) {
+ ensureMessageStart();
+ writeSse(res, 'ping', { type: 'ping' });
+ quietSince = Date.now();
+ }
const text = typeof choice.delta?.content === 'string' ? choice.delta.content : '';
if (text.length === 0) continue;
@@ -1034,10 +1041,11 @@ async function streamCompletion(
// Opt-in schema verdict, same rule as the non-streaming surface above and
// as /chat/completions: this is the surface the silent failure hurt most,
// and Claude Code streams. Placed after the dialect rescue so a rescued
- // call is judged too, and gated on `!messageStarted` — once message_start
- // has gone out there is no failing over, and tearing the SSE stream down
- // would be worse for the client than a tool_use the schema dislikes.
- if (isToolArgumentValidationEnabled() && !messageStarted && completedCalls.length > 0) {
+ // call is judged too, and gated on `!textBlockOpen`: once text has gone
+ // out there is no failing over, and tearing the SSE stream down would be
+ // worse for the client than a tool_use the schema dislikes. A #1642 early
+ // commit sent only pings, so a bad call there ends in the catch's error event.
+ if (isToolArgumentValidationEnabled() && !textBlockOpen && completedCalls.length > 0) {
const invalid = invalidToolCallReasons(
completedCalls.map(c => ({ function: { name: c.name, arguments: c.arguments } })),
schemas,
@@ -1045,9 +1053,10 @@ async function streamCompletion(
if (invalid.length > 0) throw invalidToolArgumentsError(route.displayName, invalid);
}
- // Nothing usable came out — fail over (message_start was never sent, so the
- // client never saw this attempt).
- if (!messageStarted && completedCalls.length === 0) {
+ // Nothing usable came out — fail over (normally message_start was never sent,
+ // so the client never saw this attempt; after a #1642 early commit the
+ // catch below turns this into the mid-stream error event).
+ if (!textBlockOpen && completedCalls.length === 0) {
// Disconnect before the commit point: the pump loop broke with nothing
// accumulated because the CLIENT left, not because the provider failed —
// benching a healthy model+key for it was wrong. StreamAlreadyStarted
diff --git a/server/src/routes/proxy.ts b/server/src/routes/proxy.ts
index daf06144..752fcdda 100644
--- a/server/src/routes/proxy.ts
+++ b/server/src/routes/proxy.ts
@@ -29,7 +29,7 @@ import { logRequest } from '../lib/request-log.js';
import { observeServedModel } from '../lib/served-model.js';
import { parseCacheDirective, cacheActive, isCacheableTemperature, computeCacheKey, getCachedResponse, storeCachedResponse, getCachedStreamResponse, storeCachedStreamResponse, STREAM_CACHE_MAX_BYTES } from '../services/cache.js';
import { normalizeIdempotencyKey, hashIdempotencyKey, computeIdempotencyFingerprint, lookupIdempotencyReplay, storeIdempotencyResult } from '../services/idempotency.js';
-import { runFallbackLoop, newFallbackState, fallbackRoutingTokens, recordUpstreamSuccess, exhaustedRetryError, setFallbackHeaders, exhaustionErrorPayload, setExhaustionHeaders, type AttemptRecord } from '../lib/fallback-loop.js';
+import { runFallbackLoop, newFallbackState, fallbackRoutingTokens, recordUpstreamSuccess, exhaustedRetryError, setFallbackHeaders, exhaustionErrorPayload, setExhaustionHeaders, type AttemptRecord, REASONING_HOLD_MAX_MS } from '../lib/fallback-loop.js';
import { routedViaValue, safeHeaderValue } from '../lib/header-value.js';
import { applyTokenBudget, tokenBudgetMessage } from '../lib/guardrails.js';
import { samplingParamSchemaFields, pickSamplingParams, supportedParametersForPlatforms } from '../lib/sampling-params.js';
@@ -2530,6 +2530,7 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {
let totalOutputTokens = 0;
let headerSent = false;
let ttfbMs: number | null = null;
+ const attemptStart = Date.now();
// Hold-window state: 'undecided' until the first text either matches
// a dialect marker (→ 'dialect': buffer everything, rescue at end),
@@ -2705,6 +2706,9 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {
// reasoning keys; forwarding them raw would duplicate the call).
// #764: thinking-only chunks still consumed tokens — count them.
if (reasoning.length > 0) totalOutputTokens += Math.ceil(reasoning.length / 4);
+ // #1642: long reasoning with nothing sent yet: commit, then stream it live.
+ // Not for response_format: json mode keeps its free failover (#933).
+ if (reasoning.length > 0 && !headerSent && !samplingParams.response_format && Date.now() - attemptStart >= REASONING_HOLD_MAX_MS) flushHeaders();
if (choice.delta && Object.keys(choice.delta).some(k => k !== 'content' && k !== 'tool_calls' && choice.delta[k] != null)) {
// `usage: undefined` (dropped by JSON.stringify): it was held
// above and is re-emitted once, after our finish chunk.
@@ -2785,8 +2789,9 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {
// Opt-in schema verdict, taken AFTER the rescue so it covers the
// calls the rescue reconstructed from prose — those are the ones
// most likely to be malformed, and running first exempted exactly
- // them. `!headerSent` is the whole licence to throw here: the commit
- // point is held until the first meaningful content, so the common
+ // them. No prose flushed yet is the whole licence to throw here (after
+ // a #1642 reasoning-only commit the catch ends it with `stream_error`):
+ // the commit point is held until the first meaningful content, so the common
// tool-call turn (no prose before the call) has sent no bytes yet and
// can still fail over invisibly. A turn that already flushed prose is
// past the point of no return — the catch below would have to tear
@@ -2794,7 +2799,7 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {
// for the client than forwarding a tool call the schema dislikes.
// Off-by-default or not, this check must never turn a served answer
// into a broken one.
- if (isToolArgumentValidationEnabled() && !headerSent && completedCalls.length > 0) {
+ if (isToolArgumentValidationEnabled() && mode !== 'passthrough' && completedCalls.length > 0) {
const invalid = invalidToolCallReasons(completedCalls, schemas);
if (invalid.length > 0) throw invalidToolArgumentsError(route.displayName, invalid);
}
@@ -2805,16 +2810,17 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {
// throw below, which would bench a healthy model+key for 90s and
// log a provider error for every Ctrl-C during a reasoning model's
// TTFB window.
- if (clientGone && !headerSent && heldText.trim().length === 0 && completedCalls.length === 0) {
+ if (clientGone && mode !== 'passthrough' && heldText.trim().length === 0 && completedCalls.length === 0) {
console.log(`[Proxy] client disconnected before first token from ${route.displayName} — dropping attempt without benching`);
return 'committed';
}
- const hasText = headerSent || heldText.trim().length > 0;
+ const hasText = mode === 'passthrough' || heldText.trim().length > 0;
if (!hasText && completedCalls.length === 0) {
// Nothing usable came out — same failover semantics as the
- // non-stream empty-completion path. Headers can't have been sent
- // (header flush requires payload), so the client never notices.
+ // non-stream empty-completion path. Headers are only sent here after a
+ // #1642 long-reasoning commit, where the catch emits stream_error;
+ // otherwise the client never notices.
// finish_reason 'length' = the model spent the whole output budget
// on hidden reasoning before any visible text: fail over, but skip
// the cooldown/penalty (not a provider-health signal).
@@ -2838,9 +2844,10 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {
// model that answers in prose despite the forwarded response_format
// ships the essay as a "success" — the worst case for a
// machine-readable request. json mode held every byte (headers never
- // flushed), so failing over here is free: skipBench (provider
- // healthy, the MODEL misbehaved) + skipModelForRequest (a sibling
- // key would misbehave identically). Mirrors proxy.ts non-stream.
+ // flushed, and the #1642 long-reasoning commit is skipped for it), so
+ // failing over here is free: skipBench (provider healthy, the MODEL
+ // misbehaved) + skipModelForRequest (a sibling key would misbehave
+ // identically). Mirrors proxy.ts non-stream.
if (mode === 'json' && samplingParams.response_format && completedCalls.length === 0) {
const enforced = enforceJsonContent(heldText);
if (!enforced.ok) {