7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.
TypeScript
fix(stream): commit long-reasoning streams early so idle clients stay connected (#1642) (#1645)
Fangore48367346e21d
diff --git a/docs/en/architecture/03-streaming-pipeline.md b/docs/en/architecture/03-streaming-pipeline.mdindex 140323cd..863a20cb 100644--- a/docs/en/architecture/03-streaming-pipeline.md+++ b/docs/en/architecture/03-streaming-pipeline.md@@ -66,7 +66,7 @@ headerSent = false- `undecided` + `!wantsTools` OR `!couldBecomeDialectMarker` OR `heldText > 256` → `passthrough`, flush headers, stream- `passthrough` → forward immediately (strip tool_calls from delta, re-emit complete at end)- `dialect` → keep buffering-9. **Empty chunks** (role/keep-alive/reasoning-only): held in `preamble` until flush.+9. **Empty chunks** (role/keep-alive/reasoning-only): held in `preamble` until flush. Reasoning that runs past `REASONING_HOLD_MAX_MS` (15s) flushes headers early and then streams live, so idle-watchdog clients stay connected (#1642); a failure after that is a mid-stream error.**Stream end (clean):**diff --git a/server/src/__tests__/routes/reasoning-only-failover.test.ts b/server/src/__tests__/routes/reasoning-only-failover.test.tsindex 3a8c98aa..7ccaf968 100644--- a/server/src/__tests__/routes/reasoning-only-failover.test.ts+++ b/server/src/__tests__/routes/reasoning-only-failover.test.ts@@ -1,4 +1,4 @@-import { describe, it, expect, beforeAll, beforeEach, vi } from 'vitest';+import { describe, it, expect, beforeAll, beforeEach, afterEach, vi } from 'vitest';import type { Express } from 'express';// Reasoning-only responses used to pass as success on the shared inbound-chat@@ -29,24 +29,25 @@ vi.mock('../../providers/index.js', async (importOriginal) => {});const { createApp } = await import('../../app.js');-const { initDb, getDb, setSetting } = await import('../../db/index.js');+const { initDb, getDb, setSetting, getUnifiedApiKey } = await import('../../db/index.js');const { encrypt } = await import('../../lib/crypto.js');const { setRoutingStrategy } = await import('../../services/router.js');+const { TOOL_ARGUMENT_VALIDATION_SETTING } = await import('../../lib/tool-validate.js');-async function post(app: Express, body: Record<string, unknown>) {+async function post(app: Express, body: Record<string, unknown>, path = '/api/chat', headers: Record<string, string> = {}) {const server = app.listen(0, '127.0.0.1');if (!server.listening) await new Promise<void>(resolve => server.once('listening', () => resolve()));const addr = server.address() as any;- const res = await fetch(`http://127.0.0.1:${addr.port}/api/chat`, {+ const res = await fetch(`http://127.0.0.1:${addr.port}${path}`, {method: 'POST',- headers: { 'Content-Type': 'application/json' },+ headers: { 'Content-Type': 'application/json', ...headers },body: JSON.stringify(body),});const raw = await res.text();server.close();let json: any = null;try { json = JSON.parse(raw); } catch { /* NDJSON stream */ }- return { status: res.status, body: json, raw };+ return { status: res.status, body: json, raw, routedVia: res.headers.get('x-routed-via') };}/** Collect content / thinking / tool_calls out of the Ollama NDJSON stream. */@@ -90,14 +91,16 @@ function insertKey(label: string): void {`).run(label, encrypted, iv, authTag);}+let builtApp: Express;+const app = () => builtApp;+describe('reasoning-only turns fail over on the shared inbound-chat lane (#1184)', () => {- let app: Express;beforeAll(() => {process.env.ENCRYPTION_KEY = '0'.repeat(64);initDb(':memory:');setSetting('ollama_emulation', 'open-loopback');- app = createApp();+ builtApp = createApp();setRoutingStrategy('priority');insertKey('first');insertKey('second');@@ -114,7 +117,7 @@ describe('reasoning-only turns fail over on the shared inbound-chat lane (#1184).mockResolvedValueOnce({ choices: [{ message: { reasoning_content: 'all thinking, no answer' }, finish_reason: 'stop' }] }).mockResolvedValueOnce({ choices: [{ message: { content: 'the real answer' }, finish_reason: 'stop' }] });- const { status, body } = await post(app, {+ const { status, body } = await post(app(), {model: 'auto',messages: [{ role: 'user', content: 'hi' }],stream: false,@@ -132,7 +135,7 @@ describe('reasoning-only turns fail over on the shared inbound-chat lane (#1184).mockImplementationOnce(() => reasoningOnlyTurn()).mockImplementationOnce(() => textTurn('the real answer'));- const { status, raw } = await post(app, {+ const { status, raw } = await post(app(), {model: 'auto',messages: [{ role: 'user', content: 'hi' }],stream: true,@@ -148,7 +151,7 @@ describe('reasoning-only turns fail over on the shared inbound-chat lane (#1184)it('streaming: thinking that precedes real content is delivered, in order', async () => {streamChatCompletion.mockImplementation(() => reasoningThenTextTurn());- const { status, raw } = await post(app, {+ const { status, raw } = await post(app(), {model: 'auto',messages: [{ role: 'user', content: 'hi' }],stream: true,@@ -165,3 +168,166 @@ describe('reasoning-only turns fail over on the shared inbound-chat lane (#1184)expect(thinkingIdx).toBeLessThan(contentIdx);});});++// #1642: holding reasoning uncommitted for minutes made idle-watchdog clients+// (Claude Code) hang up. Past REASONING_HOLD_MAX_MS the attempt commits with+// its real status + routing headers and keeps the stream alive; dying after+// that goes through the surface's normal mid-stream error path.+describe('long reasoning commits and keeps the stream alive (#1642)', () => {+ let skew = 0;+ const realNow = Date.now.bind(Date);+ const auth = () => ({ Authorization: `Bearer ${getUnifiedApiKey()}`, 'anthropic-version': '2023-06-01' });+ const req = { model: 'auto', max_tokens: 100, stream: true, messages: [{ role: 'user', content: 'hi' }] };++ /** Reasoning that outlasts the hold window, then optionally an answer. */+ function* longReasoning(answer: string | null) {+ yield { choices: [{ delta: { reasoning_content: 'deep thought ' } }] };+ skew += 16_000;+ yield { choices: [{ delta: { reasoning_content: 'still thinking' } }] };+ if (answer) yield { choices: [{ delta: { content: answer } }] };+ yield { choices: [{ delta: {}, finish_reason: 'stop' }] };+ }++ beforeEach(() => {+ skew = 0;+ vi.spyOn(Date, 'now').mockImplementation(() => realNow() + skew);+ streamChatCompletion.mockReset();+ getDb().prepare('DELETE FROM rate_limit_cooldowns').run();+ });+ afterEach(() => { vi.restoreAllMocks(); });++ it('/v1/messages: pings once committed, then delivers thinking and the answer', async () => {+ streamChatCompletion.mockImplementation(() => longReasoning('the answer'));+ const { status, raw, routedVia } = await post(app(), req, '/v1/messages', auth());+ expect(status).toBe(200);+ expect(routedVia).toBeTruthy();+ const start = raw.indexOf('event: message_start');+ const ping = raw.indexOf('event: ping');+ expect(start).toBeGreaterThan(-1);+ expect(ping).toBeGreaterThan(start);+ expect(raw.indexOf('deep thought still thinking')).toBeGreaterThan(ping);+ expect(raw).toContain('the answer');+ expect(raw).toContain('event: message_stop');+ });++ it('/v1/messages: a committed reasoning-only stream ends with an error event, not a blank success', async () => {+ streamChatCompletion.mockImplementation(() => longReasoning(null));+ const { status, raw } = await post(app(), req, '/v1/messages', auth());+ expect(status).toBe(200);+ expect(streamChatCompletion).toHaveBeenCalledTimes(1);+ expect(raw).toContain('event: error');+ expect(raw).not.toContain('event: message_stop');+ });++ it('/v1/chat/completions: streams the reasoning live once committed, answer after it', async () => {+ streamChatCompletion.mockImplementation(() => longReasoning('the answer'));+ const { status, raw, routedVia } = await post(app(), req, '/v1/chat/completions', auth());+ expect(status).toBe(200);+ expect(routedVia).toBeTruthy();+ const thought = raw.indexOf('still thinking');+ expect(thought).toBeGreaterThan(-1);+ expect(thought).toBeLessThan(raw.indexOf('the answer'));+ });++ it('/v1/chat/completions: a committed reasoning-only stream ends with stream_error', async () => {+ streamChatCompletion.mockImplementation(() => longReasoning(null));+ const { status, raw } = await post(app(), req, '/v1/chat/completions', auth());+ expect(status).toBe(200);+ expect(streamChatCompletion).toHaveBeenCalledTimes(1);+ expect(raw).toContain('stream_error');+ expect(raw).toContain('data: [DONE]');+ });++ describe('after an early commit', () => {+ const schema = { type: 'object', properties: { city: { type: 'string' } }, required: ['city'] };+ function* longReasoningThenBadToolCall(prefix = '') {+ yield { choices: [{ delta: { reasoning_content: 'deep thought ' } }] };+ skew += 16_000;+ yield { choices: [{ delta: { reasoning_content: 'still thinking' } }] };+ if (prefix) yield { choices: [{ delta: { content: prefix } }] };+ yield { choices: [{ delta: { tool_calls: [{ index: 0, id: 'call_1', function: { name: 'lookup', arguments: '{}' } }] } }] };+ yield { choices: [{ delta: {}, finish_reason: 'tool_calls' }] };+ }+ afterEach(() => { getDb().prepare('DELETE FROM settings WHERE key = ?').run(TOOL_ARGUMENT_VALIDATION_SETTING); });++ it('/v1/messages: still validates tool arguments and ends with an error event', async () => {+ setSetting(TOOL_ARGUMENT_VALIDATION_SETTING, '1');+ streamChatCompletion.mockImplementation(() => longReasoningThenBadToolCall());+ const { raw } = await post(app(), { ...req, tools: [{ name: 'lookup', description: 'find', input_schema: schema }] }, '/v1/messages', auth());+ expect(raw).toContain('event: ping');+ expect(raw).toContain('event: error');+ expect(raw).not.toContain('"tool_use"');+ });++ it('/v1/chat/completions: still validates tool arguments and ends with stream_error', async () => {+ setSetting(TOOL_ARGUMENT_VALIDATION_SETTING, '1');+ streamChatCompletion.mockImplementation(() => longReasoningThenBadToolCall());+ const tools = [{ type: 'function', function: { name: 'lookup', description: 'find', parameters: schema } }];+ const { raw } = await post(app(), { ...req, tools }, '/v1/chat/completions', auth());+ expect(raw).toContain('still thinking');+ expect(raw).toContain('stream_error');+ expect(raw).not.toContain('"tool_calls":[');+ });++ it.each([+ ['no text', ''],+ ['whitespace-only text', '\n\n'],+ ])('/api/chat (Ollama lane): still validates tool arguments after %s, ending with an error frame', async (_label, prefix) => {+ setSetting(TOOL_ARGUMENT_VALIDATION_SETTING, '1');+ streamChatCompletion.mockImplementation(() => longReasoningThenBadToolCall(prefix));+ const tools = [{ type: 'function', function: { name: 'lookup', description: 'find', parameters: schema } }];+ const { raw } = await post(app(), { ...req, tools });+ const frames = raw.split('\n').filter(l => l.trim()).map(l => JSON.parse(l));+ expect(readStream(raw).thinking).toContain('still thinking');+ expect(raw).not.toContain('"lookup"');+ expect(frames.at(-1)).toMatchObject({ error: expect.stringMatching(/argument/i), done: true });+ });++ it('/v1/chat/completions: json mode never commits early, so prose still fails over invisibly', async () => {+ streamChatCompletion+ .mockImplementationOnce(() => longReasoning('just some prose'))+ .mockImplementationOnce(() => textTurn('{"ok":true}'));+ const { status, raw } = await post(app(), { ...req, response_format: { type: 'json_object' } }, '/v1/chat/completions', auth());+ expect(status).toBe(200);+ expect(streamChatCompletion).toHaveBeenCalledTimes(2);+ expect(raw).toContain('{\\"ok\\":true}');+ expect(raw).not.toContain('deep thought');+ expect(raw).not.toContain('stream_error');+ });++ it.each(['/v1/messages', '/v1/chat/completions'])('%s: a client that leaves after the early commit is not logged as a provider failure', async (path) => {+ // The upstream stalls until the server has seen the hang-up, then ends empty.+ let hungUp!: () => void;+ const clientLeft = new Promise<void>(r => { hungUp = r; });+ let finished!: () => void;+ const attemptOver = new Promise<void>(r => { finished = r; });+ async function* reasoningThenStall() {+ try {+ yield { choices: [{ delta: { reasoning_content: 'deep thought ' } }] };+ skew += 16_000;+ yield { choices: [{ delta: { reasoning_content: 'still thinking' } }] };+ await clientLeft;+ yield { choices: [{ delta: {}, finish_reason: 'stop' }] };+ } finally { finished(); }+ }+ streamChatCompletion.mockImplementation(() => reasoningThenStall());+ getDb().prepare('DELETE FROM requests').run();+ const server = app().listen(0, '127.0.0.1');+ if (!server.listening) await new Promise<void>(resolve => server.once('listening', () => resolve()));+ server.on('connection', socket => socket.on('close', () => setImmediate(hungUp)));+ const abort = new AbortController();+ const res = await fetch(`http://127.0.0.1:${(server.address() as any).port}${path}`, {+ method: 'POST', signal: abort.signal,+ headers: { 'Content-Type': 'application/json', ...auth() },+ body: JSON.stringify(req),+ });+ expect(res.status).toBe(200); // committed early: headers are out+ abort.abort();+ await attemptOver;+ await new Promise(r => setImmediate(r));+ server.close();+ expect(streamChatCompletion).toHaveBeenCalledTimes(1);+ expect(getDb().prepare("SELECT COUNT(*) AS n FROM requests WHERE status = 'error'").get()).toEqual({ n: 0 });+ });+ });+});diff --git a/server/src/lib/fallback-loop.ts b/server/src/lib/fallback-loop.tsindex dbbfc4bd..9e2f32e5 100644--- a/server/src/lib/fallback-loop.ts+++ b/server/src/lib/fallback-loop.ts@@ -170,6 +170,14 @@ function benchKeyAcrossPlatform(// Precedence mirrors the response cache: the settings-table value wins when// present (runtime-tunable), then the env var, then the default.export const DEFAULT_FALLBACK_TIME_BUDGET_MS = 45_000;++// #1642: streaming surfaces hold reasoning-only output uncommitted so a dead+// attempt fails over invisibly, but the client hears nothing meanwhile and+// idle-watchdog clients (Claude Code) hang up. Once an attempt has been+// reasoning this long it has proven alive, so it commits (headers, status and+// X-Routed-Via for THIS attempt) and keeps the stream audibly alive; a later+// failure takes the surface's normal mid-stream error path.+export const REASONING_HOLD_MAX_MS = 15_000;// Share of the wall-clock budget an aborted attempt must have been silent for// before the hedge abort counts as provider health rather than bad luck. An// attempt late in a ladder inherits only the scraps of the budget itsdiff --git a/server/src/lib/inbound-chat.ts b/server/src/lib/inbound-chat.tsindex d5e70823..62b85e34 100644--- a/server/src/lib/inbound-chat.ts+++ b/server/src/lib/inbound-chat.ts@@ -26,6 +26,7 @@ import {runFallbackLoop,setExhaustionHeaders,setFallbackHeaders,+ REASONING_HOLD_MAX_MS,type AttemptRecord,type ExhaustionBody,} from './fallback-loop.js';@@ -365,6 +366,7 @@ export async function runInboundChat(// invisibly, instead of locking the ladder on a blank reply. Flushed in// order (before the text/tool output) the moment the stream commits.let heldReasoning = '';+ const attemptStart = Date.now();const commit = () => {if (committed) return;res.setHeader('X-Routed-Via', routedViaValue(route.platform, route.modelId));@@ -417,6 +419,8 @@ export async function runInboundChat(if (typeof deltaReasoning === 'string' && deltaReasoning) {reasoning += deltaReasoning;outputTokens += Math.ceil(deltaReasoning.length / 4);+ // #1642: long reasoning with nothing sent yet: commit, then stream it live.+ if (!committed && wire.sendReasoningDelta && Date.now() - attemptStart >= REASONING_HOLD_MAX_MS) commit();if (committed) {wire.sendReasoningDelta?.(res, route, deltaReasoning);} else {@@ -480,9 +484,10 @@ export async function runInboundChat(// Opt-in schema verdict, before the tool calls are committed. Tool// calls are buffered to the end of the stream on this path, so a// tool-only turn has sent nothing yet and can still fail over. A turn- // that already streamed text is past its commit point — leave it+ // that already streamed real text is past its commit point — leave it// alone rather than tearing down a stream the client is reading.- if (isToolArgumentValidationEnabled() && toolCalls.length > 0 && !committed) {+ // Whitespace or an early reasoning commit (#1642) is not real text.+ if (isToolArgumentValidationEnabled() && toolCalls.length > 0 && (!committed || !text.trim())) {const invalid = invalidToolCallReasons(toolCalls, schemas);if (invalid.length > 0) throw invalidToolArgumentsError(route.displayName, invalid);}diff --git a/server/src/routes/anthropic.ts b/server/src/routes/anthropic.tsindex 423d7cb5..9c3302a5 100644--- a/server/src/routes/anthropic.ts+++ b/server/src/routes/anthropic.ts@@ -22,7 +22,7 @@ import { convertDocumentBlock, documentRejectionMessage } from '../lib/anthropicimport { isClientAbortError, newClientAbortError, newHedgeAbortError, isUpstreamClassificationOutput } from '../lib/error-classify.js';import { logRequest } from '../lib/request-log.js';import { extractApiToken, timingSafeStringEqual, getStickyModel, setStickyModel } from './proxy.js';-import { runFallbackLoop, newFallbackState, fallbackRoutingTokens, recordUpstreamSuccess, type ExhaustionBody, setFallbackHeaders, setExhaustionHeaders, type AttemptRecord, type FallbackState } from '../lib/fallback-loop.js';+import { runFallbackLoop, newFallbackState, fallbackRoutingTokens, recordUpstreamSuccess, type ExhaustionBody, setFallbackHeaders, setExhaustionHeaders, type AttemptRecord, type FallbackState, REASONING_HOLD_MAX_MS } from '../lib/fallback-loop.js';import { routedViaValue } from '../lib/header-value.js';import { applyTokenBudget, tokenBudgetMessage } from '../lib/guardrails.js';import { resolveAnthropicModel, claudeFamilyDiscoveryEntries } from '../services/anthropic-map.js';@@ -906,6 +906,7 @@ async function streamCompletion(// point where it was: a stream that produces ONLY reasoning and dies still// fails over invisibly, exactly like the OpenAI surface's preamble hold.let reasoningBuf = '';+ let quietSince = Date.now(); // last time the client heard from this attemptconst flushThinking = () => {if (reasoningBuf.length === 0) return;const idx = nextIndex++;@@ -969,6 +970,12 @@ async function streamCompletion(// `reasoning`); the shared <think> extractor emits reasoning_content.const reasoningDelta = choice.delta?.reasoning_content ?? choice.delta?.reasoning;if (typeof reasoningDelta === 'string' && reasoningDelta.length > 0) reasoningBuf += reasoningDelta;+ // #1642: long reasoning with nothing streamed yet: commit and ping.+ if (reasoningBuf.length > 0 && !textBlockOpen && Date.now() - quietSince >= REASONING_HOLD_MAX_MS) {+ ensureMessageStart();+ writeSse(res, 'ping', { type: 'ping' });+ quietSince = Date.now();+ }const text = typeof choice.delta?.content === 'string' ? choice.delta.content : '';if (text.length === 0) continue;@@ -1034,10 +1041,11 @@ async function streamCompletion(// Opt-in schema verdict, same rule as the non-streaming surface above and// as /chat/completions: this is the surface the silent failure hurt most,// and Claude Code streams. Placed after the dialect rescue so a rescued- // call is judged too, and gated on `!messageStarted` — once message_start- // has gone out there is no failing over, and tearing the SSE stream down- // would be worse for the client than a tool_use the schema dislikes.- if (isToolArgumentValidationEnabled() && !messageStarted && completedCalls.length > 0) {+ // call is judged too, and gated on `!textBlockOpen`: once text has gone+ // out there is no failing over, and tearing the SSE stream down would be+ // worse for the client than a tool_use the schema dislikes. A #1642 early+ // commit sent only pings, so a bad call there ends in the catch's error event.+ if (isToolArgumentValidationEnabled() && !textBlockOpen && completedCalls.length > 0) {const invalid = invalidToolCallReasons(completedCalls.map(c => ({ function: { name: c.name, arguments: c.arguments } })),schemas,@@ -1045,9 +1053,10 @@ async function streamCompletion(if (invalid.length > 0) throw invalidToolArgumentsError(route.displayName, invalid);}- // Nothing usable came out — fail over (message_start was never sent, so the- // client never saw this attempt).- if (!messageStarted && completedCalls.length === 0) {+ // Nothing usable came out — fail over (normally message_start was never sent,+ // so the client never saw this attempt; after a #1642 early commit the+ // catch below turns this into the mid-stream error event).+ if (!textBlockOpen && completedCalls.length === 0) {// Disconnect before the commit point: the pump loop broke with nothing// accumulated because the CLIENT left, not because the provider failed —// benching a healthy model+key for it was wrong. StreamAlreadyStarteddiff --git a/server/src/routes/proxy.ts b/server/src/routes/proxy.tsindex daf06144..752fcdda 100644--- a/server/src/routes/proxy.ts+++ b/server/src/routes/proxy.ts@@ -29,7 +29,7 @@ import { logRequest } from '../lib/request-log.js';import { observeServedModel } from '../lib/served-model.js';import { parseCacheDirective, cacheActive, isCacheableTemperature, computeCacheKey, getCachedResponse, storeCachedResponse, getCachedStreamResponse, storeCachedStreamResponse, STREAM_CACHE_MAX_BYTES } from '../services/cache.js';import { normalizeIdempotencyKey, hashIdempotencyKey, computeIdempotencyFingerprint, lookupIdempotencyReplay, storeIdempotencyResult } from '../services/idempotency.js';-import { runFallbackLoop, newFallbackState, fallbackRoutingTokens, recordUpstreamSuccess, exhaustedRetryError, setFallbackHeaders, exhaustionErrorPayload, setExhaustionHeaders, type AttemptRecord } from '../lib/fallback-loop.js';+import { runFallbackLoop, newFallbackState, fallbackRoutingTokens, recordUpstreamSuccess, exhaustedRetryError, setFallbackHeaders, exhaustionErrorPayload, setExhaustionHeaders, type AttemptRecord, REASONING_HOLD_MAX_MS } from '../lib/fallback-loop.js';import { routedViaValue, safeHeaderValue } from '../lib/header-value.js';import { applyTokenBudget, tokenBudgetMessage } from '../lib/guardrails.js';import { samplingParamSchemaFields, pickSamplingParams, supportedParametersForPlatforms } from '../lib/sampling-params.js';@@ -2530,6 +2530,7 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {let totalOutputTokens = 0;let headerSent = false;let ttfbMs: number | null = null;+ const attemptStart = Date.now();// Hold-window state: 'undecided' until the first text either matches// a dialect marker (→ 'dialect': buffer everything, rescue at end),@@ -2705,6 +2706,9 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {// reasoning keys; forwarding them raw would duplicate the call).// #764: thinking-only chunks still consumed tokens — count them.if (reasoning.length > 0) totalOutputTokens += Math.ceil(reasoning.length / 4);+ // #1642: long reasoning with nothing sent yet: commit, then stream it live.+ // Not for response_format: json mode keeps its free failover (#933).+ if (reasoning.length > 0 && !headerSent && !samplingParams.response_format && Date.now() - attemptStart >= REASONING_HOLD_MAX_MS) flushHeaders();if (choice.delta && Object.keys(choice.delta).some(k => k !== 'content' && k !== 'tool_calls' && choice.delta[k] != null)) {// `usage: undefined` (dropped by JSON.stringify): it was held// above and is re-emitted once, after our finish chunk.@@ -2785,8 +2789,9 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {// Opt-in schema verdict, taken AFTER the rescue so it covers the// calls the rescue reconstructed from prose — those are the ones// most likely to be malformed, and running first exempted exactly- // them. `!headerSent` is the whole licence to throw here: the commit- // point is held until the first meaningful content, so the common+ // them. No prose flushed yet is the whole licence to throw here (after+ // a #1642 reasoning-only commit the catch ends it with `stream_error`):+ // the commit point is held until the first meaningful content, so the common// tool-call turn (no prose before the call) has sent no bytes yet and// can still fail over invisibly. A turn that already flushed prose is// past the point of no return — the catch below would have to tear@@ -2794,7 +2799,7 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {// for the client than forwarding a tool call the schema dislikes.// Off-by-default or not, this check must never turn a served answer// into a broken one.- if (isToolArgumentValidationEnabled() && !headerSent && completedCalls.length > 0) {+ if (isToolArgumentValidationEnabled() && mode !== 'passthrough' && completedCalls.length > 0) {const invalid = invalidToolCallReasons(completedCalls, schemas);if (invalid.length > 0) throw invalidToolArgumentsError(route.displayName, invalid);}@@ -2805,16 +2810,17 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {// throw below, which would bench a healthy model+key for 90s and// log a provider error for every Ctrl-C during a reasoning model's// TTFB window.- if (clientGone && !headerSent && heldText.trim().length === 0 && completedCalls.length === 0) {+ if (clientGone && mode !== 'passthrough' && heldText.trim().length === 0 && completedCalls.length === 0) {console.log(`[Proxy] client disconnected before first token from ${route.displayName} — dropping attempt without benching`);return 'committed';}- const hasText = headerSent || heldText.trim().length > 0;+ const hasText = mode === 'passthrough' || heldText.trim().length > 0;if (!hasText && completedCalls.length === 0) {// Nothing usable came out — same failover semantics as the- // non-stream empty-completion path. Headers can't have been sent- // (header flush requires payload), so the client never notices.+ // non-stream empty-completion path. Headers are only sent here after a+ // #1642 long-reasoning commit, where the catch emits stream_error;+ // otherwise the client never notices.// finish_reason 'length' = the model spent the whole output budget// on hidden reasoning before any visible text: fail over, but skip// the cooldown/penalty (not a provider-health signal).@@ -2838,9 +2844,10 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {// model that answers in prose despite the forwarded response_format// ships the essay as a "success" — the worst case for a// machine-readable request. json mode held every byte (headers never- // flushed), so failing over here is free: skipBench (provider- // healthy, the MODEL misbehaved) + skipModelForRequest (a sibling- // key would misbehave identically). Mirrors proxy.ts non-stream.+ // flushed, and the #1642 long-reasoning commit is skipped for it), so+ // failing over here is free: skipBench (provider healthy, the MODEL+ // misbehaved) + skipModelForRequest (a sibling key would misbehave+ // identically). Mirrors proxy.ts non-stream.if (mode === 'json' && samplingParams.response_format && completedCalls.length === 0) {const enforced = enforceJsonContent(heldText);if (!enforced.ok) {