Skip to content

Claude Code. Any Model. The most powerful AI coding agent now speaks every language.

fix(adapters): end the upstream read loop before a cancel releases a lease

Jack Rudenkofd5d2243823c
diff --git a/packages/cli/src/handlers/native-handler.ts b/packages/cli/src/handlers/native-handler.ts
index 94017672..9d05de51 100644
--- a/packages/cli/src/handlers/native-handler.ts
+++ b/packages/cli/src/handlers/native-handler.ts
@@ -9,6 +9,10 @@ import {
stripAdvisorBeta,
} from "./native-handler-advisor.js";
import { wrapAnthropicError } from "./shared/anthropic-error.js";
+import {
+ type UpstreamReaderSlot,
+ cancelEndsUpstreamWork,
+} from "./shared/stream-parsers/upstream-cancel.js";
import { stripUnsignedThinkingBlocks } from "./shared/thinking-signature.js";
import type { ModelHandler } from "./types.js";
@@ -133,38 +137,44 @@ export class NativeHandler implements ModelHandler {
// Handle streaming
if (contentType.includes("text/event-stream")) {
log("[Native] Streaming response detected");
+ // A downstream cancel cancels this reader and waits for the loop below
+ // (which logs the whole stream at its end) before it resolves.
+ const upstream: UpstreamReaderSlot = {};
return c.body(
- new ReadableStream({
- async start(controller) {
- const reader = anthropicResponse.body?.getReader();
- if (!reader) throw new Error("No reader");
-
- const decoder = new TextDecoder();
- let buffer = "";
- let eventLog = "";
-
- try {
- while (true) {
- const { done, value } = await reader.read();
- if (done) break;
-
- controller.enqueue(value);
-
- // Basic logging
- const chunkText = decoder.decode(value, { stream: true });
- buffer += chunkText;
- const lines = buffer.split("\n");
- buffer = lines.pop() || "";
- for (const line of lines) if (line.trim()) eventLog += `${line}\n`;
+ new ReadableStream(
+ cancelEndsUpstreamWork<Uint8Array>(upstream, {
+ async start(controller) {
+ const reader = anthropicResponse.body?.getReader();
+ if (!reader) throw new Error("No reader");
+ upstream.reader = reader;
+
+ const decoder = new TextDecoder();
+ let buffer = "";
+ let eventLog = "";
+
+ try {
+ while (true) {
+ const { done, value } = await reader.read();
+ if (done) break;
+
+ controller.enqueue(value);
+
+ // Basic logging
+ const chunkText = decoder.decode(value, { stream: true });
+ buffer += chunkText;
+ const lines = buffer.split("\n");
+ buffer = lines.pop() || "";
+ for (const line of lines) if (line.trim()) eventLog += `${line}\n`;
+ }
+ if (eventLog) log(eventLog);
+ controller.close();
+ } catch (e) {
+ log(`[Native] Stream Error: ${e}`);
+ controller.close();
}
- if (eventLog) log(eventLog);
- controller.close();
- } catch (e) {
- log(`[Native] Stream Error: ${e}`);
- controller.close();
- }
- },
- }),
+ },
+ })
+ ),
{
headers: {
"Content-Type": contentType,
diff --git a/packages/cli/src/handlers/shared/call-context.ts b/packages/cli/src/handlers/shared/call-context.ts
index a64a8aa9..6be920f4 100644
--- a/packages/cli/src/handlers/shared/call-context.ts
+++ b/packages/cli/src/handlers/shared/call-context.ts
@@ -182,6 +182,11 @@ export function credentialValuesFromRequest(
*
* A cancel is forwarded to the source reader FIRST, so an upstream still
* generating stops, and whatever it logs while stopping is still redacted.
+ * `release` waits for that cancel's promise, and every stream parser resolves
+ * it only after cancelling its upstream reader AND after its read loop has
+ * settled (`cancelEndsUpstreamWork`, `stream-parsers/upstream-cancel.ts`); a
+ * flag-only parser cancel let the loop keep reading and logging frames after
+ * the lease was gone.
*/
export function releaseWhenBodyEnds(response: Response, release: () => void): Response {
const source = response.body;
diff --git a/packages/cli/src/handlers/shared/devin-stream-head-sniffer.ts b/packages/cli/src/handlers/shared/devin-stream-head-sniffer.ts
index 4f670a25..1e230a1f 100644
--- a/packages/cli/src/handlers/shared/devin-stream-head-sniffer.ts
+++ b/packages/cli/src/handlers/shared/devin-stream-head-sniffer.ts
@@ -164,10 +164,9 @@ export async function sniffDevinStreamHead(
} catch {}
}
},
- cancel: () => {
- // Downstream went away — release the upstream connection.
- void reader.cancel().catch(() => {});
- },
+ // Downstream went away — release the upstream connection, and resolve
+ // only once it is released, so the parser's cancel can wait for it.
+ cancel: () => reader.cancel().catch(() => {}),
});
return new Response(body, {
status: response.status,
diff --git a/packages/cli/src/handlers/shared/stream-head-sniffer.ts b/packages/cli/src/handlers/shared/stream-head-sniffer.ts
index b39da403..55d6612d 100644
--- a/packages/cli/src/handlers/shared/stream-head-sniffer.ts
+++ b/packages/cli/src/handlers/shared/stream-head-sniffer.ts
@@ -133,10 +133,9 @@ export async function sniffResponsesStreamHead(
} catch {}
}
},
- cancel: () => {
- // Downstream went away — release the upstream connection.
- void reader.cancel().catch(() => {});
- },
+ // Downstream went away — release the upstream connection, and resolve
+ // only once it is released, so the parser's cancel can wait for it.
+ cancel: () => reader.cancel().catch(() => {}),
});
return new Response(body, {
status: response.status,
diff --git a/packages/cli/src/handlers/shared/stream-parsers/anthropic-sse.ts b/packages/cli/src/handlers/shared/stream-parsers/anthropic-sse.ts
index 78628caf..47f08502 100644
--- a/packages/cli/src/handlers/shared/stream-parsers/anthropic-sse.ts
+++ b/packages/cli/src/handlers/shared/stream-parsers/anthropic-sse.ts
@@ -14,6 +14,7 @@ import type { BaseAPIFormat } from "../../../adapters/base-api-format.js";
import { log } from "../../../logger.js";
import { redactForClient } from "../../../redact.js";
import { sseDataPayload } from "./sse-line.js";
+import { type UpstreamReaderSlot, cancelEndsUpstreamWork } from "./upstream-cancel.js";
interface AnthropicPassthroughOpts {
modelName: string;
@@ -287,210 +288,141 @@ export function createAnthropicPassthroughStream(
}
};
+ const upstream: UpstreamReaderSlot = {};
return c.body(
- new ReadableStream({
- async start(controller) {
- const sendPing = () => {
- if (!isClosed) {
- controller.enqueue(encoder.encode('event: ping\ndata: {"type":"ping"}\n\n'));
- }
- };
-
- sendPing();
-
- pingInterval = setInterval(() => {
- if (!isClosed && Date.now() - lastActivity > 1000) {
- sendPing();
- }
- }, 1000);
-
- try {
- const reader = response.body!.getReader();
- let buffer = "";
-
- let totalLines = 0;
- let textChunks = 0;
- let toolUseBlocks = 0;
-
- // Thinking-block filtering state
- let insideThinkingBlock = false;
- /** How many thinking blocks have been suppressed so far. */
- let thinkingBlocksSuppressed = 0;
- /** A frame was just dropped — swallow its trailing blank separator too. */
- let suppressedFrame = false;
-
- // Content block index tracking — detect out-of-range indices
- // that would cause "Content block not found" on the client side.
- let highestSeenIndex = -1;
- /**
- * Original → remapped index for blocks whose content_block_start
- * index jumped (e.g. z.ai 0 → 2). Every later frame of that block
- * (deltas, stop) still carries the original index, so the remap
- * must follow the block until it closes.
- */
- const remappedBlocks = new Map<number, number>();
- const trackIndex = (idx: number) => {
- if (idx > highestSeenIndex) highestSeenIndex = idx;
+ new ReadableStream(
+ cancelEndsUpstreamWork<Uint8Array>(upstream, {
+ async start(controller) {
+ const sendPing = () => {
+ if (!isClosed) {
+ controller.enqueue(encoder.encode('event: ping\ndata: {"type":"ping"}\n\n'));
+ }
};
- /**
- * THE single index-mapping layer. Every indexed content_block frame
- * goes through here, on both the thinking-filtered and unfiltered
- * paths, because Claude Code's rule is the same either way: a delta
- * or stop may only name an index some `content_block_start` already
- * opened, and the opened indices must run 0,1,2,… with no gaps.
- *
- * Three upstream shapes it repairs:
- * - a start whose index jumps (z.ai sends 0 then 2) is remapped to
- * the next sequential slot, and the mapping is remembered so the
- * block's own deltas and stop follow it;
- * - a delta or stop naming a block that was never opened (MiniMax
- * emits an implicit signature block with no start) is DROPPED
- * whole rather than clamped, since re-attaching it to another
- * block would corrupt that block's content;
- * - a suppressed thinking block simply never calls `trackIndex`,
- * so the next real block lands on the slot the suppressed one
- * would have taken. That is why no "subtract the suppressed
- * count" arithmetic is needed: the renumbering falls out of
- * `highestSeenIndex` instead of being counted separately.
- *
- * Emitting through `enqueueData` with the REWRITTEN object matters:
- * it records lifecycle against the index the client actually saw,
- * which is what `finalizeAbandonedStream` later closes.
- */
- const emitIndexed = (controller: any, data: any, line: string): void => {
- if (typeof data.index !== "number") {
- enqueueData(controller, data, line);
- return;
+ sendPing();
+
+ pingInterval = setInterval(() => {
+ if (!isClosed && Date.now() - lastActivity > 1000) {
+ sendPing();
}
+ }, 1000);
+
+ try {
+ const reader = response.body!.getReader();
+ upstream.reader = reader;
+ let buffer = "";
+
+ let totalLines = 0;
+ let textChunks = 0;
+ let toolUseBlocks = 0;
+
+ // Thinking-block filtering state
+ let insideThinkingBlock = false;
+ /** How many thinking blocks have been suppressed so far. */
+ let thinkingBlocksSuppressed = 0;
+ /** A frame was just dropped — swallow its trailing blank separator too. */
+ let suppressedFrame = false;
+
+ // Content block index tracking — detect out-of-range indices
+ // that would cause "Content block not found" on the client side.
+ let highestSeenIndex = -1;
+ /**
+ * Original → remapped index for blocks whose content_block_start
+ * index jumped (e.g. z.ai 0 → 2). Every later frame of that block
+ * (deltas, stop) still carries the original index, so the remap
+ * must follow the block until it closes.
+ */
+ const remappedBlocks = new Map<number, number>();
+ const trackIndex = (idx: number) => {
+ if (idx > highestSeenIndex) highestSeenIndex = idx;
+ };
+
+ /**
+ * THE single index-mapping layer. Every indexed content_block frame
+ * goes through here, on both the thinking-filtered and unfiltered
+ * paths, because Claude Code's rule is the same either way: a delta
+ * or stop may only name an index some `content_block_start` already
+ * opened, and the opened indices must run 0,1,2,… with no gaps.
+ *
+ * Three upstream shapes it repairs:
+ * - a start whose index jumps (z.ai sends 0 then 2) is remapped to
+ * the next sequential slot, and the mapping is remembered so the
+ * block's own deltas and stop follow it;
+ * - a delta or stop naming a block that was never opened (MiniMax
+ * emits an implicit signature block with no start) is DROPPED
+ * whole rather than clamped, since re-attaching it to another
+ * block would corrupt that block's content;
+ * - a suppressed thinking block simply never calls `trackIndex`,
+ * so the next real block lands on the slot the suppressed one
+ * would have taken. That is why no "subtract the suppressed
+ * count" arithmetic is needed: the renumbering falls out of
+ * `highestSeenIndex` instead of being counted separately.
+ *
+ * Emitting through `enqueueData` with the REWRITTEN object matters:
+ * it records lifecycle against the index the client actually saw,
+ * which is what `finalizeAbandonedStream` later closes.
+ */
+ const emitIndexed = (controller: any, data: any, line: string): void => {
+ if (typeof data.index !== "number") {
+ enqueueData(controller, data, line);
+ return;
+ }
+
+ if (data.type === "content_block_start") {
+ const expected = highestSeenIndex + 1;
+ if (data.index !== expected) {
+ log(
+ `[AnthropicSSE] content_block_start index ${data.index} remapped to ${expected} (model=${opts.modelName})`
+ );
+ remappedBlocks.set(data.index, expected);
+ const remapped = { ...data, index: expected };
+ enqueueData(controller, remapped, `data: ${JSON.stringify(remapped)}`);
+ } else {
+ enqueueData(controller, data, line);
+ }
+ trackIndex(expected);
+ return;
+ }
- if (data.type === "content_block_start") {
- const expected = highestSeenIndex + 1;
- if (data.index !== expected) {
+ const remapped = remappedBlocks.get(data.index);
+ if (data.type === "content_block_stop") remappedBlocks.delete(data.index);
+ if (remapped !== undefined) {
+ const modified = { ...data, index: remapped };
+ enqueueData(controller, modified, `data: ${JSON.stringify(modified)}`);
+ } else if (data.index > highestSeenIndex) {
log(
- `[AnthropicSSE] content_block_start index ${data.index} remapped to ${expected} (model=${opts.modelName})`
+ `[AnthropicSSE] Dropping orphan ${data.type} at index ${data.index} (no open block — model=${opts.modelName})`
);
- remappedBlocks.set(data.index, expected);
- const remapped = { ...data, index: expected };
- enqueueData(controller, remapped, `data: ${JSON.stringify(remapped)}`);
+ pendingEventLine = null; // swallow the event: line too
+ suppressedFrame = true; // ...and the blank separator that follows it
} else {
enqueueData(controller, data, line);
}
- trackIndex(expected);
- return;
- }
-
- const remapped = remappedBlocks.get(data.index);
- if (data.type === "content_block_stop") remappedBlocks.delete(data.index);
- if (remapped !== undefined) {
- const modified = { ...data, index: remapped };
- enqueueData(controller, modified, `data: ${JSON.stringify(modified)}`);
- } else if (data.index > highestSeenIndex) {
- log(
- `[AnthropicSSE] Dropping orphan ${data.type} at index ${data.index} (no open block — model=${opts.modelName})`
- );
- pendingEventLine = null; // swallow the event: line too
- suppressedFrame = true; // ...and the blank separator that follows it
- } else {
- enqueueData(controller, data, line);
- }
- };
-
- while (true) {
- const { done, value } = await reader.read();
- if (done) break;
- buffer += decoder.decode(value, { stream: true });
- lastActivity = Date.now();
- const lines = buffer.split("\n");
- buffer = lines.pop() || "";
-
- for (const line of lines) {
- totalLines++;
-
- // Tolerates both `data: {...}` and the bare `data:{...}` form.
- const payload = sseDataPayload(line);
-
- // ── Thinking-block filtering ──────────────────────────────
- if (filterThinking && payload !== null) {
- try {
- const data = JSON.parse(payload);
-
- // ── In-stream error detection (GitHub #106) ──
- // Some anthropic-compat providers (Z.AI, MiniMax, Kimi) return
- // HTTP 200 with {"error":{...}} embedded in the SSE payload.
- // Detect and surface as a proper error event.
- if (data.error) {
- const errMsg = data.error.message || JSON.stringify(data.error);
- log(`[AnthropicSSE] In-stream error detected: ${errMsg}`);
- if (!isClosed) {
- controller.enqueue(
- encoder.encode(
- `event: error\ndata: ${JSON.stringify({
- type: "error",
- error: { type: "api_error", message: redactForClient(errMsg) },
- })}\n\n`
- )
- );
- isClosed = true;
- if (pingInterval) {
- clearInterval(pingInterval);
- pingInterval = null;
- }
- controller.close();
- }
- return; // stop processing further lines
- }
+ };
- // Track: entering a thinking block
- if (
- data.type === "content_block_start" &&
- data.content_block?.type === "thinking"
- ) {
- insideThinkingBlock = true;
- thinkingBlocksSuppressed++;
- log(`[AnthropicSSE] Filtering thinking block at index ${data.index}`);
- pendingEventLine = null;
- suppressedFrame = true;
- continue; // suppress this line
- }
+ while (true) {
+ const { done, value } = await reader.read();
+ if (done) break;
+ buffer += decoder.decode(value, { stream: true });
+ lastActivity = Date.now();
+ const lines = buffer.split("\n");
+ buffer = lines.pop() || "";
- // Track: exiting a thinking block
- if (insideThinkingBlock && data.type === "content_block_stop") {
- insideThinkingBlock = false;
- pendingEventLine = null;
- suppressedFrame = true;
- continue; // suppress this line
- }
+ for (const line of lines) {
+ totalLines++;
- // Suppress all deltas while inside a thinking block
- // (thinking_delta, signature_delta)
- if (insideThinkingBlock) {
- pendingEventLine = null;
- suppressedFrame = true;
- continue;
- }
+ // Tolerates both `data: {...}` and the bare `data:{...}` form.
+ const payload = sseDataPayload(line);
- // Renumber whatever survived the thinking filter through the
- // SAME index-mapping layer the unfiltered path uses. No
- // `continue` here: frames must fall through to the
- // usage/text tracking below, which still reads the ORIGINAL
- // data rather than the rewritten frame.
- emitIndexed(controller, data, line);
- } catch {
- // Unparseable — pass through
- if (!isClosed) {
- flushPendingEvent(controller);
- controller.enqueue(encoder.encode(`${line}\n`));
- }
- }
- } else {
- // Non-data lines (event: lines, blank lines) or no filtering
- if (!filterThinking && payload !== null) {
- // Parse data lines BEFORE enqueuing to detect in-stream errors
+ // ── Thinking-block filtering ──────────────────────────────
+ if (filterThinking && payload !== null) {
try {
const data = JSON.parse(payload);
// ── In-stream error detection (GitHub #106) ──
+ // Some anthropic-compat providers (Z.AI, MiniMax, Kimi) return
+ // HTTP 200 with {"error":{...}} embedded in the SSE payload.
+ // Detect and surface as a proper error event.
if (data.error) {
const errMsg = data.error.message || JSON.stringify(data.error);
log(`[AnthropicSSE] In-stream error detected: ${errMsg}`);
@@ -513,171 +445,246 @@ export function createAnthropicPassthroughStream(
return; // stop processing further lines
}
- // No error — check index bounds before passing through
- // (still via the tool interceptor, which is a no-op unless
- // a rule asked for this tool).
- emitIndexed(controller, data, line);
+ // Track: entering a thinking block
+ if (
+ data.type === "content_block_start" &&
+ data.content_block?.type === "thinking"
+ ) {
+ insideThinkingBlock = true;
+ thinkingBlocksSuppressed++;
+ log(`[AnthropicSSE] Filtering thinking block at index ${data.index}`);
+ pendingEventLine = null;
+ suppressedFrame = true;
+ continue; // suppress this line
+ }
- // Usage/debug tracking, in its OWN try. The outer catch
- // re-enqueues the raw line, which is right for a data line
- // that would not parse — but by this point the frame has
- // already been emitted, rewritten, or deliberately dropped.
- // Letting a tracking throw reach that catch emits it a
- // second time: a double frame, or a resurrected orphan
- // carrying the very index the layer just removed. The
- // filtered path already isolates its tracking this way.
- try {
- if (data.message?.usage) {
- inputTokens = data.message.usage.input_tokens || inputTokens;
- outputTokens = data.message.usage.output_tokens || outputTokens;
- }
- if (data.usage) {
- inputTokens = data.usage.input_tokens || inputTokens;
- outputTokens = data.usage.output_tokens || outputTokens;
- }
- if (
- data.type === "content_block_delta" &&
- data.delta?.type === "text_delta"
- ) {
- const txt = data.delta.text || "";
- opts.onAssistantText?.(txt, "text");
- textChunks++;
- log(
- `[AnthropicSSE] Text chunk: "${txt.substring(0, 30).replace(/\n/g, "\\n")}" (${txt.length} chars)`
- );
- }
- if (
- data.type === "content_block_start" &&
- data.content_block?.type === "tool_use"
- ) {
- toolUseBlocks++;
- opts.onToolCallObserved?.(data.content_block.name);
- log(`[AnthropicSSE] Tool use: ${data.content_block.name}`);
- }
- if (data.type === "message_delta" && data.delta?.stop_reason) {
- stopReason = data.delta.stop_reason;
- }
- } catch {
- // Observation must never change the wire.
+ // Track: exiting a thinking block
+ if (insideThinkingBlock && data.type === "content_block_stop") {
+ insideThinkingBlock = false;
+ pendingEventLine = null;
+ suppressedFrame = true;
+ continue; // suppress this line
}
+
+ // Suppress all deltas while inside a thinking block
+ // (thinking_delta, signature_delta)
+ if (insideThinkingBlock) {
+ pendingEventLine = null;
+ suppressedFrame = true;
+ continue;
+ }
+
+ // Renumber whatever survived the thinking filter through the
+ // SAME index-mapping layer the unfiltered path uses. No
+ // `continue` here: frames must fall through to the
+ // usage/text tracking below, which still reads the ORIGINAL
+ // data rather than the rewritten frame.
+ emitIndexed(controller, data, line);
} catch {
- // Unparseable data line — pass through
+ // Unparseable — pass through
if (!isClosed) {
+ flushPendingEvent(controller);
controller.enqueue(encoder.encode(`${line}\n`));
}
}
} else {
- // Non-data line — an `event:` header or the blank separator —
- // on EITHER path.
- //
- // The header is held for exactly one line because its `data:`
- // line may still be dropped, and a header already on the wire
- // cannot be recalled. Claude Code rejects a header with no
- // body outright: `Could not parse message into JSON: From
- // chunk: [ "event:content_block_start" ]` kills the whole turn.
- //
- // This used to be conditional on `filterThinking`, on the
- // reasoning that only the thinking filter ever dropped a
- // frame. That stopped being true when the index layer began
- // dropping orphans, which it does on BOTH paths — and the
- // unfiltered one is the common case for z.ai, Kimi and qwen,
- // i.e. exactly the providers whose jumped indices the layer
- // exists to repair. Measured before this change: an orphaned
- // delta+stop pair left two bare `event:` lines on the wire.
- if (line.startsWith("event:")) {
- pendingEventLine = line;
- } else if (line.trim() === "" && suppressedFrame) {
- suppressedFrame = false; // swallow the dropped frame's separator
- } else if (!isClosed) {
- controller.enqueue(encoder.encode(`${line}\n`));
+ // Non-data lines (event: lines, blank lines) or no filtering
+ if (!filterThinking && payload !== null) {
+ // Parse data lines BEFORE enqueuing to detect in-stream errors
+ try {
+ const data = JSON.parse(payload);
+
+ // ── In-stream error detection (GitHub #106) ──
+ if (data.error) {
+ const errMsg = data.error.message || JSON.stringify(data.error);
+ log(`[AnthropicSSE] In-stream error detected: ${errMsg}`);
+ if (!isClosed) {
+ controller.enqueue(
+ encoder.encode(
+ `event: error\ndata: ${JSON.stringify({
+ type: "error",
+ error: { type: "api_error", message: redactForClient(errMsg) },
+ })}\n\n`
+ )
+ );
+ isClosed = true;
+ if (pingInterval) {
+ clearInterval(pingInterval);
+ pingInterval = null;
+ }
+ controller.close();
+ }
+ return; // stop processing further lines
+ }
+
+ // No error — check index bounds before passing through
+ // (still via the tool interceptor, which is a no-op unless
+ // a rule asked for this tool).
+ emitIndexed(controller, data, line);
+
+ // Usage/debug tracking, in its OWN try. The outer catch
+ // re-enqueues the raw line, which is right for a data line
+ // that would not parse — but by this point the frame has
+ // already been emitted, rewritten, or deliberately dropped.
+ // Letting a tracking throw reach that catch emits it a
+ // second time: a double frame, or a resurrected orphan
+ // carrying the very index the layer just removed. The
+ // filtered path already isolates its tracking this way.
+ try {
+ if (data.message?.usage) {
+ inputTokens = data.message.usage.input_tokens || inputTokens;
+ outputTokens = data.message.usage.output_tokens || outputTokens;
+ }
+ if (data.usage) {
+ inputTokens = data.usage.input_tokens || inputTokens;
+ outputTokens = data.usage.output_tokens || outputTokens;
+ }
+ if (
+ data.type === "content_block_delta" &&
+ data.delta?.type === "text_delta"
+ ) {
+ const txt = data.delta.text || "";
+ opts.onAssistantText?.(txt, "text");
+ textChunks++;
+ log(
+ `[AnthropicSSE] Text chunk: "${txt.substring(0, 30).replace(/\n/g, "\\n")}" (${txt.length} chars)`
+ );
+ }
+ if (
+ data.type === "content_block_start" &&
+ data.content_block?.type === "tool_use"
+ ) {
+ toolUseBlocks++;
+ opts.onToolCallObserved?.(data.content_block.name);
+ log(`[AnthropicSSE] Tool use: ${data.content_block.name}`);
+ }
+ if (data.type === "message_delta" && data.delta?.stop_reason) {
+ stopReason = data.delta.stop_reason;
+ }
+ } catch {
+ // Observation must never change the wire.
+ }
+ } catch {
+ // Unparseable data line — pass through
+ if (!isClosed) {
+ controller.enqueue(encoder.encode(`${line}\n`));
+ }
+ }
+ } else {
+ // Non-data line — an `event:` header or the blank separator —
+ // on EITHER path.
+ //
+ // The header is held for exactly one line because its `data:`
+ // line may still be dropped, and a header already on the wire
+ // cannot be recalled. Claude Code rejects a header with no
+ // body outright: `Could not parse message into JSON: From
+ // chunk: [ "event:content_block_start" ]` kills the whole turn.
+ //
+ // This used to be conditional on `filterThinking`, on the
+ // reasoning that only the thinking filter ever dropped a
+ // frame. That stopped being true when the index layer began
+ // dropping orphans, which it does on BOTH paths — and the
+ // unfiltered one is the common case for z.ai, Kimi and qwen,
+ // i.e. exactly the providers whose jumped indices the layer
+ // exists to repair. Measured before this change: an orphaned
+ // delta+stop pair left two bare `event:` lines on the wire.
+ if (line.startsWith("event:")) {
+ pendingEventLine = line;
+ } else if (line.trim() === "" && suppressedFrame) {
+ suppressedFrame = false; // swallow the dropped frame's separator
+ } else if (!isClosed) {
+ controller.enqueue(encoder.encode(`${line}\n`));
+ }
}
}
- }
- // ── Usage/debug tracking for filtered path ────────────────
- // We need this even when filtering, but the data was already parsed
- // above in the filterThinking branch. Re-parse for tracking only.
- if (filterThinking && payload !== null) {
- try {
- const data = JSON.parse(payload);
- if (data.message?.usage) {
- inputTokens = data.message.usage.input_tokens || inputTokens;
- outputTokens = data.message.usage.output_tokens || outputTokens;
- }
- if (data.usage) {
- inputTokens = data.usage.input_tokens || inputTokens;
- outputTokens = data.usage.output_tokens || outputTokens;
- }
- if (data.type === "content_block_delta" && data.delta?.type === "text_delta") {
- opts.onAssistantText?.(data.delta.text || "", "text");
- textChunks++;
- }
- if (
- data.type === "content_block_start" &&
- data.content_block?.type === "tool_use"
- ) {
- toolUseBlocks++;
- opts.onToolCallObserved?.(data.content_block.name);
- log(`[AnthropicSSE] Tool use: ${data.content_block.name}`);
- }
- if (data.type === "message_delta" && data.delta?.stop_reason) {
- stopReason = data.delta.stop_reason;
- }
- } catch {}
+ // ── Usage/debug tracking for filtered path ────────────────
+ // We need this even when filtering, but the data was already parsed
+ // above in the filterThinking branch. Re-parse for tracking only.
+ if (filterThinking && payload !== null) {
+ try {
+ const data = JSON.parse(payload);
+ if (data.message?.usage) {
+ inputTokens = data.message.usage.input_tokens || inputTokens;
+ outputTokens = data.message.usage.output_tokens || outputTokens;
+ }
+ if (data.usage) {
+ inputTokens = data.usage.input_tokens || inputTokens;
+ outputTokens = data.usage.output_tokens || outputTokens;
+ }
+ if (data.type === "content_block_delta" && data.delta?.type === "text_delta") {
+ opts.onAssistantText?.(data.delta.text || "", "text");
+ textChunks++;
+ }
+ if (
+ data.type === "content_block_start" &&
+ data.content_block?.type === "tool_use"
+ ) {
+ toolUseBlocks++;
+ opts.onToolCallObserved?.(data.content_block.name);
+ log(`[AnthropicSSE] Tool use: ${data.content_block.name}`);
+ }
+ if (data.type === "message_delta" && data.delta?.stop_reason) {
+ stopReason = data.delta.stop_reason;
+ }
+ } catch {}
+ }
}
}
- }
- log(
- `[AnthropicSSE] Stream complete for ${opts.modelName}: ${totalLines} lines, ${textChunks} text chunks, ${toolUseBlocks} tool_use blocks, stop_reason=${stopReason}${filterThinking ? `, filtered ${thinkingBlocksSuppressed} thinking blocks` : ""}`
- );
+ log(
+ `[AnthropicSSE] Stream complete for ${opts.modelName}: ${totalLines} lines, ${textChunks} text chunks, ${toolUseBlocks} tool_use blocks, stop_reason=${stopReason}${filterThinking ? `, filtered ${thinkingBlocksSuppressed} thinking blocks` : ""}`
+ );
- opts.onTurnEnd?.();
+ opts.onTurnEnd?.();
- if (opts.onTokenUpdate) {
- opts.onTokenUpdate(inputTokens, outputTokens);
- }
+ if (opts.onTokenUpdate) {
+ opts.onTokenUpdate(inputTokens, outputTokens);
+ }
- if (!isClosed) {
- isClosed = true;
- if (pingInterval) {
- clearInterval(pingInterval);
- pingInterval = null;
+ if (!isClosed) {
+ isClosed = true;
+ if (pingInterval) {
+ clearInterval(pingInterval);
+ pingInterval = null;
+ }
+ controller.close();
}
- controller.close();
- }
- } catch (e) {
- log(`[AnthropicSSE] Stream error: ${e}`);
- // The upstream socket dropped mid-answer (Z.AI, MiniMax and Kimi all do
- // this under load). Closing the controller here without a terminal event
- // handed Claude Code a stream that stopped but never ended, and the turn
- // hung. Emit the tail first, then close, and still run the turn-end hooks
- // so Layer 4 and the token file see a completed turn rather than nothing.
- finalizeAbandonedStream(controller);
- try {
- opts.onTurnEnd?.();
- } catch {}
- try {
- opts.onTokenUpdate?.(inputTokens, outputTokens);
- } catch {}
- if (!isClosed) {
- isClosed = true;
- if (pingInterval) {
- clearInterval(pingInterval);
- pingInterval = null;
+ } catch (e) {
+ log(`[AnthropicSSE] Stream error: ${e}`);
+ // The upstream socket dropped mid-answer (Z.AI, MiniMax and Kimi all do
+ // this under load). Closing the controller here without a terminal event
+ // handed Claude Code a stream that stopped but never ended, and the turn
+ // hung. Emit the tail first, then close, and still run the turn-end hooks
+ // so Layer 4 and the token file see a completed turn rather than nothing.
+ finalizeAbandonedStream(controller);
+ try {
+ opts.onTurnEnd?.();
+ } catch {}
+ try {
+ opts.onTokenUpdate?.(inputTokens, outputTokens);
+ } catch {}
+ if (!isClosed) {
+ isClosed = true;
+ if (pingInterval) {
+ clearInterval(pingInterval);
+ pingInterval = null;
+ }
+ controller.close();
}
- controller.close();
}
- }
- },
- cancel() {
- isClosed = true;
- if (pingInterval) {
- clearInterval(pingInterval);
- pingInterval = null;
- }
- },
- }),
+ },
+ // Downstream went away: cancelEndsUpstreamWork cancels the upstream reader
+ // after this and resolves only once the read loop above has settled.
+ cancel() {
+ isClosed = true;
+ if (pingInterval) {
+ clearInterval(pingInterval);
+ pingInterval = null;
+ }
+ },
+ })
+ ),
{
headers: {
"Content-Type": "text/event-stream",
diff --git a/packages/cli/src/handlers/shared/stream-parsers/devin-connect.ts b/packages/cli/src/handlers/shared/stream-parsers/devin-connect.ts
index f95b2248..3081fb07 100644
--- a/packages/cli/src/handlers/shared/stream-parsers/devin-connect.ts
+++ b/packages/cli/src/handlers/shared/stream-parsers/devin-connect.ts
@@ -59,6 +59,11 @@ import {
} from "../../../providers/devin/proto-codec.js";
import { redactForClient } from "../../../redact.js";
import { messageStartUsage } from "./message-start-usage.js";
+import {
+ type ParserSource,
+ type UpstreamReaderSlot,
+ cancelEndsUpstreamWork,
+} from "./upstream-cancel.js";
/** Response field numbers. Protocol constants, not model data. */
const FIELD_TEXT = 3;
@@ -149,7 +154,8 @@ export function createDevinConnectStream(
let isClosed = false;
let pingInterval: ReturnType<typeof setInterval> | null = null;
- const stream = new ReadableStream({
+ const upstream: UpstreamReaderSlot = {};
+ const source: ParserSource<Uint8Array> = {
async start(controller) {
const send = (event: string, data: any) => {
if (!isClosed) {
@@ -481,6 +487,7 @@ export function createDevinConnectStream(
}
const reader = body.getReader();
+ upstream.reader = reader;
const nextFrames = createFrameReader();
while (true) {
@@ -520,6 +527,8 @@ export function createDevinConnectStream(
finalize("error", String(e));
}
},
+ // Downstream went away: cancelEndsUpstreamWork cancels the upstream reader
+ // after this and resolves only once the read loop above has settled.
cancel() {
isClosed = true;
if (pingInterval) {
@@ -527,7 +536,8 @@ export function createDevinConnectStream(
pingInterval = null;
}
},
- });
+ };
+ const stream = new ReadableStream(cancelEndsUpstreamWork(upstream, source));
return new Response(stream, {
headers: {
diff --git a/packages/cli/src/handlers/shared/stream-parsers/gemini-sse.ts b/packages/cli/src/handlers/shared/stream-parsers/gemini-sse.ts
index 3cc47b9b..8521063d 100644
--- a/packages/cli/src/handlers/shared/stream-parsers/gemini-sse.ts
+++ b/packages/cli/src/handlers/shared/stream-parsers/gemini-sse.ts
@@ -13,6 +13,11 @@ import type { MiddlewareManager } from "../../../middleware/manager.js";
import { redactForClient } from "../../../redact.js";
import { messageStartUsage } from "./message-start-usage.js";
import { sseDataPayload } from "./sse-line.js";
+import {
+ type ParserSource,
+ type UpstreamReaderSlot,
+ cancelEndsUpstreamWork,
+} from "./upstream-cancel.js";
export interface GeminiSseOptions {
modelName: string;
@@ -55,7 +60,8 @@ export function createGeminiSseStream(
let isClosed = false;
let pingInterval: ReturnType<typeof setInterval> | null = null;
- const stream = new ReadableStream({
+ const upstream: UpstreamReaderSlot = {};
+ const source: ParserSource<Uint8Array> = {
async start(controller) {
const send = (event: string, data: any) => {
if (!isClosed) {
@@ -191,6 +197,7 @@ export function createGeminiSseStream(
try {
const reader = response.body!.getReader();
+ upstream.reader = reader;
let buffer = "";
while (true) {
@@ -384,6 +391,8 @@ export function createGeminiSseStream(
await finalize("error", String(e));
}
},
+ // Downstream went away: cancelEndsUpstreamWork cancels the upstream reader
+ // after this and resolves only once the read loop above has settled.
cancel() {
isClosed = true;
if (pingInterval) {
@@ -391,7 +400,8 @@ export function createGeminiSseStream(
pingInterval = null;
}
},
- });
+ };
+ const stream = new ReadableStream(cancelEndsUpstreamWork(upstream, source));
return new Response(stream, {
headers: {
diff --git a/packages/cli/src/handlers/shared/stream-parsers/ollama-jsonl.ts b/packages/cli/src/handlers/shared/stream-parsers/ollama-jsonl.ts
index 08803c68..8753ae9f 100644
--- a/packages/cli/src/handlers/shared/stream-parsers/ollama-jsonl.ts
+++ b/packages/cli/src/handlers/shared/stream-parsers/ollama-jsonl.ts
@@ -13,6 +13,11 @@ import type { Context } from "hono";
import { log } from "../../../logger.js";
import { redactForClient } from "../../../redact.js";
import { messageStartUsage } from "./message-start-usage.js";
+import {
+ type ParserSource,
+ type UpstreamReaderSlot,
+ cancelEndsUpstreamWork,
+} from "./upstream-cancel.js";
export function createOllamaJsonlStream(
_c: Context,
@@ -29,7 +34,8 @@ export function createOllamaJsonlStream(
let isClosed = false;
let pingInterval: ReturnType<typeof setInterval> | null = null;
- const stream = new ReadableStream({
+ const upstream: UpstreamReaderSlot = {};
+ const source: ParserSource<Uint8Array> = {
async start(controller) {
const send = (event: string, data: any) => {
if (!isClosed) {
@@ -129,6 +135,7 @@ export function createOllamaJsonlStream(
try {
const reader = response.body!.getReader();
+ upstream.reader = reader;
let buffer = "";
while (true) {
@@ -185,6 +192,8 @@ export function createOllamaJsonlStream(
finalize("error", String(error));
}
},
+ // Downstream went away: cancelEndsUpstreamWork cancels the upstream reader
+ // after this and resolves only once the read loop above has settled.
cancel() {
isClosed = true;
if (pingInterval) {
@@ -192,7 +201,8 @@ export function createOllamaJsonlStream(
pingInterval = null;
}
},
- });
+ };
+ const stream = new ReadableStream(cancelEndsUpstreamWork(upstream, source));
return new Response(stream, {
headers: {
diff --git a/packages/cli/src/handlers/shared/stream-parsers/openai-responses-sse.ts b/packages/cli/src/handlers/shared/stream-parsers/openai-responses-sse.ts
index dd077b19..c2c6af15 100644
--- a/packages/cli/src/handlers/shared/stream-parsers/openai-responses-sse.ts
+++ b/packages/cli/src/handlers/shared/stream-parsers/openai-responses-sse.ts
@@ -20,6 +20,7 @@ import { wrapAnthropicError } from "../anthropic-error.js";
import type { UsageCacheDetail } from "../token-tracker.js";
import { ClaudeSseSink } from "./responses/claude-sse-sink.js";
import { type ResponsesRetryOptions, runResponsesTurn } from "./responses/responses-turn.js";
+import { type ParserSource, cancelEndsUpstreamWork } from "./upstream-cancel.js";
export type { ResponsesRetryOptions } from "./responses/responses-turn.js";
export type { CodeModeTurnHooks, ExecTurnStep } from "../../../code-mode/types.js";
@@ -180,7 +181,7 @@ export function createResponsesStreamHandler(
}
let sink: ClaudeSseSink | undefined;
- const stream = new ReadableStream<Uint8Array>({
+ const source: ParserSource<Uint8Array> = {
start: async (controller) => {
sink = new ClaudeSseSink(controller, {
modelName: opts.modelName,
@@ -214,10 +215,14 @@ export function createResponsesStreamHandler(
// otherwise enqueue on a closed controller and throw from a timer callback,
// killing the proxy process), and aborts the turn: the upstream reader is
// cancelled rather than left streaming to completion with nobody reading.
+ // cancelEndsUpstreamWork then waits for runResponsesTurn to settle before
+ // the cancel resolves, so the lease outlives everything the turn logs while
+ // stopping. No upstream reader is passed: the turn's legs read their own.
cancel() {
sink?.cancel();
},
- });
+ };
+ const stream = new ReadableStream<Uint8Array>(cancelEndsUpstreamWork({}, source));
return new Response(stream, {
headers: {
diff --git a/packages/cli/src/handlers/shared/stream-parsers/openai-sse.ts b/packages/cli/src/handlers/shared/stream-parsers/openai-sse.ts
index fe83edc7..1320533b 100644
--- a/packages/cli/src/handlers/shared/stream-parsers/openai-sse.ts
+++ b/packages/cli/src/handlers/shared/stream-parsers/openai-sse.ts
@@ -21,6 +21,7 @@ import { type BlockRef, createBlockWriter } from "./block-writer.js";
import { messageStartUsage } from "./message-start-usage.js";
import { sseDataPayload } from "./sse-line.js";
import { type ThinkSplit, createThinkTagSplitter } from "./think-tag-splitter.js";
+import { type UpstreamReaderSlot, cancelEndsUpstreamWork } from "./upstream-cancel.js";
import { splitPromptTokens } from "./usage-cache-split.js";
/**
@@ -305,934 +306,944 @@ export function createStreamingResponseHandler(
const encoder = new TextEncoder();
const decoder = new TextDecoder();
const streamMetadata = new Map<string, any>();
+ const upstream: UpstreamReaderSlot = {};
return c.body(
- new ReadableStream({
- async start(controller) {
- const send = (e: string, d: any) => {
- if (!isClosed) {
- // Observe every tool call HERE, at the single frame writer, rather than at
- // the `content_block_start` sites. (Those now all run through
- // `block-writer.ts`, which is itself a single site — but this hook stays
- // here, one layer lower, because it must also see anything a future path
- // emits without going through the writer.)
- //
- // This parser declared `onToolCallObserved` and never called it — so on the
- // busiest wire in claudish (GLM, Kimi, Grok, DeepSeek, Qwen, OpenRouter,
- // LiteLLM) the behaviour layer's tool-name list was always empty and the
- // session summary would have reported zero tools. Hooking the writer instead
- // of the emission sites makes that class of omission impossible: a new
- // tool_use path cannot forget to opt in, and because exactly one
- // `content_block_start` is emitted per tool call, it cannot double count
- // either. The `input_json_delta` frames deliberately do not match.
- if (e === "content_block_start" && d?.content_block?.type === "tool_use") {
- try {
- behavior?.onToolCallObserved?.(String(d.content_block.name ?? ""));
- } catch (err) {
- log(`[Streaming] onToolCallObserved threw: ${err}`);
+ new ReadableStream(
+ cancelEndsUpstreamWork<Uint8Array>(upstream, {
+ async start(controller) {
+ const send = (e: string, d: any) => {
+ if (!isClosed) {
+ // Observe every tool call HERE, at the single frame writer, rather than at
+ // the `content_block_start` sites. (Those now all run through
+ // `block-writer.ts`, which is itself a single site — but this hook stays
+ // here, one layer lower, because it must also see anything a future path
+ // emits without going through the writer.)
+ //
+ // This parser declared `onToolCallObserved` and never called it — so on the
+ // busiest wire in claudish (GLM, Kimi, Grok, DeepSeek, Qwen, OpenRouter,
+ // LiteLLM) the behaviour layer's tool-name list was always empty and the
+ // session summary would have reported zero tools. Hooking the writer instead
+ // of the emission sites makes that class of omission impossible: a new
+ // tool_use path cannot forget to opt in, and because exactly one
+ // `content_block_start` is emitted per tool call, it cannot double count
+ // either. The `input_json_delta` frames deliberately do not match.
+ if (e === "content_block_start" && d?.content_block?.type === "tool_use") {
+ try {
+ behavior?.onToolCallObserved?.(String(d.content_block.name ?? ""));
+ } catch (err) {
+ log(`[Streaming] onToolCallObserved threw: ${err}`);
+ }
}
+ controller.enqueue(encoder.encode(`event: ${e}\ndata: ${JSON.stringify(d)}\n\n`));
}
- controller.enqueue(encoder.encode(`event: ${e}\ndata: ${JSON.stringify(d)}\n\n`));
- }
- };
-
- const msgId = `msg_${Date.now()}_${Math.random().toString(36).slice(2)}`;
- const state = createStreamingState();
- // Every content block for this turn is opened, appended to and closed
- // through here. It owns the block-index counter, so nothing else
- // allocates an index.
- const writer = createBlockWriter(send);
- const thinkSplitter = createThinkTagSplitter();
-
- /**
- * Route one splitter result to its blocks.
- *
- * Shared by the streaming path and by `finalize()`'s flush, so a fragment
- * released at the end of the turn takes exactly the same hold-back
- * decision as one released mid-stream — rather than bypassing it and
- * emitting a lone fragment of text that the hold-back is withholding the
- * rest of.
- */
- const emitSplitContent = ({ thinking, text }: ThinkSplit): void => {
- if (thinking) {
- behavior?.onAssistantText?.(thinking, "reasoning");
- writer.append(writer.openThinking(), thinking);
- }
- if (!text) return;
-
- // Accumulate text for potential tool call extraction
- state.accumulatedText += text;
-
- // Check if text contains STRUCTURED tool call patterns that we should hold back
- // Only hold back for patterns we can actually parse (XML, JSON), not natural language
- // Natural language patterns are extracted at finalization, not held back
- const hasStructuredToolPattern =
- // Qwen XML-style: <function=ToolName>. Same shape the
- // extractor accepts, so text held back here is always text
- // the extractor can act on. A looser test here withheld
- // text that nothing later emitted.
- hasExtractableFunctionTag(state.accumulatedText) ||
- // JSON tool call in text: {"name": "Task", "arguments":
- /\{\s*"(?:name|tool)"\s*:\s*"(?:Task|Read|Write|Edit|Bash|Grep|Glob)"/i.test(
- state.accumulatedText
- ) ||
- // XML tool_call tags: <tool_call>
- /<tool_call>/.test(state.accumulatedText);
-
- // Only hold back if we have a structured pattern AND haven't accumulated too much
- // (if we've accumulated > 1000 chars without a complete pattern, release the text)
- const shouldHoldBack = hasStructuredToolPattern && state.accumulatedText.length < 1000;
-
- if (shouldHoldBack) {
- log(
- `[Streaming] Text held back (structured tool pattern): ${state.accumulatedText.length} chars accumulated`
- );
- return;
- }
+ };
+
+ const msgId = `msg_${Date.now()}_${Math.random().toString(36).slice(2)}`;
+ const state = createStreamingState();
+ // Every content block for this turn is opened, appended to and closed
+ // through here. It owns the block-index counter, so nothing else
+ // allocates an index.
+ const writer = createBlockWriter(send);
+ const thinkSplitter = createThinkTagSplitter();
+
+ /**
+ * Route one splitter result to its blocks.
+ *
+ * Shared by the streaming path and by `finalize()`'s flush, so a fragment
+ * released at the end of the turn takes exactly the same hold-back
+ * decision as one released mid-stream — rather than bypassing it and
+ * emitting a lone fragment of text that the hold-back is withholding the
+ * rest of.
+ */
+ const emitSplitContent = ({ thinking, text }: ThinkSplit): void => {
+ if (thinking) {
+ behavior?.onAssistantText?.(thinking, "reasoning");
+ writer.append(writer.openThinking(), thinking);
+ }
+ if (!text) return;
+
+ // Accumulate text for potential tool call extraction
+ state.accumulatedText += text;
+
+ // Check if text contains STRUCTURED tool call patterns that we should hold back
+ // Only hold back for patterns we can actually parse (XML, JSON), not natural language
+ // Natural language patterns are extracted at finalization, not held back
+ const hasStructuredToolPattern =
+ // Qwen XML-style: <function=ToolName>. Same shape the
+ // extractor accepts, so text held back here is always text
+ // the extractor can act on. A looser test here withheld
+ // text that nothing later emitted.
+ hasExtractableFunctionTag(state.accumulatedText) ||
+ // JSON tool call in text: {"name": "Task", "arguments":
+ /\{\s*"(?:name|tool)"\s*:\s*"(?:Task|Read|Write|Edit|Bash|Grep|Glob)"/i.test(
+ state.accumulatedText
+ ) ||
+ // XML tool_call tags: <tool_call>
+ /<tool_call>/.test(state.accumulatedText);
- writer.append(writer.openText(), text);
- };
-
- send("message_start", {
- type: "message_start",
- message: {
- id: msgId,
- type: "message",
- role: "assistant",
- content: [],
- model: target,
- stop_reason: null,
- stop_sequence: null,
- usage: messageStartUsage(priorInputTokens),
- },
- });
- send("ping", { type: "ping" });
-
- ping = setInterval(() => {
- if (!isClosed && Date.now() - state.lastActivity > 1000) {
- send("ping", { type: "ping" });
- }
- }, 1000);
-
- // Teardown is separated from finalize() on purpose. finalize() guards
- // re-entry on state.finalized, so once it has started, a throw part-way
- // through used to mean the stream was never closed and the ping interval
- // was never cleared: the outer catch re-called finalize(), which returned
- // immediately at the guard. The client then sat on an open HTTP 200
- // forever. Teardown therefore runs from a `finally`, and stays safe to
- // call twice.
- const teardown = () => {
- if (!isClosed) {
- try {
- controller.enqueue(encoder.encode("data: [DONE]\n\n\n"));
- } catch {}
- try {
- controller.close();
- } catch {}
- isClosed = true;
- }
- if (ping) {
- clearInterval(ping);
- ping = null;
- }
- };
-
- const finalize = async (reason: string, err?: string) => {
- // A second call still has to tear down: the first may have thrown
- // before reaching its own `finally`.
- if (state.finalized) {
- teardown();
- return;
- }
- state.finalized = true;
+ // Only hold back if we have a structured pattern AND haven't accumulated too much
+ // (if we've accumulated > 1000 chars without a complete pattern, release the text)
+ const shouldHoldBack = hasStructuredToolPattern && state.accumulatedText.length < 1000;
- try {
- // Release anything the think-tag splitter is still holding — at most
- // seven characters of a tag that never completed. Runs FIRST, before
- // text-based tool-call recovery reads `accumulatedText`, so a held
- // fragment is part of the text recovery scans rather than arriving
- // after it.
- emitSplitContent(thinkSplitter.flush());
-
- // Argument fragments whose `function.name` never arrived. A call with
- // no name is not a call — there is nothing to dispatch and no schema to
- // validate against — so they are discarded. Named in the log because
- // silently dropping them is precisely the defect this buffer fixes, and
- // a buffer that survives to here means the provider is doing something
- // the accumulator did not anticipate. "error" puts it in the always-on
- // structural log.
- for (const [idx, pending] of state.pendingToolArgs) {
+ if (shouldHoldBack) {
log(
- `[Streaming] Tool argument error: discarding ${pending.length} buffered chars for tool_calls index ${idx} — function.name never arrived`
+ `[Streaming] Text held back (structured tool pattern): ${state.accumulatedText.length} chars accumulated`
);
+ return;
}
- state.pendingToolArgs.clear();
- // Debug: Log accumulated text for analysis
- if (state.accumulatedText.length > 0) {
- const preview = state.accumulatedText.slice(0, 500).replace(/\n/g, "\\n");
- log(
- `[Streaming] Accumulated text (${state.accumulatedText.length} chars): ${preview}...`
- );
+ writer.append(writer.openText(), text);
+ };
+
+ send("message_start", {
+ type: "message_start",
+ message: {
+ id: msgId,
+ type: "message",
+ role: "assistant",
+ content: [],
+ model: target,
+ stop_reason: null,
+ stop_sequence: null,
+ usage: messageStartUsage(priorInputTokens),
+ },
+ });
+ send("ping", { type: "ping" });
+
+ ping = setInterval(() => {
+ if (!isClosed && Date.now() - state.lastActivity > 1000) {
+ send("ping", { type: "ping" });
}
-
- // Check for text-based tool calls before finalizing
- // Some models (like Qwen) output tool calls as text instead of structured tool_calls
- //
- // Only when the model produced NO structured call. Recovery exists for
- // models that cannot emit `tool_calls` at all; against a model that
- // just did, it can only ADD calls, never repair one. Ungated, a turn
- // holding one real call plus prose mentioning a function tag dispatched
- // two tool_use blocks, and both were recorded.
- const textToolCalls =
- state.tools.size > 0
- ? []
- : extractToolCallsFromText(
- state.accumulatedText,
- toolSchemas?.map((t: any) => t?.name).filter((n: any): n is string => !!n),
- // The `<function=NAME><parameter=P>` envelope carries no
- // types, so every value arrives as a string. The schemas are
- // what turn `"5"` back into 5 and `"true"` into true.
- toolSchemas as ToolSchema[] | undefined,
- // Recovery reads the name the MODEL wrote, which is the
- // encoded one. The allowlist is built from the client's
- // originals, so an undecoded name is dropped in silence.
- toolNameMap ? (name: string) => toolNameMap.get(name) ?? name : undefined
- );
- if (state.tools.size > 0 && state.accumulatedText.length > 0) {
- log(
- `[Streaming] Skipping text-based tool extraction: ${state.tools.size} structured tool call(s) already present`
- );
+ }, 1000);
+
+ // Teardown is separated from finalize() on purpose. finalize() guards
+ // re-entry on state.finalized, so once it has started, a throw part-way
+ // through used to mean the stream was never closed and the ping interval
+ // was never cleared: the outer catch re-called finalize(), which returned
+ // immediately at the guard. The client then sat on an open HTTP 200
+ // forever. Teardown therefore runs from a `finally`, and stays safe to
+ // call twice.
+ const teardown = () => {
+ if (!isClosed) {
+ try {
+ controller.enqueue(encoder.encode("data: [DONE]\n\n\n"));
+ } catch {}
+ try {
+ controller.close();
+ } catch {}
+ isClosed = true;
}
- log(`[Streaming] Text-based tool calls found: ${textToolCalls.length}`);
-
- // ── How this turn ENDED ────────────────────────────────────────────
- //
- // Classified HERE — after recovery has run but BEFORE anything it
- // found is emitted — because recovery adds both content and tools, and
- // a classification taken before it runs would read a pre-recovery
- // state. Local models, the population text recovery exists for, are
- // exactly the ones most likely to end with no finish_reason.
- //
- // `finish_reason` is the ONLY completion signal. Neither the `[DONE]`
- // sentinel nor a final usage object counts: both are transport
- // punctuation that a proxy, a load balancer or a truncated body can
- // produce without the model ever having finished.
- const producedContent =
- state.accumulatedText.length > 0 ||
- writer.anyBlockEmitted ||
- state.tools.size > 0 ||
- textToolCalls.length > 0;
- const toolInFlight =
- // Already on the wire, including text-recovered calls from earlier
- // turns of this same finalize — `state.tools` does not cover those.
- writer.emittedToolRefs.length > 0 ||
- // `&& !t.closed` is load-bearing: `buffered && !started` is TRUE for
- // a tool that already failed validation and was closed.
- Array.from(state.tools.values()).some(
- (t) => !t.closed && (t.started || t.buffered)
- ) ||
- textToolCalls.length > 0;
-
- /**
- * `success` — the provider said how it finished, or produced nothing.
- * `silent-truncation` — no finish_reason, content, but no tool: the
- * partial prose is harmless and VISIBLE, and `max_tokens` is
- * Anthropic's own "the turn was cut off" label. An `error` here would
- * discard text the user can read.
- * `failure` — the turn cannot be presented as complete. Ends with an
- * SSE `error` event, which Claude Code honours by discarding a
- * partial `tool_use` and retrying (verified against a real client,
- * reports/truncated-toolcall-live-verification.md).
- */
- const ending: "success" | "silent-truncation" | "failure" =
- reason === "error"
- ? "failure"
- : state.finishReason !== null || !producedContent
- ? "success"
- : toolInFlight
- ? "failure"
- : "silent-truncation";
-
- if (ending !== "success") {
- log(
- `[Streaming] Stream ending error: reason=${reason} finish_reason=${state.finishReason ?? "null"} content=${producedContent} tool_in_flight=${toolInFlight} → ${ending}`
- );
+ if (ping) {
+ clearInterval(ping);
+ ping = null;
+ }
+ };
+
+ const finalize = async (reason: string, err?: string) => {
+ // A second call still has to tear down: the first may have thrown
+ // before reaching its own `finally`.
+ if (state.finalized) {
+ teardown();
+ return;
}
+ state.finalized = true;
- // On a failure ending claudish emits NO NEW tool block and NO completed
- // tool call: the buffered flush below is skipped and recovered calls are
- // suppressed. It cannot recall a non-buffered tool block already on the
- // wire — `content_block_start` went out before a single argument byte
- // existed — but that partial is exactly the case the client-side
- // measurement covers.
- const emitToolCalls = ending !== "failure";
+ try {
+ // Release anything the think-tag splitter is still holding — at most
+ // seven characters of a tag that never completed. Runs FIRST, before
+ // text-based tool-call recovery reads `accumulatedText`, so a held
+ // fragment is part of the text recovery scans rather than arriving
+ // after it.
+ emitSplitContent(thinkSplitter.flush());
+
+ // Argument fragments whose `function.name` never arrived. A call with
+ // no name is not a call — there is nothing to dispatch and no schema to
+ // validate against — so they are discarded. Named in the log because
+ // silently dropping them is precisely the defect this buffer fixes, and
+ // a buffer that survives to here means the provider is doing something
+ // the accumulator did not anticipate. "error" puts it in the always-on
+ // structural log.
+ for (const [idx, pending] of state.pendingToolArgs) {
+ log(
+ `[Streaming] Tool argument error: discarding ${pending.length} buffered chars for tool_calls index ${idx} — function.name never arrived`
+ );
+ }
+ state.pendingToolArgs.clear();
- if (textToolCalls.length > 0 && !emitToolCalls) {
- log(
- `[Streaming] Suppressing ${textToolCalls.length} text-recovered tool call(s): the turn ended in failure`
- );
- } else if (textToolCalls.length > 0) {
- log(
- `[Streaming] Found ${textToolCalls.length} text-based tool call(s), converting to structured format`
- );
+ // Debug: Log accumulated text for analysis
+ if (state.accumulatedText.length > 0) {
+ const preview = state.accumulatedText.slice(0, 500).replace(/\n/g, "\\n");
+ log(
+ `[Streaming] Accumulated text (${state.accumulatedText.length} chars): ${preview}...`
+ );
+ }
- // Send each extracted tool call as a proper tool_use block.
- // `openTool` closes whatever is open first, which is where the
- // hand-written "close any open text block" used to live.
- for (const tc of textToolCalls) {
- const toolIdx = writer.reserve();
- const toolId = `tool_${Date.now()}_${toolIdx}`;
- const ref = writer.openTool({ id: toolId, name: tc.name, index: toolIdx });
- writer.append(ref, repairArgs(tc.name, JSON.stringify(tc.arguments)));
- writer.close(ref);
+ // Check for text-based tool calls before finalizing
+ // Some models (like Qwen) output tool calls as text instead of structured tool_calls
+ //
+ // Only when the model produced NO structured call. Recovery exists for
+ // models that cannot emit `tool_calls` at all; against a model that
+ // just did, it can only ADD calls, never repair one. Ungated, a turn
+ // holding one real call plus prose mentioning a function tag dispatched
+ // two tool_use blocks, and both were recorded.
+ const textToolCalls =
+ state.tools.size > 0
+ ? []
+ : extractToolCallsFromText(
+ state.accumulatedText,
+ toolSchemas?.map((t: any) => t?.name).filter((n: any): n is string => !!n),
+ // The `<function=NAME><parameter=P>` envelope carries no
+ // types, so every value arrives as a string. The schemas are
+ // what turn `"5"` back into 5 and `"true"` into true.
+ toolSchemas as ToolSchema[] | undefined,
+ // Recovery reads the name the MODEL wrote, which is the
+ // encoded one. The allowlist is built from the client's
+ // originals, so an undecoded name is dropped in silence.
+ toolNameMap ? (name: string) => toolNameMap.get(name) ?? name : undefined
+ );
+ if (state.tools.size > 0 && state.accumulatedText.length > 0) {
+ log(
+ `[Streaming] Skipping text-based tool extraction: ${state.tools.size} structured tool call(s) already present`
+ );
}
- }
+ log(`[Streaming] Text-based tool calls found: ${textToolCalls.length}`);
- // Whatever is still open — thinking, text, or a tool — closes here.
- // This replaces the hand-written reasoning-then-text pair, which could
- // only ever close the two kinds it named.
- writer.closeCurrent();
-
- // Handle buffered-but-unsent structured tool calls.
- // Some models (e.g., Gemini via LiteLLM) send tool calls with finish_reason="stop"
- // instead of "tool_calls", so the normal validation path (line ~695) is never reached.
- // We must send these buffered tools here so Claude Code can execute them.
- //
- // SKIPPED ENTIRELY on a failure ending. Before this gate a failed turn
- // shipped a COMPLETE, parseable tool call and then an `error` event, so
- // the protection rested wholly on the client discarding it. Now only an
- // incomplete partial can reach the client on a failure ending, which is
- // precisely the case the client-side measurement covers.
- for (const t of emitToolCalls ? Array.from(state.tools.values()) : []) {
- if (!t.closed && t.buffered && !t.started) {
- if (toolSchemas && toolSchemas.length > 0) {
- const validation = validateToolArguments(
- t.name,
- t.arguments,
- toolSchemas,
- state.accumulatedText
- );
+ // ── How this turn ENDED ────────────────────────────────────────────
+ //
+ // Classified HERE — after recovery has run but BEFORE anything it
+ // found is emitted — because recovery adds both content and tools, and
+ // a classification taken before it runs would read a pre-recovery
+ // state. Local models, the population text recovery exists for, are
+ // exactly the ones most likely to end with no finish_reason.
+ //
+ // `finish_reason` is the ONLY completion signal. Neither the `[DONE]`
+ // sentinel nor a final usage object counts: both are transport
+ // punctuation that a proxy, a load balancer or a truncated body can
+ // produce without the model ever having finished.
+ const producedContent =
+ state.accumulatedText.length > 0 ||
+ writer.anyBlockEmitted ||
+ state.tools.size > 0 ||
+ textToolCalls.length > 0;
+ const toolInFlight =
+ // Already on the wire, including text-recovered calls from earlier
+ // turns of this same finalize — `state.tools` does not cover those.
+ writer.emittedToolRefs.length > 0 ||
+ // `&& !t.closed` is load-bearing: `buffered && !started` is TRUE for
+ // a tool that already failed validation and was closed.
+ Array.from(state.tools.values()).some(
+ (t) => !t.closed && (t.started || t.buffered)
+ ) ||
+ textToolCalls.length > 0;
+
+ /**
+ * `success` — the provider said how it finished, or produced nothing.
+ * `silent-truncation` — no finish_reason, content, but no tool: the
+ * partial prose is harmless and VISIBLE, and `max_tokens` is
+ * Anthropic's own "the turn was cut off" label. An `error` here would
+ * discard text the user can read.
+ * `failure` — the turn cannot be presented as complete. Ends with an
+ * SSE `error` event, which Claude Code honours by discarding a
+ * partial `tool_use` and retrying (verified against a real client,
+ * reports/truncated-toolcall-live-verification.md).
+ */
+ const ending: "success" | "silent-truncation" | "failure" =
+ reason === "error"
+ ? "failure"
+ : state.finishReason !== null || !producedContent
+ ? "success"
+ : toolInFlight
+ ? "failure"
+ : "silent-truncation";
+
+ if (ending !== "success") {
+ log(
+ `[Streaming] Stream ending error: reason=${reason} finish_reason=${state.finishReason ?? "null"} content=${producedContent} tool_in_flight=${toolInFlight} → ${ending}`
+ );
+ }
+
+ // On a failure ending claudish emits NO NEW tool block and NO completed
+ // tool call: the buffered flush below is skipped and recovered calls are
+ // suppressed. It cannot recall a non-buffered tool block already on the
+ // wire — `content_block_start` went out before a single argument byte
+ // existed — but that partial is exactly the case the client-side
+ // measurement covers.
+ const emitToolCalls = ending !== "failure";
- if (validation.valid || (validation.repaired && validation.repairedArgs)) {
- const argsJson = repairArgs(
+ if (textToolCalls.length > 0 && !emitToolCalls) {
+ log(
+ `[Streaming] Suppressing ${textToolCalls.length} text-recovered tool call(s): the turn ended in failure`
+ );
+ } else if (textToolCalls.length > 0) {
+ log(
+ `[Streaming] Found ${textToolCalls.length} text-based tool call(s), converting to structured format`
+ );
+
+ // Send each extracted tool call as a proper tool_use block.
+ // `openTool` closes whatever is open first, which is where the
+ // hand-written "close any open text block" used to live.
+ for (const tc of textToolCalls) {
+ const toolIdx = writer.reserve();
+ const toolId = `tool_${Date.now()}_${toolIdx}`;
+ const ref = writer.openTool({ id: toolId, name: tc.name, index: toolIdx });
+ writer.append(ref, repairArgs(tc.name, JSON.stringify(tc.arguments)));
+ writer.close(ref);
+ }
+ }
+
+ // Whatever is still open — thinking, text, or a tool — closes here.
+ // This replaces the hand-written reasoning-then-text pair, which could
+ // only ever close the two kinds it named.
+ writer.closeCurrent();
+
+ // Handle buffered-but-unsent structured tool calls.
+ // Some models (e.g., Gemini via LiteLLM) send tool calls with finish_reason="stop"
+ // instead of "tool_calls", so the normal validation path (line ~695) is never reached.
+ // We must send these buffered tools here so Claude Code can execute them.
+ //
+ // SKIPPED ENTIRELY on a failure ending. Before this gate a failed turn
+ // shipped a COMPLETE, parseable tool call and then an `error` event, so
+ // the protection rested wholly on the client discarding it. Now only an
+ // incomplete partial can reach the client on a failure ending, which is
+ // precisely the case the client-side measurement covers.
+ for (const t of emitToolCalls ? Array.from(state.tools.values()) : []) {
+ if (!t.closed && t.buffered && !t.started) {
+ if (toolSchemas && toolSchemas.length > 0) {
+ const validation = validateToolArguments(
t.name,
- JSON.stringify(
- validation.repaired ? validation.repairedArgs : validation.parsedArgs
- )
+ t.arguments,
+ toolSchemas,
+ state.accumulatedText
);
+
+ if (validation.valid || (validation.repaired && validation.repairedArgs)) {
+ const argsJson = repairArgs(
+ t.name,
+ JSON.stringify(
+ validation.repaired ? validation.repairedArgs : validation.parsedArgs
+ )
+ );
+ log(
+ `[Streaming] Sending buffered tool call (finish_reason!=tool_calls): ${t.name} with args: ${argsJson}`
+ );
+ t.ref = writer.openTool({ id: t.id, name: t.name, index: t.blockIndex });
+ writer.append(t.ref, argsJson);
+ writer.close(t.ref);
+ t.started = true;
+ t.closed = true;
+ } else {
+ log(
+ `[Streaming] Buffered tool call ${t.name} failed validation, skipping: ${validation.missingParams.join(", ")}`
+ );
+ t.closed = true;
+ }
+ } else {
+ // No schemas to validate against — send as-is
+ const argsJson = repairArgs(t.name, t.arguments || "{}");
log(
- `[Streaming] Sending buffered tool call (finish_reason!=tool_calls): ${t.name} with args: ${argsJson}`
+ `[Streaming] Sending buffered tool call (no validation): ${t.name} with args: ${argsJson}`
);
t.ref = writer.openTool({ id: t.id, name: t.name, index: t.blockIndex });
writer.append(t.ref, argsJson);
writer.close(t.ref);
t.started = true;
t.closed = true;
- } else {
- log(
- `[Streaming] Buffered tool call ${t.name} failed validation, skipping: ${validation.missingParams.join(", ")}`
- );
- t.closed = true;
}
- } else {
- // No schemas to validate against — send as-is
- const argsJson = repairArgs(t.name, t.arguments || "{}");
- log(
- `[Streaming] Sending buffered tool call (no validation): ${t.name} with args: ${argsJson}`
- );
- t.ref = writer.openTool({ id: t.id, name: t.name, index: t.blockIndex });
- writer.append(t.ref, argsJson);
- writer.close(t.ref);
- t.started = true;
- t.closed = true;
}
}
- }
-
- // Close any remaining started-but-unclosed tool calls. Under the
- // one-open-block invariant at most one of these is still open; the
- // rest were closed when the block that superseded them opened, and
- // `writer.close` is a no-op for those.
- for (const t of Array.from(state.tools.values())) {
- if (t.started && !t.closed) {
- if (t.ref) writer.close(t.ref);
- t.closed = true;
- }
- }
- if (middlewareManager) {
- await middlewareManager.afterStreamComplete(target, streamMetadata);
- }
-
- if (ending === "failure") {
- // An SSE `error` event is the ONE ending Claude Code honours by
- // discarding a partial `tool_use` and retrying the turn. `end_turn`
- // means "the turn finished, run the tool" and made the client execute
- // truncated JSON; `max_tokens` was assumed to rescue it and measurably
- // does not (adapters.md, and the 2026-09-10 live verification).
- const message =
- err ??
- "Upstream stream ended with no finish_reason while a tool call was in flight. " +
- "The tool call is incomplete and was not dispatched.";
- send("error", {
- type: "error",
- error: { type: "api_error", message: redactForClient(message) },
- });
- } else {
- // Set stop_reason based on whether we sent ANY tool calls (text-based or structured)
- const hasStructuredTools = Array.from(state.tools.values()).some((t) => t.started);
- // A turn the PROVIDER cut off must not be reported as a turn the model
- // chose to end. Anthropic's contract for a cut-off turn is "max_tokens";
- // reporting "end_turn" presents a truncated (or, when reasoning consumed
- // the whole budget, an EMPTY) answer as the model's complete final word.
- // Mirrors openai-responses-sse.ts, which already does this.
- // `content_filter` is the same class: the provider refused, which is
- // Anthropic's "refusal", not a turn the model chose to end.
- // openai-responses-sse.ts already maps both this way.
- // `length` still OUTRANKS `tool_use` — a truncated turn is reported as
- // truncated even when it carries a tool call. A silent truncation (no
- // finish_reason, content produced, no tool in flight) takes the same
- // label: it IS a cut-off turn, and `max_tokens` is Anthropic's word
- // for one. Neither can collide with the other: `finishReason` is
- // non-null in the first case and null in the second.
- const truncated = state.finishReason === "length" || ending === "silent-truncation";
- const refused = state.finishReason === "content_filter";
- const stopReason = refused
- ? "refusal"
- : truncated
- ? "max_tokens"
- : textToolCalls.length > 0 || hasStructuredTools
- ? "tool_use"
- : "end_turn";
- if (truncated || refused) {
- log(
- `[Streaming] Upstream finish_reason=${state.finishReason} → stop_reason=${stopReason} (${state.accumulatedText.length} chars produced)`
- );
+ // Close any remaining started-but-unclosed tool calls. Under the
+ // one-open-block invariant at most one of these is still open; the
+ // rest were closed when the block that superseded them opened, and
+ // `writer.close` is a no-op for those.
+ for (const t of Array.from(state.tools.values())) {
+ if (t.started && !t.closed) {
+ if (t.ref) writer.close(t.ref);
+ t.closed = true;
+ }
}
- // A turn that ended NORMALLY must carry at least one content block.
- // `stop_reason: "end_turn"` with an empty `content` array is not a
- // shape Anthropic's API produces, and a client that indexes the last
- // block, or renders the turn, has nothing to work with — the turn
- // reads as a success that delivered nothing, with no diagnostic.
- //
- // The test is `stop_reason === "end_turn"` rather than "the ending was
- // a success", because that single value already excludes every case
- // where emptiness is MEANINGFUL and must be preserved:
- // • "max_tokens" — reasoning consumed the whole budget and the turn
- // was cut off. `gemini-3.1-pro-or-maxtokens-empty.sse` is a real
- // capture of exactly this; it must not be papered over.
- // • "max_tokens" from a silent truncation (item 4) — same reasoning.
- // • "refusal" — the provider refused. Emptiness IS the answer.
- // • "tool_use" — a tool block was emitted, so it is not contentless.
- //
- // An EMPTY text block, not placeholder prose and not an error. Prose
- // would enter the conversation history as the assistant's words and be
- // replayed forever; an `error` would trigger the client's retry loop on
- // a deterministic outcome, which is adapters.md's stated reason for
- // keeping `max_tokens` over an error on the sibling case.
- if (stopReason === "end_turn" && !writer.anyBlockEmitted) {
- log(
- `[Streaming] Contentless turn error: end_turn with no content block emitted (finish_reason=${state.finishReason ?? "null"}) — emitting an empty text block`
- );
- writer.close(writer.openText());
+ if (middlewareManager) {
+ await middlewareManager.afterStreamComplete(target, streamMetadata);
}
- // The three input counters are derived ONCE and used for both the
- // wire and the cost update below, so the two cannot disagree.
- const split = splitPromptTokens(state.usage);
- // One degenerate turn is sent UNSPLIT, and this is not a special
- // case so much as the merge rule read honestly.
- //
- // Claude Code only lets a delta value override its running total
- // when that value is GREATER THAN ZERO (2.1.273:
- // `n.input_tokens !== null && n.input_tokens > 0 ? n.input_tokens : e.input_tokens`).
- // So on a turn whose input is entirely cache — possible only when
- // the request repeats one already cached, i.e. a retry — an
- // `input_tokens: 0` is DISCARDED and the message_start seed (the
- // PREVIOUS turn's full context) survives beside a full-size
- // `cache_read_input_tokens`. The client would then sum the two and
- // believe the conversation is roughly twice its real size.
- //
- // Reporting that turn as ordinary input keeps the client's sum
- // exactly equal to `prompt_tokens`, which is the invariant that
- // matters. It costs nothing on the money side: the cost split is
- // taken from `splitPromptTokens` independently, below.
- const fullyCached = split.promptTokens > 0 && split.inputTokens === 0;
- const wireInput = fullyCached ? split.promptTokens : split.inputTokens;
- const wireCacheRead = fullyCached ? 0 : split.cacheReadTokens;
- const wireCacheCreation = fullyCached ? 0 : split.cacheCreationTokens;
- send("message_delta", {
- type: "message_delta",
- delta: { stop_reason: stopReason, stop_sequence: null },
- // input_tokens must ride the delta too: Claude Code takes the
- // context size from the last assistant message, and message_start
- // could only carry an estimate. Omitting it left the client
- // believing every conversation was 100 tokens, which silently
- // disabled auto-compaction on every openai-sse provider.
- //
- // All THREE keys ship TOGETHER, unconditionally, and that is not
- // stylistic. Claude Code reconstructs the conversation size by
- // summing them (verified in the 2.1.273 binary:
- // `input_tokens + cache_creation_input_tokens + cache_read_input_tokens`,
- // read by both the context meter and the auto-compaction
- // threshold), so a reduced `input_tokens` sent WITHOUT its two
- // siblings understates the context by exactly the cached portion
- // — on the measured xAI turn, 20352 of 20379 tokens. That is the
- // one shape that reproduces the failure the paragraph above
- // records. `splitPromptTokens` guarantees the three sum back to
- // `prompt_tokens`.
- //
- // `input_tokens` is emitted even when it is 0 (a fully-cached
- // turn), because omitting it is the same silent-omission bug in a
- // different disguise.
- usage: {
- input_tokens: wireInput,
- cache_read_input_tokens: wireCacheRead,
- cache_creation_input_tokens: wireCacheCreation,
- output_tokens: state.usage?.completion_tokens || 0,
- },
- });
- behavior?.onTurnEnd?.();
- send("message_stop", { type: "message_stop" });
- }
-
- // Update token counts - use actual usage if available, otherwise estimate
- if (onTokenUpdate) {
- if (state.usage) {
- log(
- `[Streaming] Final usage: prompt=${state.usage.prompt_tokens || 0}, completion=${state.usage.completion_tokens || 0}`
- );
- // FIRST ARGUMENT STAYS `prompt_tokens`: the tracker's input number
- // is the context-occupancy figure the status line renders and the
- // billing baseline the delta strategy compares against. The split
- // goes in the third argument, where only the cost arithmetic sees
- // it.
- const costSplit = splitPromptTokens(state.usage);
- onTokenUpdate(state.usage.prompt_tokens || 0, state.usage.completion_tokens || 0, {
- cacheReadTokens: costSplit.cacheReadTokens,
- cacheCreationTokens: costSplit.cacheCreationTokens,
+ if (ending === "failure") {
+ // An SSE `error` event is the ONE ending Claude Code honours by
+ // discarding a partial `tool_use` and retrying the turn. `end_turn`
+ // means "the turn finished, run the tool" and made the client execute
+ // truncated JSON; `max_tokens` was assumed to rescue it and measurably
+ // does not (adapters.md, and the 2026-09-10 live verification).
+ const message =
+ err ??
+ "Upstream stream ended with no finish_reason while a tool call was in flight. " +
+ "The tool call is incomplete and was not dispatched.";
+ send("error", {
+ type: "error",
+ error: { type: "api_error", message: redactForClient(message) },
});
} else {
- // Estimate tokens for local models that don't return usage data
- // Rough estimate: ~4 characters per token
- const estimatedOutputTokens = Math.ceil(state.accumulatedText.length / 4);
- log(
- `[Streaming] No usage data from provider, estimating: ~${estimatedOutputTokens} output tokens`
- );
- // Carry the previous context size forward rather than a literal
- // 100 — the status line reads this value, and 100 would make the
- // bar collapse to "empty" on any turn the provider skips usage.
- onTokenUpdate(priorInputTokens || 100, estimatedOutputTokens);
- }
- }
- } finally {
- teardown();
- }
- };
-
- try {
- const reader = response.body!.getReader();
- let buffer = "";
-
- while (true) {
- const { done, value } = await reader.read();
- if (done) break;
- buffer += decoder.decode(value, { stream: true });
- const lines = buffer.split("\n");
- buffer = lines.pop() || "";
-
- for (const line of lines) {
- const dataStr = line.trim() ? sseDataPayload(line) : null;
- if (dataStr === null) continue;
- // Verbatim: this line IS the fixture source (see SSE_LOG_MAX_CHARS).
- log(`[SSE:openai] ${formatRawSseLogPayload(dataStr)}`);
- if (dataStr === "[DONE]") {
- await finalize("done");
- return;
- }
-
- try {
- const chunk = JSON.parse(dataStr);
- if (chunk.usage) {
- state.usage = chunk.usage;
+ // Set stop_reason based on whether we sent ANY tool calls (text-based or structured)
+ const hasStructuredTools = Array.from(state.tools.values()).some((t) => t.started);
+ // A turn the PROVIDER cut off must not be reported as a turn the model
+ // chose to end. Anthropic's contract for a cut-off turn is "max_tokens";
+ // reporting "end_turn" presents a truncated (or, when reasoning consumed
+ // the whole budget, an EMPTY) answer as the model's complete final word.
+ // Mirrors openai-responses-sse.ts, which already does this.
+ // `content_filter` is the same class: the provider refused, which is
+ // Anthropic's "refusal", not a turn the model chose to end.
+ // openai-responses-sse.ts already maps both this way.
+ // `length` still OUTRANKS `tool_use` — a truncated turn is reported as
+ // truncated even when it carries a tool call. A silent truncation (no
+ // finish_reason, content produced, no tool in flight) takes the same
+ // label: it IS a cut-off turn, and `max_tokens` is Anthropic's word
+ // for one. Neither can collide with the other: `finishReason` is
+ // non-null in the first case and null in the second.
+ const truncated = state.finishReason === "length" || ending === "silent-truncation";
+ const refused = state.finishReason === "content_filter";
+ const stopReason = refused
+ ? "refusal"
+ : truncated
+ ? "max_tokens"
+ : textToolCalls.length > 0 || hasStructuredTools
+ ? "tool_use"
+ : "end_turn";
+ if (truncated || refused) {
log(
- `[Streaming] Usage data received: prompt=${chunk.usage.prompt_tokens}, completion=${chunk.usage.completion_tokens}, total=${chunk.usage.total_tokens}`
+ `[Streaming] Upstream finish_reason=${state.finishReason} → stop_reason=${stopReason} (${state.accumulatedText.length} chars produced)`
);
}
- // An error carried inside the 200 body. Checked AFTER `usage`
- // so the tokens the turn already spent are still reported, and
- // BEFORE the `choices[0]` reads below, which an error frame's
- // empty `choices` array silently fails to match.
+ // A turn that ended NORMALLY must carry at least one content block.
+ // `stop_reason: "end_turn"` with an empty `content` array is not a
+ // shape Anthropic's API produces, and a client that indexes the last
+ // block, or renders the turn, has nothing to work with — the turn
+ // reads as a success that delivered nothing, with no diagnostic.
//
- // `finalize("error", …)` emits an SSE `error` event, which is
- // the only thing that makes Claude Code surface the failure. The
- // alternative — turning it into an assistant text block — is the
- // mistake `stream-head-sniffer.ts` documents: it freezes a
- // provider failure into the transcript as a successful answer.
- const inStreamError = describeInStreamError(chunk);
- if (inStreamError) {
- log(`[Streaming] Upstream error inside a 200 stream: ${inStreamError}`);
- await finalize("error", inStreamError);
- return;
+ // The test is `stop_reason === "end_turn"` rather than "the ending was
+ // a success", because that single value already excludes every case
+ // where emptiness is MEANINGFUL and must be preserved:
+ // • "max_tokens" — reasoning consumed the whole budget and the turn
+ // was cut off. `gemini-3.1-pro-or-maxtokens-empty.sse` is a real
+ // capture of exactly this; it must not be papered over.
+ // • "max_tokens" from a silent truncation (item 4) — same reasoning.
+ // • "refusal" — the provider refused. Emptiness IS the answer.
+ // • "tool_use" — a tool block was emitted, so it is not contentless.
+ //
+ // An EMPTY text block, not placeholder prose and not an error. Prose
+ // would enter the conversation history as the assistant's words and be
+ // replayed forever; an `error` would trigger the client's retry loop on
+ // a deterministic outcome, which is adapters.md's stated reason for
+ // keeping `max_tokens` over an error on the sibling case.
+ if (stopReason === "end_turn" && !writer.anyBlockEmitted) {
+ log(
+ `[Streaming] Contentless turn error: end_turn with no content block emitted (finish_reason=${state.finishReason ?? "null"}) — emitting an empty text block`
+ );
+ writer.close(writer.openText());
}
- const delta = chunk.choices?.[0]?.delta;
- const finishReason = chunk.choices?.[0]?.finish_reason;
- if (finishReason) state.finishReason = finishReason;
+ // The three input counters are derived ONCE and used for both the
+ // wire and the cost update below, so the two cannot disagree.
+ const split = splitPromptTokens(state.usage);
+ // One degenerate turn is sent UNSPLIT, and this is not a special
+ // case so much as the merge rule read honestly.
+ //
+ // Claude Code only lets a delta value override its running total
+ // when that value is GREATER THAN ZERO (2.1.273:
+ // `n.input_tokens !== null && n.input_tokens > 0 ? n.input_tokens : e.input_tokens`).
+ // So on a turn whose input is entirely cache — possible only when
+ // the request repeats one already cached, i.e. a retry — an
+ // `input_tokens: 0` is DISCARDED and the message_start seed (the
+ // PREVIOUS turn's full context) survives beside a full-size
+ // `cache_read_input_tokens`. The client would then sum the two and
+ // believe the conversation is roughly twice its real size.
+ //
+ // Reporting that turn as ordinary input keeps the client's sum
+ // exactly equal to `prompt_tokens`, which is the invariant that
+ // matters. It costs nothing on the money side: the cost split is
+ // taken from `splitPromptTokens` independently, below.
+ const fullyCached = split.promptTokens > 0 && split.inputTokens === 0;
+ const wireInput = fullyCached ? split.promptTokens : split.inputTokens;
+ const wireCacheRead = fullyCached ? 0 : split.cacheReadTokens;
+ const wireCacheCreation = fullyCached ? 0 : split.cacheCreationTokens;
+ send("message_delta", {
+ type: "message_delta",
+ delta: { stop_reason: stopReason, stop_sequence: null },
+ // input_tokens must ride the delta too: Claude Code takes the
+ // context size from the last assistant message, and message_start
+ // could only carry an estimate. Omitting it left the client
+ // believing every conversation was 100 tokens, which silently
+ // disabled auto-compaction on every openai-sse provider.
+ //
+ // All THREE keys ship TOGETHER, unconditionally, and that is not
+ // stylistic. Claude Code reconstructs the conversation size by
+ // summing them (verified in the 2.1.273 binary:
+ // `input_tokens + cache_creation_input_tokens + cache_read_input_tokens`,
+ // read by both the context meter and the auto-compaction
+ // threshold), so a reduced `input_tokens` sent WITHOUT its two
+ // siblings understates the context by exactly the cached portion
+ // — on the measured xAI turn, 20352 of 20379 tokens. That is the
+ // one shape that reproduces the failure the paragraph above
+ // records. `splitPromptTokens` guarantees the three sum back to
+ // `prompt_tokens`.
+ //
+ // `input_tokens` is emitted even when it is 0 (a fully-cached
+ // turn), because omitting it is the same silent-omission bug in a
+ // different disguise.
+ usage: {
+ input_tokens: wireInput,
+ cache_read_input_tokens: wireCacheRead,
+ cache_creation_input_tokens: wireCacheCreation,
+ output_tokens: state.usage?.completion_tokens || 0,
+ },
+ });
+ behavior?.onTurnEnd?.();
+ send("message_stop", { type: "message_stop" });
+ }
- // Debug: Log chunk details for troubleshooting early termination
- if (delta?.content || finishReason) {
+ // Update token counts - use actual usage if available, otherwise estimate
+ if (onTokenUpdate) {
+ if (state.usage) {
log(
- `[Streaming] Chunk: content=${delta?.content?.length || 0} chars, finish_reason=${finishReason || "null"}`
+ `[Streaming] Final usage: prompt=${state.usage.prompt_tokens || 0}, completion=${state.usage.completion_tokens || 0}`
);
+ // FIRST ARGUMENT STAYS `prompt_tokens`: the tracker's input number
+ // is the context-occupancy figure the status line renders and the
+ // billing baseline the delta strategy compares against. The split
+ // goes in the third argument, where only the cost arithmetic sees
+ // it.
+ const costSplit = splitPromptTokens(state.usage);
+ onTokenUpdate(
+ state.usage.prompt_tokens || 0,
+ state.usage.completion_tokens || 0,
+ {
+ cacheReadTokens: costSplit.cacheReadTokens,
+ cacheCreationTokens: costSplit.cacheCreationTokens,
+ }
+ );
+ } else {
+ // Estimate tokens for local models that don't return usage data
+ // Rough estimate: ~4 characters per token
+ const estimatedOutputTokens = Math.ceil(state.accumulatedText.length / 4);
+ log(
+ `[Streaming] No usage data from provider, estimating: ~${estimatedOutputTokens} output tokens`
+ );
+ // Carry the previous context size forward rather than a literal
+ // 100 — the status line reads this value, and 100 would make the
+ // bar collapse to "empty" on any turn the provider skips usage.
+ onTokenUpdate(priorInputTokens || 100, estimatedOutputTokens);
+ }
+ }
+ } finally {
+ teardown();
+ }
+ };
+
+ try {
+ const reader = response.body!.getReader();
+ upstream.reader = reader;
+ let buffer = "";
+
+ while (true) {
+ const { done, value } = await reader.read();
+ if (done) break;
+ buffer += decoder.decode(value, { stream: true });
+ const lines = buffer.split("\n");
+ buffer = lines.pop() || "";
+
+ for (const line of lines) {
+ const dataStr = line.trim() ? sseDataPayload(line) : null;
+ if (dataStr === null) continue;
+ // Verbatim: this line IS the fixture source (see SSE_LOG_MAX_CHARS).
+ log(`[SSE:openai] ${formatRawSseLogPayload(dataStr)}`);
+ if (dataStr === "[DONE]") {
+ await finalize("done");
+ return;
}
- if (delta) {
- if (middlewareManager) {
- await middlewareManager.afterStreamChunk({
- modelId: target,
- chunk,
- delta,
- metadata: streamMetadata,
- });
+ try {
+ const chunk = JSON.parse(dataStr);
+ if (chunk.usage) {
+ state.usage = chunk.usage;
+ log(
+ `[Streaming] Usage data received: prompt=${chunk.usage.prompt_tokens}, completion=${chunk.usage.completion_tokens}, total=${chunk.usage.total_tokens}`
+ );
}
- // Reasoning arrives under two different field names on this one
- // wire format: `reasoning_content` (Kimi, DeepSeek via LiteLLM)
- // and `reasoning` (OpenRouter). Reading only the former silently
- // dropped every OpenRouter thinking model's reasoning.
- const reasoningText = delta.reasoning_content || delta.reasoning;
- if (reasoningText) {
- // This provider has a reasoning FIELD, so a `<think>` in its
- // content is a model writing about tags, not speaking in them.
- // Close-tag handling stays armed: these are exactly the
- // providers whose chat template opens `<think>` server-side and
- // leaks the bare close into the content.
- thinkSplitter.disarmOpen();
- behavior?.onAssistantText?.(reasoningText, "reasoning");
- state.lastActivity = Date.now();
- // Reasoning arriving AFTER text used to open a thinking block
- // while the text block was still open — two open blocks, which
- // Anthropic's wire does not allow. `openThinking` closes the
- // text block first. That difference is the point of this change.
- writer.append(writer.openThinking(), reasoningText);
+ // An error carried inside the 200 body. Checked AFTER `usage`
+ // so the tokens the turn already spent are still reported, and
+ // BEFORE the `choices[0]` reads below, which an error frame's
+ // empty `choices` array silently fails to match.
+ //
+ // `finalize("error", …)` emits an SSE `error` event, which is
+ // the only thing that makes Claude Code surface the failure. The
+ // alternative — turning it into an assistant text block — is the
+ // mistake `stream-head-sniffer.ts` documents: it freezes a
+ // provider failure into the transcript as a successful answer.
+ const inStreamError = describeInStreamError(chunk);
+ if (inStreamError) {
+ log(`[Streaming] Upstream error inside a 200 stream: ${inStreamError}`);
+ await finalize("error", inStreamError);
+ return;
}
- // Handle text content
- const txt = delta.content || "";
- if (txt) behavior?.onAssistantText?.(txt, "text");
- log(
- `[Streaming] Text chunk: "${txt.substring(0, 30).replace(/\n/g, "\\n")}" (${txt.length} chars)`
- );
- if (txt) {
- state.lastActivity = Date.now();
- // The thinking block is NOT closed here any more: it closes
- // when `writer.openText()` actually runs below. A chunk that
- // the adapter empties, or that is held back pending a tool
- // pattern, no longer ends the thinking block on the strength
- // of text that never reaches the client.
- const res = adapter.processTextContent(txt, "");
+ const delta = chunk.choices?.[0]?.delta;
+ const finishReason = chunk.choices?.[0]?.finish_reason;
+ if (finishReason) state.finishReason = finishReason;
+
+ // Debug: Log chunk details for troubleshooting early termination
+ if (delta?.content || finishReason) {
log(
- `[Streaming] After adapter: "${res.cleanedText.substring(0, 30).replace(/\n/g, "\\n")}" (${res.cleanedText.length} chars, transformed=${res.wasTransformed})`
+ `[Streaming] Chunk: content=${delta?.content?.length || 0} chars, finish_reason=${finishReason || "null"}`
);
+ }
- // An adapter emptying a chunk is LEGITIMATE on this path, so
- // there is deliberately no "non-empty in, non-empty out" guard
- // here — unlike gemini-sse.ts, which has one. Two adapters
- // return "" by design:
- // • QwenModelDialect — the chunk was entirely chat-template
- // special tokens (`<|im_start|>` &c.); passing the original
- // through would leak them to the user.
- // • GrokModelDialect — it is buffering a `<xai:function_call>`
- // XML block split across chunks; passing the original
- // through would emit half a tool call as visible text.
- // Per-chunk emptiness is therefore not evidence of loss here.
- // What WOULD be a bug is a whole turn arriving empty; that is
- // caught at the turn level by the probe and by stop_reason.
- if (txt.length > 0 && res.cleanedText.length === 0) {
- log(`[Streaming] Text filtered out by adapter: "${txt.substring(0, 50)}"`);
+ if (delta) {
+ if (middlewareManager) {
+ await middlewareManager.afterStreamChunk({
+ modelId: target,
+ chunk,
+ delta,
+ metadata: streamMetadata,
+ });
}
- if (res.cleanedText) {
- // `<think>…</think>` carried in the ORDINARY content field is
- // reasoning, not the answer. Split it out here, after the
- // adapter, so the thinking half never enters
- // `accumulatedText` (tool-call recovery scans that) and never
- // renders as the assistant's words.
- emitSplitContent(thinkSplitter.push(res.cleanedText));
+ // Reasoning arrives under two different field names on this one
+ // wire format: `reasoning_content` (Kimi, DeepSeek via LiteLLM)
+ // and `reasoning` (OpenRouter). Reading only the former silently
+ // dropped every OpenRouter thinking model's reasoning.
+ const reasoningText = delta.reasoning_content || delta.reasoning;
+ if (reasoningText) {
+ // This provider has a reasoning FIELD, so a `<think>` in its
+ // content is a model writing about tags, not speaking in them.
+ // Close-tag handling stays armed: these are exactly the
+ // providers whose chat template opens `<think>` server-side and
+ // leaks the bare close into the content.
+ thinkSplitter.disarmOpen();
+ behavior?.onAssistantText?.(reasoningText, "reasoning");
+ state.lastActivity = Date.now();
+ // Reasoning arriving AFTER text used to open a thinking block
+ // while the text block was still open — two open blocks, which
+ // Anthropic's wire does not allow. `openThinking` closes the
+ // text block first. That difference is the point of this change.
+ writer.append(writer.openThinking(), reasoningText);
}
- }
- // Handle tool calls
- if (delta.tool_calls) {
+ // Handle text content
+ const txt = delta.content || "";
+ if (txt) behavior?.onAssistantText?.(txt, "text");
log(
- `[Streaming] Received ${delta.tool_calls.length} structured tool call(s) from model`
+ `[Streaming] Text chunk: "${txt.substring(0, 30).replace(/\n/g, "\\n")}" (${txt.length} chars)`
);
- for (const tc of delta.tool_calls) {
- const idx = tc.index;
- let t = state.tools.get(idx);
- if (tc.function?.name) {
- // Accumulate the name BEFORE anything reads it: a provider
- // may split `function.name` across chunks, and this is the
- // one place a complete name exists.
- const accumulatedName =
- (state.pendingToolName.get(idx) ?? "") + tc.function.name;
- state.pendingToolName.set(idx, accumulatedName);
- // THIS IS THE DECODE POINT: it reads the accumulated name,
- // never a single chunk's fragment. Decoding a fragment
- // misses the map, and the allowlist then drops the call
- // in silence.
- const restoredName = toolNameMap?.get(accumulatedName) || accumulatedName;
- if (!t) {
- // The hand-written "close thinking, then close text"
- // pair that used to stand here is `openTool`'s job now.
- t = {
- id: tc.id || `tool_${Date.now()}_${idx}`,
- name: restoredName,
- // Reserved, not opened: a buffered tool keeps its place
- // in the index order and emits at finish_reason time.
- blockIndex: writer.reserve(),
- started: false,
- closed: false,
- // Seeded, not empty: fragments that arrived for this
- // index before the name did are drained in here.
- arguments: state.pendingToolArgs.get(idx) ?? "",
- ref: null,
- // Buffer if we have schemas to validate, OR if a behavior
- // rule wants to rewrite this call — repair is only
- // possible while the arguments are still withheld.
- buffered:
- (!!toolSchemas && toolSchemas.length > 0) ||
- behavior?.shouldBufferTool?.(restoredName) === true,
- };
- if (t.arguments) {
- log(
- `[Streaming] tool ${t.name} (index ${idx}): seeded ${t.arguments.length} argument chars that arrived before function.name`
- );
- }
- state.pendingToolArgs.delete(idx);
- state.tools.set(idx, t);
- if (isWebSearchToolCall(restoredName)) {
- warnWebSearchUnsupported(restoredName, target);
- }
- } else if (t.name !== restoredName) {
- // A LATER fragment completed the name. The tool was
- // created from the first fragment, so its name is a
- // prefix — and a prefix of an encoded name decodes to
- // nothing, which is how a call gets dropped without a
- // word anywhere.
- if (t.started) {
- // The block is already on the wire under the short
- // name; it cannot be recalled. "error" is deliberate
- // — it is what carries this to the structural log.
- log(
- `[Streaming] error: tool block ${t.blockIndex} was started as "${t.name}" but the full name is "${restoredName}" — the client sees the wrong name`
- );
- } else {
- t.name = restoredName;
- t.buffered =
- (!!toolSchemas && toolSchemas.length > 0) ||
- behavior?.shouldBufferTool?.(restoredName) === true;
+ if (txt) {
+ state.lastActivity = Date.now();
+ // The thinking block is NOT closed here any more: it closes
+ // when `writer.openText()` actually runs below. A chunk that
+ // the adapter empties, or that is held back pending a tool
+ // pattern, no longer ends the thinking block on the strength
+ // of text that never reaches the client.
+ const res = adapter.processTextContent(txt, "");
+ log(
+ `[Streaming] After adapter: "${res.cleanedText.substring(0, 30).replace(/\n/g, "\\n")}" (${res.cleanedText.length} chars, transformed=${res.wasTransformed})`
+ );
+
+ // An adapter emptying a chunk is LEGITIMATE on this path, so
+ // there is deliberately no "non-empty in, non-empty out" guard
+ // here — unlike gemini-sse.ts, which has one. Two adapters
+ // return "" by design:
+ // • QwenModelDialect — the chunk was entirely chat-template
+ // special tokens (`<|im_start|>` &c.); passing the original
+ // through would leak them to the user.
+ // • GrokModelDialect — it is buffering a `<xai:function_call>`
+ // XML block split across chunks; passing the original
+ // through would emit half a tool call as visible text.
+ // Per-chunk emptiness is therefore not evidence of loss here.
+ // What WOULD be a bug is a whole turn arriving empty; that is
+ // caught at the turn level by the probe and by stop_reason.
+ if (txt.length > 0 && res.cleanedText.length === 0) {
+ log(`[Streaming] Text filtered out by adapter: "${txt.substring(0, 50)}"`);
+ }
+
+ if (res.cleanedText) {
+ // `<think>…</think>` carried in the ORDINARY content field is
+ // reasoning, not the answer. Split it out here, after the
+ // adapter, so the thinking half never enters
+ // `accumulatedText` (tool-call recovery scans that) and never
+ // renders as the assistant's words.
+ emitSplitContent(thinkSplitter.push(res.cleanedText));
+ }
+ }
+
+ // Handle tool calls
+ if (delta.tool_calls) {
+ log(
+ `[Streaming] Received ${delta.tool_calls.length} structured tool call(s) from model`
+ );
+ for (const tc of delta.tool_calls) {
+ const idx = tc.index;
+ let t = state.tools.get(idx);
+ if (tc.function?.name) {
+ // Accumulate the name BEFORE anything reads it: a provider
+ // may split `function.name` across chunks, and this is the
+ // one place a complete name exists.
+ const accumulatedName =
+ (state.pendingToolName.get(idx) ?? "") + tc.function.name;
+ state.pendingToolName.set(idx, accumulatedName);
+ // THIS IS THE DECODE POINT: it reads the accumulated name,
+ // never a single chunk's fragment. Decoding a fragment
+ // misses the map, and the allowlist then drops the call
+ // in silence.
+ const restoredName = toolNameMap?.get(accumulatedName) || accumulatedName;
+ if (!t) {
+ // The hand-written "close thinking, then close text"
+ // pair that used to stand here is `openTool`'s job now.
+ t = {
+ id: tc.id || `tool_${Date.now()}_${idx}`,
+ name: restoredName,
+ // Reserved, not opened: a buffered tool keeps its place
+ // in the index order and emits at finish_reason time.
+ blockIndex: writer.reserve(),
+ started: false,
+ closed: false,
+ // Seeded, not empty: fragments that arrived for this
+ // index before the name did are drained in here.
+ arguments: state.pendingToolArgs.get(idx) ?? "",
+ ref: null,
+ // Buffer if we have schemas to validate, OR if a behavior
+ // rule wants to rewrite this call — repair is only
+ // possible while the arguments are still withheld.
+ buffered:
+ (!!toolSchemas && toolSchemas.length > 0) ||
+ behavior?.shouldBufferTool?.(restoredName) === true,
+ };
+ if (t.arguments) {
+ log(
+ `[Streaming] tool ${t.name} (index ${idx}): seeded ${t.arguments.length} argument chars that arrived before function.name`
+ );
+ }
+ state.pendingToolArgs.delete(idx);
+ state.tools.set(idx, t);
if (isWebSearchToolCall(restoredName)) {
warnWebSearchUnsupported(restoredName, target);
}
+ } else if (t.name !== restoredName) {
+ // A LATER fragment completed the name. The tool was
+ // created from the first fragment, so its name is a
+ // prefix — and a prefix of an encoded name decodes to
+ // nothing, which is how a call gets dropped without a
+ // word anywhere.
+ if (t.started) {
+ // The block is already on the wire under the short
+ // name; it cannot be recalled. "error" is deliberate
+ // — it is what carries this to the structural log.
+ log(
+ `[Streaming] error: tool block ${t.blockIndex} was started as "${t.name}" but the full name is "${restoredName}" — the client sees the wrong name`
+ );
+ } else {
+ t.name = restoredName;
+ t.buffered =
+ (!!toolSchemas && toolSchemas.length > 0) ||
+ behavior?.shouldBufferTool?.(restoredName) === true;
+ if (isWebSearchToolCall(restoredName)) {
+ warnWebSearchUnsupported(restoredName, target);
+ }
+ }
+ }
+ // Only send content_block_start immediately if NOT buffering
+ if (!t.started && !t.buffered) {
+ t.ref = writer.openTool({
+ id: t.id,
+ name: t.name,
+ index: t.blockIndex,
+ });
+ t.started = true;
+ // Flush the seed as ONE delta, right after the start.
+ // Skipped when the seed is empty, which is every capture
+ // in the tree — so the common case stays byte-identical.
+ if (t.arguments) writer.append(t.ref, t.arguments);
}
}
- // Only send content_block_start immediately if NOT buffering
- if (!t.started && !t.buffered) {
- t.ref = writer.openTool({
- id: t.id,
- name: t.name,
- index: t.blockIndex,
- });
- t.started = true;
- // Flush the seed as ONE delta, right after the start.
- // Skipped when the seed is empty, which is every capture
- // in the tree — so the common case stays byte-identical.
- if (t.arguments) writer.append(t.ref, t.arguments);
+ if (tc.function?.arguments && !t) {
+ // Arguments before the name. This used to be dropped by an
+ // `&& t` guard, so the head of the JSON object vanished and
+ // what survived began mid-object and could not parse —
+ // indistinguishable downstream from a model emitting bad
+ // JSON. Hold it until the name creates the tool.
+ state.pendingToolArgs.set(
+ idx,
+ (state.pendingToolArgs.get(idx) ?? "") + tc.function.arguments
+ );
}
- }
- if (tc.function?.arguments && !t) {
- // Arguments before the name. This used to be dropped by an
- // `&& t` guard, so the head of the JSON object vanished and
- // what survived began mid-object and could not parse —
- // indistinguishable downstream from a model emitting bad
- // JSON. Hold it until the name creates the tool.
- state.pendingToolArgs.set(
- idx,
- (state.pendingToolArgs.get(idx) ?? "") + tc.function.arguments
- );
- }
- if (tc.function?.arguments && t) {
- // Always accumulate arguments
- t.arguments += tc.function.arguments;
- // Only stream immediately if NOT buffering
- if (!t.buffered) {
- if (!t.ref || !writer.append(t.ref, tc.function.arguments)) {
- // OpenAI's wire lets `tool_calls[0]` and `tool_calls[1]`
- // fragments interleave; Anthropic's allows one open
- // block. So this tool's block is no longer the open one
- // — something else (another tool, or text) took over and
- // closed it. Degrade THIS tool to the buffered path: the
- // complete arguments go out as one block when the call
- // closes. Same recovery shape the repair path already
- // uses when it supersedes a partially-streamed block.
- log(
- `[Streaming] tool ${t.name} (index ${idx}) lost its open block mid-arguments — buffering the rest`
- );
- t.buffered = true;
- t.started = false;
- t.ref = null;
+ if (tc.function?.arguments && t) {
+ // Always accumulate arguments
+ t.arguments += tc.function.arguments;
+ // Only stream immediately if NOT buffering
+ if (!t.buffered) {
+ if (!t.ref || !writer.append(t.ref, tc.function.arguments)) {
+ // OpenAI's wire lets `tool_calls[0]` and `tool_calls[1]`
+ // fragments interleave; Anthropic's allows one open
+ // block. So this tool's block is no longer the open one
+ // — something else (another tool, or text) took over and
+ // closed it. Degrade THIS tool to the buffered path: the
+ // complete arguments go out as one block when the call
+ // closes. Same recovery shape the repair path already
+ // uses when it supersedes a partially-streamed block.
+ log(
+ `[Streaming] tool ${t.name} (index ${idx}) lost its open block mid-arguments — buffering the rest`
+ );
+ t.buffered = true;
+ t.started = false;
+ t.ref = null;
+ }
}
}
}
}
}
- }
- if (chunk.choices?.[0]?.finish_reason === "tool_calls") {
- for (const t of Array.from(state.tools.values())) {
- if (!t.closed) {
- // Validate and potentially repair tool arguments
- if (toolSchemas && toolSchemas.length > 0) {
- const validation = validateToolArguments(
- t.name,
- t.arguments,
- toolSchemas,
- state.accumulatedText
- );
-
- if (validation.repaired && validation.repairedArgs) {
- // Tool call was repaired - send the complete repaired arguments
- log(
- `[Streaming] Tool call ${t.name} was repaired with inferred parameters`
- );
- const repairedJson = repairArgs(
+ if (chunk.choices?.[0]?.finish_reason === "tool_calls") {
+ for (const t of Array.from(state.tools.values())) {
+ if (!t.closed) {
+ // Validate and potentially repair tool arguments
+ if (toolSchemas && toolSchemas.length > 0) {
+ const validation = validateToolArguments(
t.name,
- JSON.stringify(validation.repairedArgs)
- );
- log(
- `[Streaming] Sending repaired tool call: ${t.name} with args: ${repairedJson}`
+ t.arguments,
+ toolSchemas,
+ state.accumulatedText
);
- // If buffered, this is the first time we're sending this tool call
- // Send the complete repaired tool call as a single block
+ if (validation.repaired && validation.repairedArgs) {
+ // Tool call was repaired - send the complete repaired arguments
+ log(
+ `[Streaming] Tool call ${t.name} was repaired with inferred parameters`
+ );
+ const repairedJson = repairArgs(
+ t.name,
+ JSON.stringify(validation.repairedArgs)
+ );
+ log(
+ `[Streaming] Sending repaired tool call: ${t.name} with args: ${repairedJson}`
+ );
+
+ // If buffered, this is the first time we're sending this tool call
+ // Send the complete repaired tool call as a single block
+ if (t.buffered && !t.started) {
+ t.ref = writer.openTool({
+ id: t.id,
+ name: t.name,
+ index: t.blockIndex,
+ });
+ writer.append(t.ref, repairedJson);
+ writer.close(t.ref);
+ t.started = true;
+ t.closed = true;
+ continue;
+ }
+
+ // If already started (non-buffered), close old and send new.
+ // This is the one path that mints a SECOND block for a tool
+ // that already has one — the partially-streamed original is
+ // closed and superseded, under a new id.
+ if (t.started) {
+ if (t.ref) writer.close(t.ref);
+ const repairedIdx = writer.reserve();
+ const repairedId = `tool_repaired_${Date.now()}_${repairedIdx}`;
+ const repairedRef = writer.openTool({
+ id: repairedId,
+ name: t.name,
+ index: repairedIdx,
+ });
+ writer.append(repairedRef, repairedJson);
+ writer.close(repairedRef);
+ t.ref = repairedRef;
+ t.closed = true;
+ continue;
+ }
+ }
+
+ if (!validation.valid) {
+ // Repair failed - send error message instead of invalid tool call
+ log(
+ `[Streaming] Tool call ${t.name} validation failed: ${validation.missingParams.join(", ")}`
+ );
+ // A buffered tool never emitted its block, so its reserved
+ // index is free and the warning text takes it. A
+ // non-buffered one already spent its index on the tool
+ // block, so the warning needs a fresh one.
+ const errorIdx = t.buffered ? t.blockIndex : undefined;
+ const errorMsg = `\n\n⚠️ Tool call "${t.name}" failed: missing required parameters: ${validation.missingParams.join(", ")}. Local models sometimes generate incomplete tool calls. Please try again or use a model with better tool support.`;
+ const errorRef = writer.openText({ index: errorIdx });
+ writer.append(errorRef, errorMsg);
+ writer.close(errorRef);
+ // Close the invalid tool if it was already started.
+ // `openText` above will already have closed it when it was
+ // the open block; this covers the case where it was not.
+ if (t.started && !t.buffered && t.ref) {
+ writer.close(t.ref);
+ }
+ t.closed = true;
+ continue;
+ }
+
+ // Valid tool call - send if buffered, close if not
if (t.buffered && !t.started) {
+ const argsJson = repairArgs(
+ t.name,
+ JSON.stringify(validation.parsedArgs)
+ );
t.ref = writer.openTool({
id: t.id,
name: t.name,
index: t.blockIndex,
});
- writer.append(t.ref, repairedJson);
+ writer.append(t.ref, argsJson);
writer.close(t.ref);
t.started = true;
t.closed = true;
continue;
}
-
- // If already started (non-buffered), close old and send new.
- // This is the one path that mints a SECOND block for a tool
- // that already has one — the partially-streamed original is
- // closed and superseded, under a new id.
- if (t.started) {
- if (t.ref) writer.close(t.ref);
- const repairedIdx = writer.reserve();
- const repairedId = `tool_repaired_${Date.now()}_${repairedIdx}`;
- const repairedRef = writer.openTool({
- id: repairedId,
- name: t.name,
- index: repairedIdx,
- });
- writer.append(repairedRef, repairedJson);
- writer.close(repairedRef);
- t.ref = repairedRef;
- t.closed = true;
- continue;
- }
- }
-
- if (!validation.valid) {
- // Repair failed - send error message instead of invalid tool call
- log(
- `[Streaming] Tool call ${t.name} validation failed: ${validation.missingParams.join(", ")}`
- );
- // A buffered tool never emitted its block, so its reserved
- // index is free and the warning text takes it. A
- // non-buffered one already spent its index on the tool
- // block, so the warning needs a fresh one.
- const errorIdx = t.buffered ? t.blockIndex : undefined;
- const errorMsg = `\n\n⚠️ Tool call "${t.name}" failed: missing required parameters: ${validation.missingParams.join(", ")}. Local models sometimes generate incomplete tool calls. Please try again or use a model with better tool support.`;
- const errorRef = writer.openText({ index: errorIdx });
- writer.append(errorRef, errorMsg);
- writer.close(errorRef);
- // Close the invalid tool if it was already started.
- // `openText` above will already have closed it when it was
- // the open block; this covers the case where it was not.
- if (t.started && !t.buffered && t.ref) {
- writer.close(t.ref);
- }
- t.closed = true;
- continue;
}
- // Valid tool call - send if buffered, close if not
- if (t.buffered && !t.started) {
- const argsJson = repairArgs(
- t.name,
- JSON.stringify(validation.parsedArgs)
- );
- t.ref = writer.openTool({
- id: t.id,
- name: t.name,
- index: t.blockIndex,
- });
- writer.append(t.ref, argsJson);
- writer.close(t.ref);
- t.started = true;
+ // Non-buffered valid tool call or no validation - just close
+ if (t.started && !t.closed) {
+ if (t.ref) writer.close(t.ref);
t.closed = true;
- continue;
}
}
-
- // Non-buffered valid tool call or no validation - just close
- if (t.started && !t.closed) {
- if (t.ref) writer.close(t.ref);
- t.closed = true;
- }
}
}
+ } catch (e) {
+ // NEVER swallow silently. Everything a chunk would have emitted —
+ // a content block, a tool call, the finish_reason — is lost here,
+ // and the turn still ends HTTP 200, so the only symptom is a
+ // missing block several layers away. The bare `catch {}` this
+ // replaces made every such fault undiagnosable from the log.
+ //
+ // "error" in the text is load-bearing: `isStructuralLogWorthy`
+ // (logger.ts) matches on it, so this reaches the always-on
+ // structural log, not just a `--debug` run. The payload itself
+ // was already logged verbatim above as `[SSE:openai]`, so only a
+ // short locator is repeated here.
+ log(
+ `[Streaming] Chunk processing error (chunk dropped): ${e} — payload starts: ${dataStr.slice(0, 120)}`
+ );
}
- } catch (e) {
- // NEVER swallow silently. Everything a chunk would have emitted —
- // a content block, a tool call, the finish_reason — is lost here,
- // and the turn still ends HTTP 200, so the only symptom is a
- // missing block several layers away. The bare `catch {}` this
- // replaces made every such fault undiagnosable from the log.
- //
- // "error" in the text is load-bearing: `isStructuralLogWorthy`
- // (logger.ts) matches on it, so this reaches the always-on
- // structural log, not just a `--debug` run. The payload itself
- // was already logged verbatim above as `[SSE:openai]`, so only a
- // short locator is repeated here.
- log(
- `[Streaming] Chunk processing error (chunk dropped): ${e} — payload starts: ${dataStr.slice(0, 120)}`
- );
}
}
+ await finalize("unexpected");
+ } catch (e) {
+ await finalize("error", String(e));
}
- await finalize("unexpected");
- } catch (e) {
- await finalize("error", String(e));
- }
- },
- cancel() {
- isClosed = true;
- if (ping) clearInterval(ping);
- },
- }),
+ },
+ // Downstream went away: cancelEndsUpstreamWork cancels the upstream reader
+ // after this and resolves only once the read loop above has settled.
+ cancel() {
+ isClosed = true;
+ if (ping) clearInterval(ping);
+ },
+ })
+ ),
{
headers: {
"Content-Type": "text/event-stream",
diff --git a/packages/cli/src/handlers/shared/stream-parsers/upstream-cancel.ts b/packages/cli/src/handlers/shared/stream-parsers/upstream-cancel.ts
new file mode 100644
index 00000000..0fe6e1a5
--- /dev/null
+++ b/packages/cli/src/handlers/shared/stream-parsers/upstream-cancel.ts
@@ -0,0 +1,65 @@
+/**
+ * A downstream cancel that ends the upstream work, and resolves only once it has.
+ *
+ * Every parser here runs its whole upstream read loop inside `start()` and
+ * used to answer `cancel()` by setting a flag: the loop never looked at it, so
+ * it kept reading and logging frames after the consumer left, until the
+ * provider finished generating. The session app's signing lease is released
+ * when `cancel()` resolves (`releaseWhenBodyEnds`, `call-context.ts`), so a
+ * frame that echoed the signing credential after that point reached the log
+ * with its registry entry already gone (A2: a lease lives as long as the
+ * protected work).
+ *
+ * `cancelEndsUpstreamWork` wraps a parser's underlying source so that its
+ * `cancel()`:
+ * 1. runs the parser's own cancel first (flags, ping), so nothing more is
+ * emitted downstream;
+ * 2. cancels the upstream reader the loop is reading, so its pending
+ * `read()` resolves `done` and the provider connection is dropped;
+ * 3. waits until the `start()` task itself has settled, including whatever
+ * the loop logs and records while it winds down.
+ * A ReadableStream's `cancel()` promise resolves after the source's cancel
+ * promise, so a caller that awaits it (the lease middleware, the advisor tee)
+ * now waits for the upstream work to end.
+ */
+
+/** Where the parse loop publishes the upstream reader it reads. */
+export interface UpstreamReaderSlot {
+ reader?: ReadableStreamDefaultReader<Uint8Array>;
+}
+
+/** A parser source: an async `start` that runs the whole loop, and an optional flag-setting `cancel`. */
+export interface ParserSource<T> {
+ start(controller: ReadableStreamDefaultController<T>): Promise<void>;
+ cancel?(reason?: unknown): void | Promise<void>;
+}
+
+export function cancelEndsUpstreamWork<T>(
+ upstream: UpstreamReaderSlot,
+ source: ParserSource<T>
+): UnderlyingDefaultSource<T> {
+ let task: Promise<void> | undefined;
+ return {
+ start(controller) {
+ task = source.start(controller);
+ return task;
+ },
+ async cancel(reason) {
+ try {
+ await source.cancel?.(reason);
+ } catch {
+ // The parser's own cancel failing must not stop the upstream cancel.
+ }
+ try {
+ await upstream.reader?.cancel(reason);
+ } catch {
+ // Already released or errored: the loop has stopped reading either way.
+ }
+ try {
+ await task;
+ } catch {
+ // The loop's failure is its own to report; the cancel only waits for it.
+ }
+ },
+ };
+}