Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

fix(ai): keep silent reasoning streams connected (#6540)

Ezra Elletted06c2fdf66c2
diff --git a/packages/ai-gateway/src/test/stream-usage-tracker.unit.test.ts b/packages/ai-gateway/src/test/stream-usage-tracker.unit.test.ts
index 7e98c70f8..0cbd47583 100644
--- a/packages/ai-gateway/src/test/stream-usage-tracker.unit.test.ts
+++ b/packages/ai-gateway/src/test/stream-usage-tracker.unit.test.ts
@@ -91,6 +91,36 @@ describe('trackStreamUsage — anthropic format', () => {
});
describe('trackStreamUsage — openai format', () => {
+ it('keeps a silent thinking stream connected without fabricating output', async () => {
+ const finalEvent = `data: ${JSON.stringify({
+ choices: [],
+ usage: { prompt_tokens: 44, completion_tokens: 7 },
+ })}\n\ndata: [DONE]\n\n`;
+ const source = new ReadableStream<Uint8Array>({
+ start(controller) {
+ setTimeout(() => {
+ controller.enqueue(new TextEncoder().encode(finalEvent));
+ controller.close();
+ }, 35);
+ },
+ });
+ const { response, usage } = trackResponseUsage(
+ new Response(source),
+ 'openai',
+ 10,
+ );
+
+ const out = await drain(response.body!);
+ expect(out).toContain(': keep-alive\n\n');
+ expect(out.endsWith(finalEvent)).toBe(true);
+ expect(await usage).toMatchObject({
+ input_tokens: 44,
+ output_tokens: 7,
+ usage_complete: true,
+ termination: 'complete',
+ });
+ });
+
it('captures cached_tokens from prompt_tokens_details and custom cache_creation field', async () => {
const events = [
`data: ${JSON.stringify({ choices: [{ delta: { content: 'hello' } }] })}\n\n`,
diff --git a/packages/ai-gateway/src/utils/stream-usage-tracker.ts b/packages/ai-gateway/src/utils/stream-usage-tracker.ts
index 879103ec1..7d04754d0 100644
--- a/packages/ai-gateway/src/utils/stream-usage-tracker.ts
+++ b/packages/ai-gateway/src/utils/stream-usage-tracker.ts
@@ -20,6 +20,9 @@ export interface StreamUsage {
termination: 'complete' | 'cancelled' | 'error';
}
+const SSE_KEEP_ALIVE_INTERVAL_MS = 15_000;
+const SSE_KEEP_ALIVE = new TextEncoder().encode(': keep-alive\n\n');
+
/**
* Wrap a streaming Response body to extract token usage from SSE events.
* Data passes through unchanged to the client; usage is captured via callback.
@@ -33,6 +36,7 @@ export function trackStreamUsage(
body: ReadableStream<Uint8Array>,
format: 'anthropic' | 'openai',
onComplete: (usage: StreamUsage) => void,
+ keepAliveIntervalMs = SSE_KEEP_ALIVE_INTERVAL_MS,
): ReadableStream<Uint8Array> {
let buffer = '';
let inputTokens = 0;
@@ -43,6 +47,7 @@ export function trackStreamUsage(
const decoder = new TextDecoder();
const reader = body.getReader();
let completed = false;
+ let pendingRead: Promise<ReadableStreamReadResult<Uint8Array>> | null = null;
const completeOnce = (termination: StreamUsage['termination']) => {
if (completed) return;
@@ -107,7 +112,36 @@ export function trackStreamUsage(
return new ReadableStream<Uint8Array>({
async pull(controller) {
try {
- const { done, value } = await reader.read();
+ pendingRead ??= reader.read();
+ let timer: ReturnType<typeof setTimeout> | undefined;
+ let next: { type: 'heartbeat' } | {
+ type: 'read';
+ result: ReadableStreamReadResult<Uint8Array>;
+ };
+ try {
+ next = await Promise.race([
+ pendingRead.then((result) => ({ type: 'read' as const, result })),
+ new Promise<{ type: 'heartbeat' }>((resolve) => {
+ timer = setTimeout(
+ () => resolve({ type: 'heartbeat' }),
+ keepAliveIntervalMs,
+ );
+ }),
+ ]);
+ } finally {
+ if (timer !== undefined) clearTimeout(timer);
+ }
+
+ if (next.type === 'heartbeat') {
+ // SSE comments are protocol-valid and ignored by clients. Sending
+ // one during provider silence prevents idle network hops from
+ // closing a healthy high-reasoning request before its first token.
+ controller.enqueue(SSE_KEEP_ALIVE);
+ return;
+ }
+
+ pendingRead = null;
+ const { done, value } = next.result;
if (done) {
// Providers occasionally omit the trailing newline. Flush the final
// decoder bytes and buffered SSE line before settling the request.
@@ -144,6 +178,7 @@ export function trackStreamUsage(
export function trackResponseUsage(
response: Response,
format: 'anthropic' | 'openai',
+ keepAliveIntervalMs = SSE_KEEP_ALIVE_INTERVAL_MS,
): { response: Response; usage: Promise<StreamUsage> } {
if (!response.body) {
return {
@@ -162,7 +197,12 @@ export function trackResponseUsage(
let resolveUsage!: (u: StreamUsage) => void;
const usage = new Promise<StreamUsage>(r => resolveUsage = r);
- const trackedBody = trackStreamUsage(response.body, format, u => resolveUsage(u));
+ const trackedBody = trackStreamUsage(
+ response.body,
+ format,
+ u => resolveUsage(u),
+ keepAliveIntervalMs,
+ );
return {
response: new Response(trackedBody, {