Skip to content

7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.

TypeScript

Guard streaming chunks that carry no choices array (#156)

Tashfeena36dfd7b5ecb
diff --git a/server/src/__tests__/routes/proxy-stream-chunk.test.ts b/server/src/__tests__/routes/proxy-stream-chunk.test.ts
new file mode 100644
index 00000000..4fdbff12
--- /dev/null
+++ b/server/src/__tests__/routes/proxy-stream-chunk.test.ts
@@ -0,0 +1,32 @@
+import { describe, it, expect } from 'vitest';
+import { streamChunkText } from '../../routes/proxy.js';
+
+// Regression for the mid-stream crash: a streaming chunk with no `choices`
+// array (Groq and others emit usage/keepalive frames like this) used to throw
+// "Cannot read properties of undefined (reading '0')" via `chunk.choices[0]`,
+// aborting the SSE response after headers were already sent (so no fallback).
+describe('streamChunkText', () => {
+ it('extracts delta content from a normal chunk', () => {
+ expect(streamChunkText({ choices: [{ delta: { content: 'hello' } }] })).toBe('hello');
+ });
+
+ it('returns "" for a usage/keepalive chunk that has no choices array', () => {
+ expect(streamChunkText({ usage: { prompt_tokens: 1, completion_tokens: 2 } })).toBe('');
+ expect(streamChunkText({ x_groq: { usage: {} } })).toBe('');
+ });
+
+ it('returns "" for an empty choices array', () => {
+ expect(streamChunkText({ choices: [] })).toBe('');
+ });
+
+ it('returns "" for a tool-call delta with no text content', () => {
+ expect(streamChunkText({ choices: [{ delta: { tool_calls: [{ index: 0 }] } }] })).toBe('');
+ });
+
+ it('returns "" for null / undefined / malformed chunks', () => {
+ expect(streamChunkText(null)).toBe('');
+ expect(streamChunkText(undefined)).toBe('');
+ expect(streamChunkText({})).toBe('');
+ expect(streamChunkText({ choices: [{}] })).toBe('');
+ });
+});
diff --git a/server/src/routes/proxy.ts b/server/src/routes/proxy.ts
index 2cdc8068..afdbadc6 100644
--- a/server/src/routes/proxy.ts
+++ b/server/src/routes/proxy.ts
@@ -243,6 +243,16 @@ export function isRetryableError(err: any): boolean {
|| msg.includes('api error 400');
}
+// Pull the incremental text out of a streaming chunk for token counting.
+// Must tolerate chunks that carry no `choices` array at all: some providers
+// (e.g. Groq) emit usage/keepalive frames shaped like `{usage:{...}}` with no
+// `choices`. Indexing `chunk.choices[0]` on those throws "Cannot read
+// properties of undefined (reading '0')", which — once the SSE stream has
+// started — aborts the response mid-flight with no chance to fall back.
+export function streamChunkText(chunk: any): string {
+ return chunk?.choices?.[0]?.delta?.content ?? '';
+}
+
proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {
const start = Date.now();
@@ -412,7 +422,7 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {
if (attempt > 0) res.setHeader('X-Fallback-Attempts', String(attempt));
streamStarted = true;
}
- const text = chunk.choices[0]?.delta?.content ?? '';
+ const text = streamChunkText(chunk);
totalOutputTokens += Math.ceil(text.length / 4);
res.write(`data: ${JSON.stringify(chunk)}\n\n`);
}