7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.
TypeScript
Guard streaming chunks that carry no choices array (#156)
Tashfeena36dfd7b5ecb
diff --git a/server/src/__tests__/routes/proxy-stream-chunk.test.ts b/server/src/__tests__/routes/proxy-stream-chunk.test.tsnew file mode 100644index 00000000..4fdbff12--- /dev/null+++ b/server/src/__tests__/routes/proxy-stream-chunk.test.ts@@ -0,0 +1,32 @@+import { describe, it, expect } from 'vitest';+import { streamChunkText } from '../../routes/proxy.js';++// Regression for the mid-stream crash: a streaming chunk with no `choices`+// array (Groq and others emit usage/keepalive frames like this) used to throw+// "Cannot read properties of undefined (reading '0')" via `chunk.choices[0]`,+// aborting the SSE response after headers were already sent (so no fallback).+describe('streamChunkText', () => {+ it('extracts delta content from a normal chunk', () => {+ expect(streamChunkText({ choices: [{ delta: { content: 'hello' } }] })).toBe('hello');+ });++ it('returns "" for a usage/keepalive chunk that has no choices array', () => {+ expect(streamChunkText({ usage: { prompt_tokens: 1, completion_tokens: 2 } })).toBe('');+ expect(streamChunkText({ x_groq: { usage: {} } })).toBe('');+ });++ it('returns "" for an empty choices array', () => {+ expect(streamChunkText({ choices: [] })).toBe('');+ });++ it('returns "" for a tool-call delta with no text content', () => {+ expect(streamChunkText({ choices: [{ delta: { tool_calls: [{ index: 0 }] } }] })).toBe('');+ });++ it('returns "" for null / undefined / malformed chunks', () => {+ expect(streamChunkText(null)).toBe('');+ expect(streamChunkText(undefined)).toBe('');+ expect(streamChunkText({})).toBe('');+ expect(streamChunkText({ choices: [{}] })).toBe('');+ });+});diff --git a/server/src/routes/proxy.ts b/server/src/routes/proxy.tsindex 2cdc8068..afdbadc6 100644--- a/server/src/routes/proxy.ts+++ b/server/src/routes/proxy.ts@@ -243,6 +243,16 @@ export function isRetryableError(err: any): boolean {|| msg.includes('api error 400');}+// Pull the incremental text out of a streaming chunk for token counting.+// Must tolerate chunks that carry no `choices` array at all: some providers+// (e.g. Groq) emit usage/keepalive frames shaped like `{usage:{...}}` with no+// `choices`. Indexing `chunk.choices[0]` on those throws "Cannot read+// properties of undefined (reading '0')", which — once the SSE stream has+// started — aborts the response mid-flight with no chance to fall back.+export function streamChunkText(chunk: any): string {+ return chunk?.choices?.[0]?.delta?.content ?? '';+}+proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {const start = Date.now();@@ -412,7 +422,7 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {if (attempt > 0) res.setHeader('X-Fallback-Attempts', String(attempt));streamStarted = true;}- const text = chunk.choices[0]?.delta?.content ?? '';+ const text = streamChunkText(chunk);totalOutputTokens += Math.ceil(text.length / 4);res.write(`data: ${JSON.stringify(chunk)}\n\n`);}