7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.
TypeScript
fix(providers): give each streamed Gemini and Sail tool call its own index (#1623)
Jay Hemnanic9d147397270
diff --git a/server/src/__tests__/providers/google.test.ts b/server/src/__tests__/providers/google.test.tsindex 04ed6bbb..1c89d581 100644--- a/server/src/__tests__/providers/google.test.ts+++ b/server/src/__tests__/providers/google.test.ts@@ -273,6 +273,32 @@ describe('GoogleProvider', () => {expect(chunks[chunks.length - 1].choices[0].finish_reason).toBe('tool_calls');});+ it('gives every streamed tool call its own index and id, including parallel calls in one chunk', async () => {+ vi.spyOn(global, 'fetch').mockImplementation(async () => {+ const frames = [+ { candidates: [{ content: { parts: [+ { functionCall: { name: 'get_weather', args: { city: 'Dublin' } } },+ { functionCall: { name: 'get_weather', args: { city: 'Paris' } } },+ ] } }] },+ { candidates: [{ content: { parts: [{ functionCall: { name: 'get_time', args: { tz: 'UTC' } } }] } }] },+ { candidates: [{ content: { parts: [] }, finishReason: 'STOP' }] },+ ];+ // One body, so every frame is parsed within the same millisecond.+ return new Response(frames.map(f => `data: ${JSON.stringify(f)}\n\n`).join(''), { status: 200 });+ });++ const chunks: any[] = [];+ for await (const c of provider.streamChatCompletion('test-key', [{ role: 'user', content: 'Weather and time?' }], 'gemini-3.5-flash')) chunks.push(c);++ const calls = chunks.flatMap(c => c.choices[0].delta.tool_calls ?? []);+ expect(calls.map(c => [c.index, c.function.name, c.function.arguments])).toEqual([+ [0, 'get_weather', '{"city":"Dublin"}'],+ [1, 'get_weather', '{"city":"Paris"}'],+ [2, 'get_time', '{"tz":"UTC"}'],+ ]);+ expect(new Set(calls.map(c => c.id)).size).toBe(3);+ });+it('keeps functionCall/functionResponse history for Gemma multi-turn tool round-trips (#582)', async () => {let capturedBody: any;vi.spyOn(global, 'fetch').mockImplementation(async (_url, init) => {diff --git a/server/src/__tests__/providers/sail.test.ts b/server/src/__tests__/providers/sail.test.tsindex a4a621c1..1c5d16be 100644--- a/server/src/__tests__/providers/sail.test.ts+++ b/server/src/__tests__/providers/sail.test.ts@@ -155,6 +155,27 @@ describe('SailProvider', () => {expect(chunks.at(-1)?.choices[0].finish_reason).toBe('stop');});+ it('indexes each tool call in the synthesized stream', async () => {+ vi.spyOn(global, 'fetch').mockResolvedValue(jsonResponse({+ id: 'resp_tools',+ model: 'moonshotai/Kimi-K2.6',+ status: 'completed',+ output: [+ { type: 'function_call', call_id: 'call_a', name: 'weather', arguments: '{"city":"Dublin"}' },+ { type: 'function_call', call_id: 'call_b', name: 'weather', arguments: '{"city":"Paris"}' },+ ],+ usage: { input_tokens: 10, output_tokens: 5, total_tokens: 15 },+ }));+ const provider = new SailProvider({ pollIntervalMs: 0 });+ const chunks = [];+ for await (const chunk of provider.streamChatCompletion(+ 'k', [{ role: 'user', content: 'Weather?' }], 'moonshotai/Kimi-K2.6',+ )) chunks.push(chunk);++ const calls = chunks.flatMap(chunk => (chunk.choices[0].delta.tool_calls ?? []) as any[]);+ expect(calls.map(c => [c.index, c.id])).toEqual([[0, 'call_a'], [1, 'call_b']]);+ });+it('validates keys against the authenticated model roster and preserves errors', async () => {vi.spyOn(global, 'fetch').mockResolvedValueOnce(jsonResponse({ object: 'list', data: [] }))diff --git a/server/src/__tests__/routes/fusion.test.ts b/server/src/__tests__/routes/fusion.test.tsindex a8652e0b..889e5e03 100644--- a/server/src/__tests__/routes/fusion.test.ts+++ b/server/src/__tests__/routes/fusion.test.ts@@ -970,7 +970,8 @@ describe('fusion route (/v1/chat/completions, model: "fusion")', () => {expect(traceFrames.some(f => f._fusion?.tool_calls?.[0]?.id === 'call_weather_stream')).toBe(true);const toolFrame = frames.find(f => f.choices?.[0]?.delta?.tool_calls);- expect(toolFrame.choices[0].delta.tool_calls).toEqual(toolCalls);+ // A streamed tool-call delta carries its `index` (OpenAI chunk schema).+ expect(toolFrame.choices[0].delta.tool_calls).toEqual(toolCalls.map((call, index) => ({ ...call, index })));expect(frames.map(f => f.choices?.[0]?.finish_reason).filter(Boolean)).toEqual(['tool_calls']);expect(text).not.toContain('JUDGE SHOULD NOT RUN');expect(text.trimEnd().endsWith('data: [DONE]')).toBe(true);diff --git a/server/src/providers/google.ts b/server/src/providers/google.tsindex f9905f8b..f3b7f854 100644--- a/server/src/providers/google.ts+++ b/server/src/providers/google.ts@@ -492,11 +492,11 @@ async function toGeminiContents(messages: ChatMessage[]): Promise<{};}-function extractToolCalls(parts: GeminiPart[] | undefined): ChatToolCall[] {+function extractToolCalls(parts: GeminiPart[] | undefined, firstIndex = 0): ChatToolCall[] {const calls: ChatToolCall[] = [];if (!parts) return calls;- let fallbackIndex = 0;+ let fallbackIndex = firstIndex;for (const part of parts) {if (!part.functionCall?.name) continue;@@ -702,6 +702,10 @@ export class GoogleProvider extends BaseProvider {let sawToolCalls = false;const seenToolCallKeys = new Set<string>();+ // Stream position of the next tool call. Routes assemble streamed calls by+ // `index`, so every call needs its own, including parallel calls that+ // arrive in one chunk.+ let nextToolIndex = 0;// Same mid-stream inactivity watchdog as readSseStream (#553): this adapter// parses Gemini's own frame format, so it reads the body itself and used to@@ -761,12 +765,12 @@ export class GoogleProvider extends BaseProvider {const text = extractText(parts);const reasoningContent = extractReasoningContent(parts);- const toolCalls = extractToolCalls(parts).filter(call => {+ const toolCalls = extractToolCalls(parts, nextToolIndex).filter(call => {const key = `${call.id}:${call.function.name}:${call.function.arguments}`;if (seenToolCallKeys.has(key)) return false;seenToolCallKeys.add(key);return true;- });+ }).map(call => ({ ...call, index: nextToolIndex++ }));if ((text && text.length > 0) || (reasoningContent && reasoningContent.length > 0) || toolCalls.length > 0) {sawToolCalls = sawToolCalls || toolCalls.length > 0;diff --git a/server/src/providers/sail.ts b/server/src/providers/sail.tsindex b73220ff..7482589f 100644--- a/server/src/providers/sail.ts+++ b/server/src/providers/sail.ts@@ -381,7 +381,7 @@ export class SailProvider extends BaseProvider {yield { ...base, choices: [{ index: 0, delta: { role: 'assistant' }, finish_reason: null }] };if (content) yield { ...base, choices: [{ index: 0, delta: { content }, finish_reason: null }] };if (toolCalls.length > 0) {- yield { ...base, choices: [{ index: 0, delta: { tool_calls: toolCalls }, finish_reason: null }] };+ yield { ...base, choices: [{ index: 0, delta: { tool_calls: toolCalls.map((call, index) => ({ ...call, index })) }, finish_reason: null }] };}yield { ...base, choices: [{ index: 0, delta: {}, finish_reason: choice.finish_reason }] };}diff --git a/server/src/routes/proxy.ts b/server/src/routes/proxy.tsindex adf6d6bb..9363fc29 100644--- a/server/src/routes/proxy.ts+++ b/server/src/routes/proxy.ts@@ -2156,7 +2156,7 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {const hasFinalToolCalls = Array.isArray(finalToolCalls) && finalToolCalls.length > 0;if (hasFinalToolCalls) {writeFrame({ ...base, choices: [{ index: 0, delta: { role: 'assistant' }, finish_reason: null }] });- writeFrame({ ...base, choices: [{ index: 0, delta: { tool_calls: finalToolCalls }, finish_reason: null }] });+ writeFrame({ ...base, choices: [{ index: 0, delta: { tool_calls: finalToolCalls.map((call, index) => ({ ...call, index })) }, finish_reason: null }] });writeFrame({ ...base, choices: [{ index: 0, delta: {}, finish_reason: 'tool_calls' }], usage: response.usage });} else {if (!answerStarted) {