Skip to content

7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.

TypeScript

fix(providers): give each streamed Gemini and Sail tool call its own index (#1623)

Jay Hemnanic9d147397270
diff --git a/server/src/__tests__/providers/google.test.ts b/server/src/__tests__/providers/google.test.ts
index 04ed6bbb..1c89d581 100644
--- a/server/src/__tests__/providers/google.test.ts
+++ b/server/src/__tests__/providers/google.test.ts
@@ -273,6 +273,32 @@ describe('GoogleProvider', () => {
expect(chunks[chunks.length - 1].choices[0].finish_reason).toBe('tool_calls');
});
+ it('gives every streamed tool call its own index and id, including parallel calls in one chunk', async () => {
+ vi.spyOn(global, 'fetch').mockImplementation(async () => {
+ const frames = [
+ { candidates: [{ content: { parts: [
+ { functionCall: { name: 'get_weather', args: { city: 'Dublin' } } },
+ { functionCall: { name: 'get_weather', args: { city: 'Paris' } } },
+ ] } }] },
+ { candidates: [{ content: { parts: [{ functionCall: { name: 'get_time', args: { tz: 'UTC' } } }] } }] },
+ { candidates: [{ content: { parts: [] }, finishReason: 'STOP' }] },
+ ];
+ // One body, so every frame is parsed within the same millisecond.
+ return new Response(frames.map(f => `data: ${JSON.stringify(f)}\n\n`).join(''), { status: 200 });
+ });
+
+ const chunks: any[] = [];
+ for await (const c of provider.streamChatCompletion('test-key', [{ role: 'user', content: 'Weather and time?' }], 'gemini-3.5-flash')) chunks.push(c);
+
+ const calls = chunks.flatMap(c => c.choices[0].delta.tool_calls ?? []);
+ expect(calls.map(c => [c.index, c.function.name, c.function.arguments])).toEqual([
+ [0, 'get_weather', '{"city":"Dublin"}'],
+ [1, 'get_weather', '{"city":"Paris"}'],
+ [2, 'get_time', '{"tz":"UTC"}'],
+ ]);
+ expect(new Set(calls.map(c => c.id)).size).toBe(3);
+ });
+
it('keeps functionCall/functionResponse history for Gemma multi-turn tool round-trips (#582)', async () => {
let capturedBody: any;
vi.spyOn(global, 'fetch').mockImplementation(async (_url, init) => {
diff --git a/server/src/__tests__/providers/sail.test.ts b/server/src/__tests__/providers/sail.test.ts
index a4a621c1..1c5d16be 100644
--- a/server/src/__tests__/providers/sail.test.ts
+++ b/server/src/__tests__/providers/sail.test.ts
@@ -155,6 +155,27 @@ describe('SailProvider', () => {
expect(chunks.at(-1)?.choices[0].finish_reason).toBe('stop');
});
+ it('indexes each tool call in the synthesized stream', async () => {
+ vi.spyOn(global, 'fetch').mockResolvedValue(jsonResponse({
+ id: 'resp_tools',
+ model: 'moonshotai/Kimi-K2.6',
+ status: 'completed',
+ output: [
+ { type: 'function_call', call_id: 'call_a', name: 'weather', arguments: '{"city":"Dublin"}' },
+ { type: 'function_call', call_id: 'call_b', name: 'weather', arguments: '{"city":"Paris"}' },
+ ],
+ usage: { input_tokens: 10, output_tokens: 5, total_tokens: 15 },
+ }));
+ const provider = new SailProvider({ pollIntervalMs: 0 });
+ const chunks = [];
+ for await (const chunk of provider.streamChatCompletion(
+ 'k', [{ role: 'user', content: 'Weather?' }], 'moonshotai/Kimi-K2.6',
+ )) chunks.push(chunk);
+
+ const calls = chunks.flatMap(chunk => (chunk.choices[0].delta.tool_calls ?? []) as any[]);
+ expect(calls.map(c => [c.index, c.id])).toEqual([[0, 'call_a'], [1, 'call_b']]);
+ });
+
it('validates keys against the authenticated model roster and preserves errors', async () => {
vi.spyOn(global, 'fetch')
.mockResolvedValueOnce(jsonResponse({ object: 'list', data: [] }))
diff --git a/server/src/__tests__/routes/fusion.test.ts b/server/src/__tests__/routes/fusion.test.ts
index a8652e0b..889e5e03 100644
--- a/server/src/__tests__/routes/fusion.test.ts
+++ b/server/src/__tests__/routes/fusion.test.ts
@@ -970,7 +970,8 @@ describe('fusion route (/v1/chat/completions, model: "fusion")', () => {
expect(traceFrames.some(f => f._fusion?.tool_calls?.[0]?.id === 'call_weather_stream')).toBe(true);
const toolFrame = frames.find(f => f.choices?.[0]?.delta?.tool_calls);
- expect(toolFrame.choices[0].delta.tool_calls).toEqual(toolCalls);
+ // A streamed tool-call delta carries its `index` (OpenAI chunk schema).
+ expect(toolFrame.choices[0].delta.tool_calls).toEqual(toolCalls.map((call, index) => ({ ...call, index })));
expect(frames.map(f => f.choices?.[0]?.finish_reason).filter(Boolean)).toEqual(['tool_calls']);
expect(text).not.toContain('JUDGE SHOULD NOT RUN');
expect(text.trimEnd().endsWith('data: [DONE]')).toBe(true);
diff --git a/server/src/providers/google.ts b/server/src/providers/google.ts
index f9905f8b..f3b7f854 100644
--- a/server/src/providers/google.ts
+++ b/server/src/providers/google.ts
@@ -492,11 +492,11 @@ async function toGeminiContents(messages: ChatMessage[]): Promise<{
};
}
-function extractToolCalls(parts: GeminiPart[] | undefined): ChatToolCall[] {
+function extractToolCalls(parts: GeminiPart[] | undefined, firstIndex = 0): ChatToolCall[] {
const calls: ChatToolCall[] = [];
if (!parts) return calls;
- let fallbackIndex = 0;
+ let fallbackIndex = firstIndex;
for (const part of parts) {
if (!part.functionCall?.name) continue;
@@ -702,6 +702,10 @@ export class GoogleProvider extends BaseProvider {
let sawToolCalls = false;
const seenToolCallKeys = new Set<string>();
+ // Stream position of the next tool call. Routes assemble streamed calls by
+ // `index`, so every call needs its own, including parallel calls that
+ // arrive in one chunk.
+ let nextToolIndex = 0;
// Same mid-stream inactivity watchdog as readSseStream (#553): this adapter
// parses Gemini's own frame format, so it reads the body itself and used to
@@ -761,12 +765,12 @@ export class GoogleProvider extends BaseProvider {
const text = extractText(parts);
const reasoningContent = extractReasoningContent(parts);
- const toolCalls = extractToolCalls(parts).filter(call => {
+ const toolCalls = extractToolCalls(parts, nextToolIndex).filter(call => {
const key = `${call.id}:${call.function.name}:${call.function.arguments}`;
if (seenToolCallKeys.has(key)) return false;
seenToolCallKeys.add(key);
return true;
- });
+ }).map(call => ({ ...call, index: nextToolIndex++ }));
if ((text && text.length > 0) || (reasoningContent && reasoningContent.length > 0) || toolCalls.length > 0) {
sawToolCalls = sawToolCalls || toolCalls.length > 0;
diff --git a/server/src/providers/sail.ts b/server/src/providers/sail.ts
index b73220ff..7482589f 100644
--- a/server/src/providers/sail.ts
+++ b/server/src/providers/sail.ts
@@ -381,7 +381,7 @@ export class SailProvider extends BaseProvider {
yield { ...base, choices: [{ index: 0, delta: { role: 'assistant' }, finish_reason: null }] };
if (content) yield { ...base, choices: [{ index: 0, delta: { content }, finish_reason: null }] };
if (toolCalls.length > 0) {
- yield { ...base, choices: [{ index: 0, delta: { tool_calls: toolCalls }, finish_reason: null }] };
+ yield { ...base, choices: [{ index: 0, delta: { tool_calls: toolCalls.map((call, index) => ({ ...call, index })) }, finish_reason: null }] };
}
yield { ...base, choices: [{ index: 0, delta: {}, finish_reason: choice.finish_reason }] };
}
diff --git a/server/src/routes/proxy.ts b/server/src/routes/proxy.ts
index adf6d6bb..9363fc29 100644
--- a/server/src/routes/proxy.ts
+++ b/server/src/routes/proxy.ts
@@ -2156,7 +2156,7 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {
const hasFinalToolCalls = Array.isArray(finalToolCalls) && finalToolCalls.length > 0;
if (hasFinalToolCalls) {
writeFrame({ ...base, choices: [{ index: 0, delta: { role: 'assistant' }, finish_reason: null }] });
- writeFrame({ ...base, choices: [{ index: 0, delta: { tool_calls: finalToolCalls }, finish_reason: null }] });
+ writeFrame({ ...base, choices: [{ index: 0, delta: { tool_calls: finalToolCalls.map((call, index) => ({ ...call, index })) }, finish_reason: null }] });
writeFrame({ ...base, choices: [{ index: 0, delta: {}, finish_reason: 'tool_calls' }], usage: response.usage });
} else {
if (!answerStarted) {