7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.
TypeScript
fix(nvidia): force single tool-call to avoid NIM 400 (#255)
Tashfeen4400f2e8e20f
diff --git a/server/src/__tests__/providers/openai-compat.test.ts b/server/src/__tests__/providers/openai-compat.test.tsindex 9b083c86..d38e4783 100644--- a/server/src/__tests__/providers/openai-compat.test.ts+++ b/server/src/__tests__/providers/openai-compat.test.ts@@ -92,6 +92,45 @@ describe('OpenAICompatProvider', () => {expect(capturedBody.parallel_tool_calls).toBe(true);});+ describe('forceSingleToolCall (NVIDIA NIM single-tool-call 400 — issue #255)', () => {+ const nim = () => new OpenAICompatProvider({+ platform: 'nvidia',+ name: 'NVIDIA NIM',+ baseUrl: 'https://integrate.api.nvidia.com/v1',+ forceSingleToolCall: true,+ });+ const okResponse = {+ ok: true,+ json: () => Promise.resolve({+ id: 'x', object: 'chat.completion', created: 1, model: 'm',+ choices: [{ index: 0, message: { role: 'assistant', content: 'hi' }, finish_reason: 'stop' }],+ usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 },+ }),+ } as any;+ const tools = [{ type: 'function' as const, function: { name: 'f', description: 'd', parameters: { type: 'object', properties: {} } } }];++ it('pins parallel_tool_calls to false when tools are present, even if the caller asked for true', async () => {+ let body: any = null;+ vi.spyOn(global, 'fetch').mockImplementation(async (_u, init) => { body = JSON.parse((init as any).body); return okResponse; });+ await nim().chatCompletion('k', [{ role: 'user', content: 'hi' }], 'm', { tools, parallel_tool_calls: true });+ expect(body.parallel_tool_calls).toBe(false);+ });++ it('leaves parallel_tool_calls untouched when there are no tools', async () => {+ let body: any = null;+ vi.spyOn(global, 'fetch').mockImplementation(async (_u, init) => { body = JSON.parse((init as any).body); return okResponse; });+ await nim().chatCompletion('k', [{ role: 'user', content: 'hi' }], 'm', {});+ expect(body.parallel_tool_calls).toBeUndefined();+ });++ it('does not affect providers without the flag (parallel_tool_calls passes through)', async () => {+ let body: any = null;+ vi.spyOn(global, 'fetch').mockImplementation(async (_u, init) => { body = JSON.parse((init as any).body); return okResponse; });+ await provider.chatCompletion('k', [{ role: 'user', content: 'hi' }], 'm', { tools, parallel_tool_calls: true });+ expect(body.parallel_tool_calls).toBe(true);+ });+ });+it('should throw on error response', async () => {vi.spyOn(global, 'fetch').mockResolvedValueOnce({ok: false,diff --git a/server/src/providers/index.ts b/server/src/providers/index.tsindex 08fc425f..2a29f278 100644--- a/server/src/providers/index.ts+++ b/server/src/providers/index.ts@@ -33,11 +33,14 @@ register(new OpenAICompatProvider({// credit (expires in 3 months); once it lapses, every chat call 402s// "payment method required" with no recurring no-card path back.-// NVIDIA NIM - OpenAI-compatible+// NVIDIA NIM - OpenAI-compatible. Several NIM models reject parallel tool calls+// ("This model only supports single tool-calls at once!"), so pin+// parallel_tool_calls to false when tools are present. See issue #255.register(new OpenAICompatProvider({platform: 'nvidia',name: 'NVIDIA NIM',baseUrl: 'https://integrate.api.nvidia.com/v1',+ forceSingleToolCall: true,}));// Mistral - OpenAI-compatiblediff --git a/server/src/providers/openai-compat.ts b/server/src/providers/openai-compat.tsindex e441865b..afeb992e 100644--- a/server/src/providers/openai-compat.ts+++ b/server/src/providers/openai-compat.ts@@ -20,6 +20,10 @@ export class OpenAICompatProvider extends BaseProvider {/** Per-provider HTTP timeout override. Cloud APIs finish in ~15s; locally-hosted* inference (llama.cpp / vLLM on CPU) can take 30-120s for long prompts. Default 15000. */private readonly timeoutMs: number;+ /** NVIDIA NIM models reject any request that permits parallel tool calls with+ * `400 This model only supports single tool-calls at once!`. When set, pin+ * parallel_tool_calls to false whenever tools are in play. See issue #255. */+ private readonly forceSingleToolCall: boolean;constructor(opts: {platform: Platform;@@ -29,6 +33,7 @@ export class OpenAICompatProvider extends BaseProvider {validateUrl?: string;timeoutMs?: number;keyless?: boolean;+ forceSingleToolCall?: boolean;}) {super();this.platform = opts.platform;@@ -38,6 +43,16 @@ export class OpenAICompatProvider extends BaseProvider {this.validateUrl = opts.validateUrl;this.timeoutMs = opts.timeoutMs ?? 15000;this.keyless = opts.keyless ?? false;+ this.forceSingleToolCall = opts.forceSingleToolCall ?? false;+ }++ /** Resolve the parallel_tool_calls flag to send upstream. For providers that+ * only accept single tool calls (NVIDIA NIM), force `false` whenever tools are+ * present so the model never tries to emit two at once and 400s; otherwise pass+ * the caller's value through unchanged. See issue #255. */+ private resolveParallelToolCalls(options?: CompletionOptions): boolean | undefined {+ if (this.forceSingleToolCall && options?.tools && options.tools.length > 0) return false;+ return options?.parallel_tool_calls;}/** Keyless providers (Kilo's anonymous free tier) must send NO Authorization@@ -68,7 +83,7 @@ export class OpenAICompatProvider extends BaseProvider {top_p: options?.top_p,tools: options?.tools,tool_choice: options?.tool_choice,- parallel_tool_calls: options?.parallel_tool_calls,+ parallel_tool_calls: this.resolveParallelToolCalls(options),}),}, options?.timeoutMs ?? this.timeoutMs);@@ -116,7 +131,7 @@ export class OpenAICompatProvider extends BaseProvider {top_p: options?.top_p,tools: options?.tools,tool_choice: options?.tool_choice,- parallel_tool_calls: options?.parallel_tool_calls,+ parallel_tool_calls: this.resolveParallelToolCalls(options),stream: true,}),}, this.timeoutMs);