Skip to content

7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.

TypeScript

fix(nvidia): force single tool-call to avoid NIM 400 (#255)

Tashfeen4400f2e8e20f
diff --git a/server/src/__tests__/providers/openai-compat.test.ts b/server/src/__tests__/providers/openai-compat.test.ts
index 9b083c86..d38e4783 100644
--- a/server/src/__tests__/providers/openai-compat.test.ts
+++ b/server/src/__tests__/providers/openai-compat.test.ts
@@ -92,6 +92,45 @@ describe('OpenAICompatProvider', () => {
expect(capturedBody.parallel_tool_calls).toBe(true);
});
+ describe('forceSingleToolCall (NVIDIA NIM single-tool-call 400 — issue #255)', () => {
+ const nim = () => new OpenAICompatProvider({
+ platform: 'nvidia',
+ name: 'NVIDIA NIM',
+ baseUrl: 'https://integrate.api.nvidia.com/v1',
+ forceSingleToolCall: true,
+ });
+ const okResponse = {
+ ok: true,
+ json: () => Promise.resolve({
+ id: 'x', object: 'chat.completion', created: 1, model: 'm',
+ choices: [{ index: 0, message: { role: 'assistant', content: 'hi' }, finish_reason: 'stop' }],
+ usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 },
+ }),
+ } as any;
+ const tools = [{ type: 'function' as const, function: { name: 'f', description: 'd', parameters: { type: 'object', properties: {} } } }];
+
+ it('pins parallel_tool_calls to false when tools are present, even if the caller asked for true', async () => {
+ let body: any = null;
+ vi.spyOn(global, 'fetch').mockImplementation(async (_u, init) => { body = JSON.parse((init as any).body); return okResponse; });
+ await nim().chatCompletion('k', [{ role: 'user', content: 'hi' }], 'm', { tools, parallel_tool_calls: true });
+ expect(body.parallel_tool_calls).toBe(false);
+ });
+
+ it('leaves parallel_tool_calls untouched when there are no tools', async () => {
+ let body: any = null;
+ vi.spyOn(global, 'fetch').mockImplementation(async (_u, init) => { body = JSON.parse((init as any).body); return okResponse; });
+ await nim().chatCompletion('k', [{ role: 'user', content: 'hi' }], 'm', {});
+ expect(body.parallel_tool_calls).toBeUndefined();
+ });
+
+ it('does not affect providers without the flag (parallel_tool_calls passes through)', async () => {
+ let body: any = null;
+ vi.spyOn(global, 'fetch').mockImplementation(async (_u, init) => { body = JSON.parse((init as any).body); return okResponse; });
+ await provider.chatCompletion('k', [{ role: 'user', content: 'hi' }], 'm', { tools, parallel_tool_calls: true });
+ expect(body.parallel_tool_calls).toBe(true);
+ });
+ });
+
it('should throw on error response', async () => {
vi.spyOn(global, 'fetch').mockResolvedValueOnce({
ok: false,
diff --git a/server/src/providers/index.ts b/server/src/providers/index.ts
index 08fc425f..2a29f278 100644
--- a/server/src/providers/index.ts
+++ b/server/src/providers/index.ts
@@ -33,11 +33,14 @@ register(new OpenAICompatProvider({
// credit (expires in 3 months); once it lapses, every chat call 402s
// "payment method required" with no recurring no-card path back.
-// NVIDIA NIM - OpenAI-compatible
+// NVIDIA NIM - OpenAI-compatible. Several NIM models reject parallel tool calls
+// ("This model only supports single tool-calls at once!"), so pin
+// parallel_tool_calls to false when tools are present. See issue #255.
register(new OpenAICompatProvider({
platform: 'nvidia',
name: 'NVIDIA NIM',
baseUrl: 'https://integrate.api.nvidia.com/v1',
+ forceSingleToolCall: true,
}));
// Mistral - OpenAI-compatible
diff --git a/server/src/providers/openai-compat.ts b/server/src/providers/openai-compat.ts
index e441865b..afeb992e 100644
--- a/server/src/providers/openai-compat.ts
+++ b/server/src/providers/openai-compat.ts
@@ -20,6 +20,10 @@ export class OpenAICompatProvider extends BaseProvider {
/** Per-provider HTTP timeout override. Cloud APIs finish in ~15s; locally-hosted
* inference (llama.cpp / vLLM on CPU) can take 30-120s for long prompts. Default 15000. */
private readonly timeoutMs: number;
+ /** NVIDIA NIM models reject any request that permits parallel tool calls with
+ * `400 This model only supports single tool-calls at once!`. When set, pin
+ * parallel_tool_calls to false whenever tools are in play. See issue #255. */
+ private readonly forceSingleToolCall: boolean;
constructor(opts: {
platform: Platform;
@@ -29,6 +33,7 @@ export class OpenAICompatProvider extends BaseProvider {
validateUrl?: string;
timeoutMs?: number;
keyless?: boolean;
+ forceSingleToolCall?: boolean;
}) {
super();
this.platform = opts.platform;
@@ -38,6 +43,16 @@ export class OpenAICompatProvider extends BaseProvider {
this.validateUrl = opts.validateUrl;
this.timeoutMs = opts.timeoutMs ?? 15000;
this.keyless = opts.keyless ?? false;
+ this.forceSingleToolCall = opts.forceSingleToolCall ?? false;
+ }
+
+ /** Resolve the parallel_tool_calls flag to send upstream. For providers that
+ * only accept single tool calls (NVIDIA NIM), force `false` whenever tools are
+ * present so the model never tries to emit two at once and 400s; otherwise pass
+ * the caller's value through unchanged. See issue #255. */
+ private resolveParallelToolCalls(options?: CompletionOptions): boolean | undefined {
+ if (this.forceSingleToolCall && options?.tools && options.tools.length > 0) return false;
+ return options?.parallel_tool_calls;
}
/** Keyless providers (Kilo's anonymous free tier) must send NO Authorization
@@ -68,7 +83,7 @@ export class OpenAICompatProvider extends BaseProvider {
top_p: options?.top_p,
tools: options?.tools,
tool_choice: options?.tool_choice,
- parallel_tool_calls: options?.parallel_tool_calls,
+ parallel_tool_calls: this.resolveParallelToolCalls(options),
}),
}, options?.timeoutMs ?? this.timeoutMs);
@@ -116,7 +131,7 @@ export class OpenAICompatProvider extends BaseProvider {
top_p: options?.top_p,
tools: options?.tools,
tool_choice: options?.tool_choice,
- parallel_tool_calls: options?.parallel_tool_calls,
+ parallel_tool_calls: this.resolveParallelToolCalls(options),
stream: true,
}),
}, this.timeoutMs);