Skip to content

7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.

TypeScript

Fix provider timeouts and Gemma quirks

Tashfeen6bac51958623
diff --git a/README.md b/README.md
index 4f1a5365..29da7c05 100644
--- a/README.md
+++ b/README.md
@@ -2,7 +2,7 @@
# FreeLLMAPI
-**One OpenAI-compatible endpoint. 28 free LLM providers. 339 free model endpoints. ~4B tokens per month.**
+**One OpenAI-compatible endpoint. 28 free LLM providers. 339 free model endpoints. ~4 billion tokens per month.**
Aggregate the free tiers from Google, Groq, Cerebras, NVIDIA, Mistral, OpenRouter, GitHub Models, Cohere, Cloudflare, HuggingFace, Z.ai (Zhipu), Ollama, Kilo, Pollinations, LLM7, OVH AI Endpoints, OpenCode Zen, AI Horde, NaraRouter, Aion Labs, Requesty, NavyAI, Agnes AI, Reka, SiliconFlow, Routeway, BazaarLink, and AINative Studio, plus custom OpenAI-compatible chat, embedding, image, and audio endpoints, behind a single `/v1` API. Keys are stored encrypted. A router picks the best available model for each request, falls over to the next provider when one is rate-limited, and tracks per-key usage so you stay under every free-tier cap.
@@ -31,7 +31,7 @@ Your router updates its own model catalog from a signed feed: new free models, q
- [Features](#features)
- [Not yet supported](#not-yet-supported)
- [Quick start](#quick-start)
-- [Works with OB-1 and other clients](#works-with-ob-1-and-other-clients)
+- [Works with OpenAI-compatible clients](#works-with-openai-compatible-clients)
- [Docker](#docker)
- [Desktop app](#desktop-app)
- [Languages](#languages)
@@ -397,17 +397,10 @@ flat JSON files. To add a language, copy `en.json`, translate the values, and
register the locale in `client/src/i18n/I18nProvider.tsx` (and
`desktop/src/i18n.ts` for the tray strings) — PRs welcome.
-## Works with OB-1 and other clients
+## Works with OpenAI-compatible clients
-FreeLLMAPI is the free tier for **[OB-1](https://github.com/Overbrilliant/ob-1)**:
-the OB-1 CLI can clone, configure, start, health-check, and wire this proxy into
-its settings automatically. A new OB-1 user can pick **Start free** and reach a
-working OpenAI-compatible endpoint before creating any hosted account.
+Any client that can target an OpenAI-compatible base URL can use FreeLLMAPI:
-It is also useful on its own. Any client that can target an OpenAI-compatible
-base URL can use FreeLLMAPI:
-
-- **OB-1**: managed automatically by the CLI, including anonymous providers.
- **LangChain, LlamaIndex, official OpenAI SDKs**: set `base_url` to
`http://localhost:3001/v1` and use the unified key from the dashboard.
- **Local GPU boxes**: add custom OpenAI-compatible endpoints for Ollama,
@@ -454,7 +447,7 @@ enable/disable choices and custom providers are never touched, and every
download is verified against a pinned Ed25519 key before it is applied.
The catalog currently tracks **28 providers**, **235 model families**, **339
-provider/model endpoints**, and roughly **4B tokens per month** of listed
+provider/model endpoints**, and roughly **4 billion tokens per month** of listed
free-tier capacity. Browse the full set at
**[freellmapi.co/models](https://freellmapi.co/models.html)**.
diff --git a/server/src/__tests__/providers/google.test.ts b/server/src/__tests__/providers/google.test.ts
index 1d8b79e9..4de07d14 100644
--- a/server/src/__tests__/providers/google.test.ts
+++ b/server/src/__tests__/providers/google.test.ts
@@ -154,6 +154,74 @@ describe('GoogleProvider', () => {
expect(capturedBody.contents[0].role).toBe('user');
});
+ it('folds system prompts into user content and strips function tools for Gemma models (#500)', async () => {
+ let capturedBody: any;
+ vi.spyOn(global, 'fetch').mockImplementation(async (_url, init) => {
+ capturedBody = JSON.parse((init as any).body);
+ return {
+ ok: true,
+ json: () => Promise.resolve({
+ candidates: [{ content: { parts: [{ text: 'ok' }] }, finishReason: 'STOP' }],
+ usageMetadata: { promptTokenCount: 1, candidatesTokenCount: 1, totalTokenCount: 2 },
+ }),
+ } as any;
+ });
+
+ await provider.chatCompletion(
+ 'test-key',
+ [
+ { role: 'system', content: 'You are helpful' },
+ { role: 'user', content: 'Hi' },
+ ],
+ 'gemma-4-31b-it',
+ {
+ tools: [{
+ type: 'function',
+ function: {
+ name: 'get_weather',
+ description: 'Get weather for a city',
+ parameters: { type: 'object', properties: { city: { type: 'string' } } },
+ },
+ }],
+ tool_choice: 'auto',
+ parallel_tool_calls: true,
+ },
+ );
+
+ expect(capturedBody.systemInstruction).toBeUndefined();
+ expect(capturedBody.tools).toBeUndefined();
+ expect(capturedBody.toolConfig).toBeUndefined();
+ expect(capturedBody.contents[0]).toEqual({ role: 'user', parts: [{ text: 'You are helpful' }] });
+ expect(capturedBody.contents[1]).toEqual({ role: 'user', parts: [{ text: 'Hi' }] });
+ });
+
+ it('does not expose Gemini thought parts as visible content (#539)', async () => {
+ vi.spyOn(global, 'fetch').mockResolvedValueOnce({
+ ok: true,
+ json: () => Promise.resolve({
+ candidates: [{
+ content: {
+ parts: [
+ { text: 'private reasoning', thought: true },
+ { text: 'visible answer' },
+ ],
+ },
+ finishReason: 'STOP',
+ }],
+ usageMetadata: { promptTokenCount: 1, candidatesTokenCount: 2, totalTokenCount: 3 },
+ }),
+ } as any);
+
+ const result = await provider.chatCompletion(
+ 'test-key',
+ [{ role: 'user', content: 'Hi' }],
+ 'gemma-4-31b-it',
+ );
+
+ expect(result.choices[0].message.content).toBe('visible answer');
+ expect(result.choices[0].message.reasoning_content).toBe('private reasoning');
+ });
+
it('should translate OpenAI tools/tool_choice to Gemini tools/toolConfig', async () => {
let capturedBody: any;
vi.spyOn(global, 'fetch').mockImplementation(async (_url, init) => {
@@ -448,6 +516,26 @@ describe('GoogleProvider', () => {
expect(chunks[chunks.length - 1].choices[0].finish_reason).toBe('stop');
});
+ it('streams Gemini thought parts as reasoning_content, not visible content (#539)', async () => {
+ vi.spyOn(global, 'fetch').mockResolvedValueOnce(sseResponse([
+ 'data: {"candidates":[{"content":{"parts":[{"text":"private","thought":true}]}}]}\n\n',
+ 'data: {"candidates":[{"content":{"parts":[{"text":" answer"}]}}]}\n\n',
+ 'data: {"candidates":[{"content":{"parts":[]},"finishReason":"STOP"}]}\n\n',
+ ]));
+
+ const chunks = await collect(provider.streamChatCompletion(
+ 'test-key',
+ [{ role: 'user', content: 'Hi' }],
+ 'gemma-4-31b-it',
+ ));
+
+ const reasoning = chunks.map(c => c.choices[0].delta.reasoning_content ?? '').join('');
+ const text = chunks.map(c => c.choices[0].delta.content ?? '').join('');
+ expect(reasoning).toBe('private');
+ expect(text).toBe(' answer');
+ expect(chunks[chunks.length - 1].choices[0].finish_reason).toBe('stop');
+ });
+
it('skips a malformed SSE frame instead of aborting the whole stream', async () => {
// Regression: previously an unguarded JSON.parse would propagate, killing
// the stream after a single bad chunk. Other providers (openai-compat,
diff --git a/server/src/__tests__/providers/openai-compat.test.ts b/server/src/__tests__/providers/openai-compat.test.ts
index 1b2d70b4..2304ce1a 100644
--- a/server/src/__tests__/providers/openai-compat.test.ts
+++ b/server/src/__tests__/providers/openai-compat.test.ts
@@ -1,4 +1,4 @@
-import { describe, it, expect, vi, beforeEach } from 'vitest';
+import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
import { OpenAICompatProvider } from '../../providers/openai-compat.js';
describe('OpenAICompatProvider', () => {
@@ -13,6 +13,27 @@ describe('OpenAICompatProvider', () => {
});
});
+ afterEach(() => {
+ vi.restoreAllMocks();
+ });
+
+ function sseResponse(frames: string[]): any {
+ const stream = new ReadableStream<Uint8Array>({
+ start(controller) {
+ const encoder = new TextEncoder();
+ for (const frame of frames) controller.enqueue(encoder.encode(frame));
+ controller.close();
+ },
+ });
+ return { ok: true, body: stream, headers: new Headers() };
+ }
+
+ async function collect<T>(gen: AsyncGenerator<T>): Promise<T[]> {
+ const out: T[] = [];
+ for await (const chunk of gen) out.push(chunk);
+ return out;
+ }
+
it('should set platform and name from config', () => {
expect(provider.platform).toBe('groq');
expect(provider.name).toBe('TestProvider');
@@ -48,6 +69,57 @@ describe('OpenAICompatProvider', () => {
expect(capturedBody.messages[0].role).toBe('user');
});
+ it('uses a 60s chat timeout by default for OpenAI-compatible providers (#530)', async () => {
+ const delays: number[] = [];
+ const origSetTimeout = global.setTimeout;
+ vi.spyOn(global, 'setTimeout').mockImplementation(((fn: () => void, ms?: number) => {
+ delays.push(ms ?? 0);
+ return origSetTimeout(fn, ms);
+ }) as typeof setTimeout);
+ vi.spyOn(global, 'fetch').mockResolvedValueOnce({
+ ok: true,
+ json: () => Promise.resolve({
+ id: 'test-id',
+ object: 'chat.completion',
+ created: 123,
+ model: 'test-model',
+ choices: [{ index: 0, message: { role: 'assistant', content: 'hi' }, finish_reason: 'stop' }],
+ usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 },
+ }),
+ } as any);
+
+ await provider.chatCompletion('my-key', [{ role: 'user', content: 'test' }], 'test-model');
+
+ expect(delays).toContain(60_000);
+ expect(delays).not.toContain(15_000);
+ });
+
+ it('honors CompletionOptions.timeoutMs for OpenAI-compatible streams (#530)', async () => {
+ const delays: number[] = [];
+ const origSetTimeout = global.setTimeout;
+ vi.spyOn(global, 'setTimeout').mockImplementation(((fn: () => void, ms?: number) => {
+ delays.push(ms ?? 0);
+ return origSetTimeout(fn, ms);
+ }) as typeof setTimeout);
+ vi.spyOn(global, 'fetch').mockResolvedValueOnce(sseResponse([
+ 'data: {"id":"x","object":"chat.completion.chunk","created":1,"model":"m","choices":[{"index":0,"delta":{"content":"hi"},"finish_reason":null}]}\n\n',
+ 'data: {"id":"x","object":"chat.completion.chunk","created":1,"model":"m","choices":[{"index":0,"delta":{},"finish_reason":"stop"}]}\n\n',
+ 'data: [DONE]\n\n',
+ ]));
+
+ const chunks = await collect(provider.streamChatCompletion(
+ 'my-key',
+ [{ role: 'user', content: 'test' }],
+ 'test-model',
+ { timeoutMs: 12_345 },
+ ));
+
+ expect(chunks.length).toBeGreaterThan(0);
+ expect(delays).toContain(12_345);
+ expect(delays).not.toContain(60_000);
+ expect(delays).not.toContain(15_000);
+ });
+
it('should pass tool-calling params through untouched', async () => {
let capturedBody: any = null;
vi.spyOn(global, 'fetch').mockImplementation(async (_url, init) => {
@@ -293,6 +365,46 @@ describe('OpenAICompatProvider', () => {
expect(result.choices[0].message.content).toBe('part one part two');
});
+ it('strips internal reasoning/thought fields before sending messages to Mistral (#530)', async () => {
+ let body: any = null;
+ vi.spyOn(global, 'fetch').mockImplementation(async (_url, init) => {
+ body = JSON.parse((init as any).body);
+ return {
+ ok: true,
+ json: () => Promise.resolve({
+ id: 'id', object: 'chat.completion', created: 1, model: 'm',
+ choices: [{ index: 0, message: { role: 'assistant', content: 'ok' }, finish_reason: 'stop' }],
+ usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 },
+ }),
+ } as any;
+ });
+
+ const mistral = new OpenAICompatProvider({ platform: 'mistral', name: 'Mistral', baseUrl: 'https://api.mistral.ai/v1' });
+ await mistral.chatCompletion(
+ 'k',
+ [{
+ role: 'assistant',
+ content: null,
+ reasoning_content: 'private chain',
+ tool_calls: [{
+ id: 'call_1',
+ type: 'function',
+ function: { name: 'lookup', arguments: '{}' },
+ thought_signature: 'google-private-signature',
+ }],
+ }],
+ 'mistral-medium-3',
+ );
+
+ expect(body.messages[0]).not.toHaveProperty('reasoning_content');
+ expect(body.messages[0].tool_calls[0]).not.toHaveProperty('thought_signature');
+ expect(body.messages[0].tool_calls[0]).toEqual({
+ id: 'call_1',
+ type: 'function',
+ function: { name: 'lookup', arguments: '{}' },
+ });
+ });
+
it('folds reasoning into content when content is empty (Ollama style — bare `reasoning` field)', async () => {
vi.spyOn(global, 'fetch').mockResolvedValueOnce({
ok: true,
diff --git a/server/src/__tests__/providers/reasoning-timeouts.test.ts b/server/src/__tests__/providers/reasoning-timeouts.test.ts
index b2d7e8f7..eab9ba27 100644
--- a/server/src/__tests__/providers/reasoning-timeouts.test.ts
+++ b/server/src/__tests__/providers/reasoning-timeouts.test.ts
@@ -3,10 +3,11 @@ import { resolveProvider } from '../../providers/index.js';
import { CloudflareProvider } from '../../providers/cloudflare.js';
// Regression guard for the 2026-07-11 live-sweep finding: platforms hosting
-// hidden-reasoning models (zhipu glm-4.7-flash 41s TTFB, agnes-2.0-flash 20s,
+// hidden-reasoning or buffered non-streaming models (zhipu glm-4.7-flash 41s
+// TTFB, agnes-2.0-flash 20s, OpenRouter/OpenCode/Mistral long generations,
// @cf/zai-org/glm-4.7-flash repeated 15s aborts) need a chat timeout above
-// the 15s fetch default or every attempt — streaming included — is aborted
-// before the first byte. The value is a private construction detail, so the
+// 15s or every attempt — streaming included — is aborted before the first byte.
+// The value is a private construction detail, so the
// registry entries are asserted via the stored field and Cloudflare via the
// setTimeout the abort rides on.
@@ -16,8 +17,11 @@ describe('reasoning-model chat timeouts', () => {
});
it.each([
+ ['mistral', 60_000],
+ ['openrouter', 60_000],
['zhipu', 60_000],
['agnes', 60_000],
+ ['opencode', 60_000],
['ollama', 120_000], // pre-existing bump; keep it from regressing too
] as const)('%s is registered with a %dms chat timeout', (platform, ms) => {
const provider = resolveProvider(platform);
diff --git a/server/src/__tests__/routes/proxy-retry.test.ts b/server/src/__tests__/routes/proxy-retry.test.ts
index de550a2f..470f556f 100644
--- a/server/src/__tests__/routes/proxy-retry.test.ts
+++ b/server/src/__tests__/routes/proxy-retry.test.ts
@@ -99,6 +99,17 @@ describe('isRetryableError', () => {
expect(isProviderBadRequestError(new Error('400 Bad Request'))).toBe(false);
expect(isProviderBadRequestError(Object.assign(new Error('Bad Request'), { status: 400 }))).toBe(false);
});
+
+ it('treats provider API 422s like Mistral validation rejects: retryable, then invalid-request on exhaustion', () => {
+ const err = Object.assign(
+ new Error('Mistral API error 422: Unprocessable Entity'),
+ { status: 422 },
+ );
+ expect(isRetryableError(err)).toBe(true);
+ expect(isProviderBadRequestError(err)).toBe(true);
+ expect(isRetryableError(new Error('Mistral API error 422: tool messages failed validation'))).toBe(true);
+ expect(isProviderBadRequestError(new Error('Mistral API error 422: tool messages failed validation'))).toBe(true);
+ });
});
describe('403 model not on this key\'s tier fails over instead of 502 (issue #256)', () => {
diff --git a/server/src/lib/error-classify.ts b/server/src/lib/error-classify.ts
index 41f1b1a3..710565a9 100644
--- a/server/src/lib/error-classify.ts
+++ b/server/src/lib/error-classify.ts
@@ -17,8 +17,11 @@ export function isRetryableError(err: any): boolean {
// 410 (model pulled upstream), 429 (rate limit) and all 5xx are transient or
// fail-over-able; 400/401 stay fatal (status 0 here, handled by the absence of a
// matching rule) and 403 is handled by isModelAccessForbiddenError below.
+ // 422 (notably Mistral's strict validation) is a provider-side request-shape
+ // rejection: fail over to another provider, but if the whole chain rejects it
+ // exhaustedRetryError renders a client-facing 400 via isProviderBadRequestError.
const status = typeof err?.status === 'number' ? err.status : 0;
- if (status === 408 || status === 409 || status === 410 || status === 429 || status >= 500) return true;
+ if (status === 408 || status === 409 || status === 410 || status === 422 || status === 429 || status >= 500) return true;
return msg.includes('429') || msg.includes('rate limit') || msg.includes('too many requests')
|| msg.includes('quota') || msg.includes('resource_exhausted')
|| msg.includes('aborted') || msg.includes('timeout') || msg.includes('etimedout')
@@ -52,6 +55,11 @@ export function isRetryableError(err: any): boolean {
// which comes from the OpenAI-compat provider's error formatting, not
// a bare "400" which is deliberately non-retryable for validation errors.
|| msg.includes('api error 400')
+ // 422: Mistral and other strict OpenAI-compatible endpoints use
+ // Unprocessable Entity for request-shape validation. Rotate to a provider
+ // that accepts the same OpenAI payload; if every provider rejects it, render
+ // a clean invalid_request_error instead of a rate-limit exhaustion.
+ || msg.includes('api error 422') || msg.includes('unprocessable entity')
// 402: this provider/key is out of credits (e.g. HuggingFace Router
// "API error 402: Payment required"). The SAME model often lives on another
// provider (Kimi K2.6 is on HF + Cloudflare + NVIDIA), so fail over instead
@@ -132,7 +140,10 @@ export function isDailyQuotaExhaustedError(err: any): boolean {
export function isProviderBadRequestError(err: any): boolean {
const status = typeof err?.status === 'number' ? err.status : 0;
const msg = (err?.message ?? '').toLowerCase();
- return (status === 0 || status === 400) && msg.includes('api error 400');
+ if (status === 400) return msg.includes('api error 400');
+ if (status === 422) return msg.includes('api error 422') || msg.includes('unprocessable entity');
+ if (status !== 0) return false;
+ return msg.includes('api error 400') || msg.includes('api error 422') || msg.includes('unprocessable entity');
}
// A 402 Payment Required / out-of-credits error. Distinct from a transient 429:
diff --git a/server/src/providers/google.ts b/server/src/providers/google.ts
index eb94e340..3f79d40f 100644
--- a/server/src/providers/google.ts
+++ b/server/src/providers/google.ts
@@ -74,6 +74,7 @@ function recallThoughtSig(callId: string | undefined, name?: string, args?: unkn
interface GeminiPart {
text?: string;
+ thought?: boolean;
inlineData?: {
mimeType: string;
data: string;
@@ -105,6 +106,56 @@ interface GeminiResponse {
};
}
+type GeminiContent = { role: 'user' | 'model'; parts: GeminiPart[] };
+
+function isGemmaModel(modelId: string): boolean {
+ const normalized = modelId.toLowerCase().replace(/^models\//, '');
+ return /(?:^|[/.:])gemma[-_]/.test(normalized);
+}
+
+function optionsForModel(modelId: string, options?: CompletionOptions): CompletionOptions | undefined {
+ if (!isGemmaModel(modelId) || !options) return options;
+ const { tools: _tools, tool_choice: _toolChoice, parallel_tool_calls: _parallelToolCalls, ...rest } = options;
+ return rest;
+}
+
+function systemInstructionText(systemInstruction: { parts?: Array<{ text?: string }> } | undefined): string | null {
+ const text = systemInstruction?.parts
+ ?.map(part => part.text ?? '')
+ .join('\n\n')
+ .trim();
+ return text ? text : null;
+}
+
+function contentsForModel(
+ modelId: string,
+ contents: GeminiContent[],
+ systemInstruction: { parts: Array<{ text: string }> } | undefined,
+): { contents: GeminiContent[]; systemInstruction?: { parts: Array<{ text: string }> } } {
+ if (!isGemmaModel(modelId)) return { contents, systemInstruction };
+
+ const cleaned = contents
+ .map((entry): GeminiContent | null => {
+ const parts = entry.parts.filter(part => !part.functionCall && !part.functionResponse);
+ if (parts.length === 0) return null;
+ return { ...entry, parts };
+ })
+ .filter((entry): entry is GeminiContent => entry !== null);
+ const safeContents = cleaned.length > 0
+ ? cleaned
+ : [{ role: 'user' as const, parts: [{ text: '' }] }];
+
+ const systemText = systemInstructionText(systemInstruction);
+ if (!systemText) return { contents: safeContents };
+
+ return {
+ contents: [
+ { role: 'user', parts: [{ text: systemText }] },
+ ...safeContents,
+ ],
+ };
+}
+
function safeParseObject(raw: string): Record<string, unknown> {
try {
const parsed = JSON.parse(raw) as unknown;
@@ -336,7 +387,10 @@ async function userContentToParts(content: ChatMessage['content']): Promise<Gemi
// Translate OpenAI messages to Gemini format. Content may arrive as a string,
// null, or the OpenAI multimodal array envelope. System/assistant/tool messages
// flatten to text; user messages additionally carry images as inlineData parts.
-async function toGeminiContents(messages: ChatMessage[]) {
+async function toGeminiContents(messages: ChatMessage[]): Promise<{
+ contents: GeminiContent[];
+ systemInstruction?: { parts: Array<{ text: string }> };
+}> {
const systemMessages = messages
.filter(m => m.role === 'system')
.map(m => contentToString(m.content))
@@ -447,6 +501,16 @@ function extractToolCalls(parts: GeminiPart[] | undefined): ChatToolCall[] {
function extractText(parts: GeminiPart[] | undefined): string | null {
if (!parts) return null;
const text = parts
+ .filter(p => p.thought !== true)
+ .map(p => p.text ?? '')
+ .join('');
+ return text.length > 0 ? text : null;
+}
+
+function extractReasoningContent(parts: GeminiPart[] | undefined): string | null {
+ if (!parts) return null;
+ const text = parts
+ .filter(p => p.thought === true)
.map(p => p.text ?? '')
.join('');
return text.length > 0 ? text : null;
@@ -481,24 +545,26 @@ export class GoogleProvider extends BaseProvider {
options?: CompletionOptions,
quotaContext?: QuotaObservationContext,
): Promise<ChatCompletionResponse> {
- const { contents, systemInstruction } = await toGeminiContents(messages);
+ const translated = await toGeminiContents(messages);
+ const request = contentsForModel(modelId, translated.contents, translated.systemInstruction);
+ const modelOptions = optionsForModel(modelId, options);
- const tools = toGeminiTools(options?.tools);
+ const tools = toGeminiTools(modelOptions?.tools);
const body: Record<string, unknown> = {
- contents,
+ contents: request.contents,
generationConfig: {
- temperature: options?.temperature,
- maxOutputTokens: options?.max_tokens,
- topP: options?.top_p,
- stopSequences: toGeminiStopSequences(options?.stop),
- ...toGeminiExtendedConfig(options),
+ temperature: modelOptions?.temperature,
+ maxOutputTokens: modelOptions?.max_tokens,
+ topP: modelOptions?.top_p,
+ stopSequences: toGeminiStopSequences(modelOptions?.stop),
+ ...toGeminiExtendedConfig(modelOptions),
},
tools,
// functionCallingConfig is only valid when real function tools are present;
// a grounding-only request (just google_search) must omit it. (#59)
- toolConfig: hasFunctionDeclarations(tools) ? toGeminiToolConfig(options?.tool_choice) : undefined,
+ toolConfig: hasFunctionDeclarations(tools) ? toGeminiToolConfig(modelOptions?.tool_choice) : undefined,
};
- if (systemInstruction) body.systemInstruction = systemInstruction;
+ if (request.systemInstruction) body.systemInstruction = request.systemInstruction;
const url = `${API_BASE}/models/${modelId}:generateContent?key=${apiKey}`;
const res = await this.fetchWithTimeout(url, {
@@ -526,6 +592,7 @@ export class GoogleProvider extends BaseProvider {
const parts = candidate?.content?.parts;
const toolCalls = extractToolCalls(parts);
const text = extractText(parts);
+ const reasoningContent = extractReasoningContent(parts);
const usage: TokenUsage = {
prompt_tokens: data.usageMetadata?.promptTokenCount ?? 0,
@@ -543,6 +610,7 @@ export class GoogleProvider extends BaseProvider {
message: {
role: 'assistant',
content: text,
+ ...(reasoningContent ? { reasoning_content: reasoningContent } : {}),
...(toolCalls.length > 0 ? { tool_calls: toolCalls } : {}),
},
finish_reason: toolCalls.length > 0 ? 'tool_calls' : toGeminiFinishReason(candidate?.finishReason),
@@ -559,22 +627,24 @@ export class GoogleProvider extends BaseProvider {
options?: CompletionOptions,
quotaContext?: QuotaObservationContext,
): AsyncGenerator<ChatCompletionChunk> {
- const { contents, systemInstruction } = await toGeminiContents(messages);
+ const translated = await toGeminiContents(messages);
+ const request = contentsForModel(modelId, translated.contents, translated.systemInstruction);
+ const modelOptions = optionsForModel(modelId, options);
- const tools = toGeminiTools(options?.tools);
+ const tools = toGeminiTools(modelOptions?.tools);
const body: Record<string, unknown> = {
- contents,
+ contents: request.contents,
generationConfig: {
- temperature: options?.temperature,
- maxOutputTokens: options?.max_tokens,
- topP: options?.top_p,
- stopSequences: toGeminiStopSequences(options?.stop),
- ...toGeminiExtendedConfig(options),
+ temperature: modelOptions?.temperature,
+ maxOutputTokens: modelOptions?.max_tokens,
+ topP: modelOptions?.top_p,
+ stopSequences: toGeminiStopSequences(modelOptions?.stop),
+ ...toGeminiExtendedConfig(modelOptions),
},
tools,
- toolConfig: hasFunctionDeclarations(tools) ? toGeminiToolConfig(options?.tool_choice) : undefined,
+ toolConfig: hasFunctionDeclarations(tools) ? toGeminiToolConfig(modelOptions?.tool_choice) : undefined,
};
- if (systemInstruction) body.systemInstruction = systemInstruction;
+ if (request.systemInstruction) body.systemInstruction = request.systemInstruction;
const url = `${API_BASE}/models/${modelId}:streamGenerateContent?alt=sse&key=${apiKey}`;
const res = await this.fetchWithTimeout(url, {
@@ -652,6 +722,7 @@ export class GoogleProvider extends BaseProvider {
const parts = candidate?.content?.parts ?? [];
const text = extractText(parts);
+ const reasoningContent = extractReasoningContent(parts);
const toolCalls = extractToolCalls(parts).filter(call => {
const key = `${call.id}:${call.function.name}:${call.function.arguments}`;
if (seenToolCallKeys.has(key)) return false;
@@ -659,7 +730,7 @@ export class GoogleProvider extends BaseProvider {
return true;
});
- if ((text && text.length > 0) || toolCalls.length > 0) {
+ if ((text && text.length > 0) || (reasoningContent && reasoningContent.length > 0) || toolCalls.length > 0) {
sawToolCalls = sawToolCalls || toolCalls.length > 0;
yield {
id,
@@ -670,6 +741,7 @@ export class GoogleProvider extends BaseProvider {
index: 0,
delta: {
...(text ? { content: text } : {}),
+ ...(reasoningContent ? { reasoning_content: reasoningContent } : {}),
...(toolCalls.length > 0 ? { tool_calls: toolCalls } : {}),
},
finish_reason: null,
diff --git a/server/src/providers/openai-compat.ts b/server/src/providers/openai-compat.ts
index 65149bf6..7c6dddf8 100644
--- a/server/src/providers/openai-compat.ts
+++ b/server/src/providers/openai-compat.ts
@@ -22,8 +22,9 @@ export class OpenAICompatProvider extends BaseProvider {
private readonly baseUrl: string;
private readonly extraHeaders: Record<string, string>;
private readonly validateUrl?: string;
- /** Per-provider HTTP timeout override. Cloud APIs finish in ~15s; locally-hosted
- * inference (llama.cpp / vLLM on CPU) can take 30-120s for long prompts. Default 15000. */
+ /** Per-provider HTTP timeout override. OpenAI-compatible gateways often buffer
+ * non-streaming responses until generation completes, and reasoning models can
+ * take >15s before first byte. Default 60000. */
private readonly timeoutMs: number;
/** NVIDIA NIM models reject any request that permits parallel tool calls with
* `400 This model only supports single tool-calls at once!`. When set, pin
@@ -46,7 +47,7 @@ export class OpenAICompatProvider extends BaseProvider {
this.baseUrl = opts.baseUrl;
this.extraHeaders = opts.extraHeaders ?? {};
this.validateUrl = opts.validateUrl;
- this.timeoutMs = opts.timeoutMs ?? 15000;
+ this.timeoutMs = opts.timeoutMs ?? 60_000;
this.keyless = opts.keyless ?? false;
this.forceSingleToolCall = opts.forceSingleToolCall ?? false;
}
@@ -92,6 +93,47 @@ export class OpenAICompatProvider extends BaseProvider {
return this.keyless ? {} : { 'Authorization': `Bearer ${apiKey}` };
}
+ /** Mistral's OpenAI-compatible endpoint is strict about unknown nested fields
+ * and returns 422 for provider-private replay fields that other gateways
+ * ignore. Keep the OpenAI wire shape, but strip our internal reasoning /
+ * thought-signature extensions before sending to Mistral. */
+ private messagesForPlatform(messages: ChatMessage[]): ChatMessage[] {
+ if (this.platform !== 'mistral') return messages;
+
+ return messages.map((m) => {
+ if (m.role === 'assistant') {
+ return {
+ role: m.role,
+ content: m.content,
+ ...(m.name ? { name: m.name } : {}),
+ ...(m.tool_calls && m.tool_calls.length > 0 ? {
+ tool_calls: m.tool_calls.map((tc) => ({
+ id: tc.id,
+ type: tc.type,
+ function: {
+ name: tc.function.name,
+ arguments: tc.function.arguments,
+ },
+ })),
+ } : {}),
+ };
+ }
+ if (m.role === 'tool') {
+ return {
+ role: m.role,
+ content: m.content,
+ ...(m.tool_call_id ? { tool_call_id: m.tool_call_id } : {}),
+ ...(m.name ? { name: m.name } : {}),
+ };
+ }
+ return {
+ role: m.role,
+ content: m.content,
+ ...(m.name ? { name: m.name } : {}),
+ };
+ });
+ }
+
async chatCompletion(
apiKey: string,
messages: ChatMessage[],
@@ -108,7 +150,7 @@ export class OpenAICompatProvider extends BaseProvider {
},
body: JSON.stringify({
model: modelId,
- messages,
+ messages: this.messagesForPlatform(messages),
temperature: options?.temperature,
max_tokens: options?.max_tokens,
top_p: options?.top_p,
@@ -220,7 +262,7 @@ export class OpenAICompatProvider extends BaseProvider {
},
body: JSON.stringify({
model: modelId,
- messages,
+ messages: this.messagesForPlatform(messages),
temperature: options?.temperature,
max_tokens: options?.max_tokens,
top_p: options?.top_p,
@@ -231,7 +273,7 @@ export class OpenAICompatProvider extends BaseProvider {
...extendedBodyParams(this.platform, options),
stream: true,
}),
- }, this.timeoutMs);
+ }, options?.timeoutMs ?? this.timeoutMs);
recordQuotaObservationsFromResponse(res, {
platform: this.platform,
diff --git a/shared/types.ts b/shared/types.ts
index f5fad42a..4a3e4311 100644
--- a/shared/types.ts
+++ b/shared/types.ts
@@ -335,6 +335,7 @@ export interface ChatCompletionChunk {
delta: {
role?: 'assistant';
content?: string;
+ reasoning_content?: string;
tool_calls?: ChatToolCall[];
};
finish_reason: string | null;