7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.
TypeScript
feat(gateway): forward structured outputs + extended sampling params to providers (#514)
Tashfeen407001cda985
diff --git a/README.md b/README.mdindex f8f3fb7a..d139b521 100644--- a/README.md+++ b/README.md@@ -106,6 +106,7 @@ The full, always-current list lives at **[freellmapi.co/models](https://freellma- **Self-updating model catalog** — the router syncs a signed catalog from freellmapi.co twice a day: new models, quota changes, and provider quirk fixes land in your install automatically. See [Premium (live catalog)](#premium-live-catalog).- **Streaming and non-streaming** — Server-Sent Events for `stream: true`, JSON response otherwise. Every provider adapter implements both.- **Tool calling** — OpenAI-style `tools` / `tool_choice` requests are passed through, and assistant `tool_calls` + `tool` role follow-up messages round-trip across providers. Models that emit tool calls as plain text instead of structured JSON are rescued into real `tool_calls` automatically, and tool requests only route to models that actually support them.+- **Structured outputs & full sampling passthrough** — `response_format` (`json_object` / `json_schema`, translated to Gemini's native `responseSchema`), plus `seed`, `top_k`, `min_p`, presence/frequency/repetition penalties, `logit_bias`, `logprobs`, and the `max_completion_tokens` alias. Params a provider is known to reject are dropped per platform (Mistral's strict API, Groq's logprobs family…), and every model advertises its honest list in `/v1/models` `supported_parameters`.- **Embeddings** — `/v1/embeddings` with family-based routing, including custom OpenAI-compatible embedding endpoints: failover only ever happens between providers serving the *same* model (vectors from different models are incompatible), never across models. See [Embeddings](#embeddings).- **Automatic fallover** — If the chosen provider returns a 429, 5xx, or times out, the router skips it, puts the key on a short cooldown, and retries on the next model in your fallback chain (up to 20 attempts, bounded by a wall-clock retry budget). A dead key rotates to its siblings instead of failing the request, and exhaustion errors carry the full attempt trail so you can see exactly what was tried.- **Smart routing, six strategies** — the chain is ranked by a selectable strategy: `priority` (your manual order), `balanced`, `smartest`, `fastest`, `reliable`, or `custom` with your own weight mix. Scores come from live per-model measurements (speed, capability, rate-limit headroom, recent errors) with a Thompson-sampling bandit under the hood; one-click sort presets reorder the chain from the dashboard.diff --git a/server/src/__tests__/lib/sampling-params.test.ts b/server/src/__tests__/lib/sampling-params.test.tsnew file mode 100644index 00000000..e26937ca--- /dev/null+++ b/server/src/__tests__/lib/sampling-params.test.ts@@ -0,0 +1,105 @@+import { describe, it, expect } from 'vitest';+import {+ pickSamplingParams,+ extendedBodyParams,+ supportedParametersFor,+ supportedParametersForPlatforms,+ EXTENDED_SAMPLING_KEYS,+} from '../../lib/sampling-params.js';++describe('pickSamplingParams', () => {+ it('forwards set values and skips undefined/null (#200 explicit-null tolerance)', () => {+ expect(pickSamplingParams({+ seed: 42,+ top_k: 50,+ presence_penalty: 0.5,+ frequency_penalty: null,+ logit_bias: undefined,+ logprobs: true,+ top_logprobs: 5,+ })).toEqual({ seed: 42, top_k: 50, presence_penalty: 0.5, logprobs: true, top_logprobs: 5 });+ });++ it('drops response_format {type:"text"} (the default; some providers 400 on it)', () => {+ expect(pickSamplingParams({ response_format: { type: 'text' } })).toEqual({});+ expect(pickSamplingParams({ response_format: { type: 'json_object' } }))+ .toEqual({ response_format: { type: 'json_object' } });+ });++ it('keeps a full json_schema response_format intact', () => {+ const rf = { type: 'json_schema', json_schema: { name: 'answer', strict: true, schema: { type: 'object' } } };+ expect(pickSamplingParams({ response_format: rf })).toEqual({ response_format: rf });+ });++ it('returns {} for an empty body', () => {+ expect(pickSamplingParams({})).toEqual({});+ });+});++describe('extendedBodyParams (per-platform policy)', () => {+ const allSet = {+ top_k: 40, min_p: 0.05, seed: 7, presence_penalty: 1, frequency_penalty: -1,+ repetition_penalty: 1.1, logit_bias: { '50256': -100 }, logprobs: true, top_logprobs: 3,+ response_format: { type: 'json_object' as const },+ };++ it('forwards everything for platforms without a policy', () => {+ const body = extendedBodyParams('cerebras', allSet);+ expect(Object.keys(body).sort()).toEqual([...EXTENDED_SAMPLING_KEYS].sort());+ });++ it('mistral: renames seed to random_seed and drops the unsupported set', () => {+ const body = extendedBodyParams('mistral', allSet);+ expect(body.random_seed).toBe(7);+ expect(body).not.toHaveProperty('seed');+ for (const k of ['top_k', 'min_p', 'repetition_penalty', 'logit_bias', 'logprobs', 'top_logprobs']) {+ expect(body).not.toHaveProperty(k);+ }+ expect(body.presence_penalty).toBe(1);+ expect(body.response_format).toEqual({ type: 'json_object' });+ });++ it('groq: drops the logprobs family, keeps seed and response_format', () => {+ const body = extendedBodyParams('groq', allSet);+ expect(body).not.toHaveProperty('logprobs');+ expect(body).not.toHaveProperty('top_logprobs');+ expect(body).not.toHaveProperty('logit_bias');+ expect(body.seed).toBe(7);+ expect(body.response_format).toEqual({ type: 'json_object' });+ });++ it('aihorde: drops the entire extended set', () => {+ expect(extendedBodyParams('aihorde', allSet)).toEqual({});+ });++ it('returns {} for undefined options and for options with nothing set', () => {+ expect(extendedBodyParams('groq', undefined)).toEqual({});+ expect(extendedBodyParams('groq', {})).toEqual({});+ });+});++describe('supportedParametersFor / supportedParametersForPlatforms', () => {+ it('advertises the base set minus the platform droplist', () => {+ const groq = supportedParametersFor('groq');+ expect(groq).toContain('seed');+ expect(groq).toContain('response_format');+ expect(groq).not.toContain('logprobs');+ expect(groq).not.toContain('logit_bias');+ });++ it('appends tool params only for tool-capable models', () => {+ expect(supportedParametersFor('groq', { tools: true })).toContain('tools');+ expect(supportedParametersFor('groq', { tools: false })).not.toContain('tools');+ });++ it('intersects across a unify group\'s platforms', () => {+ const both = supportedParametersForPlatforms(['groq', 'mistral']);+ // groq drops logprobs; mistral drops top_k — neither survives the intersection.+ expect(both).not.toContain('logprobs');+ expect(both).not.toContain('top_k');+ // seed survives both (mistral renames it on the wire but honors it).+ expect(both).toContain('seed');+ // single platform = its own list+ expect(supportedParametersForPlatforms(['groq'])).toEqual(supportedParametersFor('groq'));+ });+});diff --git a/server/src/__tests__/providers/google-params.test.ts b/server/src/__tests__/providers/google-params.test.tsnew file mode 100644index 00000000..195d54e1--- /dev/null+++ b/server/src/__tests__/providers/google-params.test.ts@@ -0,0 +1,54 @@+import { describe, it, expect } from 'vitest';+import { toGeminiExtendedConfig } from '../../providers/google.js';++describe('toGeminiExtendedConfig', () => {+ it('maps top_k/seed/penalties to generationConfig names', () => {+ const cfg = toGeminiExtendedConfig({+ top_k: 40, seed: 7, presence_penalty: 0.5, frequency_penalty: -0.5,+ });+ expect(cfg).toEqual({ topK: 40, seed: 7, presencePenalty: 0.5, frequencyPenalty: -0.5 });+ });++ it('json_object → responseMimeType only', () => {+ const cfg = toGeminiExtendedConfig({ response_format: { type: 'json_object' } });+ expect(cfg.responseMimeType).toBe('application/json');+ expect(cfg).not.toHaveProperty('responseSchema');+ });++ it('json_schema → responseMimeType + sanitized responseSchema', () => {+ const cfg = toGeminiExtendedConfig({+ response_format: {+ type: 'json_schema',+ json_schema: {+ name: 'answer',+ schema: {+ type: 'object',+ additionalProperties: false, // Gemini rejects this key — must be stripped+ properties: { city: { type: 'string' } },+ required: ['city'],+ },+ },+ },+ });+ expect(cfg.responseMimeType).toBe('application/json');+ const schema = cfg.responseSchema as Record<string, unknown>;+ expect(schema).toBeDefined();+ expect(JSON.stringify(schema)).not.toContain('additionalProperties');+ expect((schema.properties as Record<string, unknown>).city).toBeDefined();+ });++ it('skips JSON output entirely when tools are present (Gemini rejects the combination)', () => {+ const cfg = toGeminiExtendedConfig({+ response_format: { type: 'json_object' },+ tools: [{ type: 'function', function: { name: 'f', parameters: {} } }],+ });+ expect(cfg).not.toHaveProperty('responseMimeType');+ expect(cfg).not.toHaveProperty('responseSchema');+ });++ it('returns all-undefined fields for empty options (JSON.stringify drops them)', () => {+ const cfg = toGeminiExtendedConfig(undefined);+ expect(cfg.topK).toBeUndefined();+ expect(cfg.responseMimeType).toBeUndefined();+ });+});diff --git a/server/src/__tests__/providers/openai-compat.test.ts b/server/src/__tests__/providers/openai-compat.test.tsindex ad4e9acc..165d184d 100644--- a/server/src/__tests__/providers/openai-compat.test.ts+++ b/server/src/__tests__/providers/openai-compat.test.ts@@ -474,3 +474,70 @@ describe('OpenAICompatProvider - platform instances', () => {});});});++describe('extended sampling param passthrough', () => {+ const mockOk = () => ({+ ok: true,+ json: () => Promise.resolve({+ id: 'x', object: 'chat.completion', created: 1, model: 'm',+ choices: [{ index: 0, message: { role: 'assistant', content: 'hi' }, finish_reason: 'stop' }],+ usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 },+ }),+ }) as any;++ const extended = {+ seed: 42, top_k: 40, min_p: 0.05, presence_penalty: 0.5, frequency_penalty: -0.25,+ logit_bias: { '50256': -100 }, logprobs: true, top_logprobs: 3,+ response_format: { type: 'json_schema' as const, json_schema: { name: 'a', schema: { type: 'object' } } },+ };++ it('forwards the full extended set for a policy-free platform', async () => {+ let body: any = null;+ vi.spyOn(global, 'fetch').mockImplementation(async (_url, init) => {+ body = JSON.parse((init as any).body);+ return mockOk();+ });+ const p = new OpenAICompatProvider({ platform: 'cerebras', name: 'T', baseUrl: 'https://x/v1' });+ await p.chatCompletion('k', [{ role: 'user', content: 'q' }], 'm', extended);++ expect(body.seed).toBe(42);+ expect(body.top_k).toBe(40);+ expect(body.min_p).toBe(0.05);+ expect(body.logit_bias).toEqual({ '50256': -100 });+ expect(body.logprobs).toBe(true);+ expect(body.top_logprobs).toBe(3);+ expect(body.response_format.type).toBe('json_schema');+ });++ it('applies the mistral policy on the wire: random_seed rename, strict-API params dropped', async () => {+ let body: any = null;+ vi.spyOn(global, 'fetch').mockImplementation(async (_url, init) => {+ body = JSON.parse((init as any).body);+ return mockOk();+ });+ const p = new OpenAICompatProvider({ platform: 'mistral', name: 'T', baseUrl: 'https://x/v1' });+ await p.chatCompletion('k', [{ role: 'user', content: 'q' }], 'm', extended);++ expect(body.random_seed).toBe(42);+ expect(body).not.toHaveProperty('seed');+ expect(body).not.toHaveProperty('top_k');+ expect(body).not.toHaveProperty('logit_bias');+ expect(body).not.toHaveProperty('logprobs');+ expect(body.presence_penalty).toBe(0.5);+ expect(body.response_format.type).toBe('json_schema');+ });++ it('sends no extended keys when none were requested (undefined stays omitted)', async () => {+ let raw = '';+ vi.spyOn(global, 'fetch').mockImplementation(async (_url, init) => {+ raw = (init as any).body;+ return mockOk();+ });+ const p = new OpenAICompatProvider({ platform: 'groq', name: 'T', baseUrl: 'https://x/v1' });+ await p.chatCompletion('k', [{ role: 'user', content: 'q' }], 'm', { temperature: 0.7 });++ expect(raw).not.toContain('seed');+ expect(raw).not.toContain('response_format');+ expect(raw).not.toContain('logit_bias');+ });+});diff --git a/server/src/lib/sampling-params.ts b/server/src/lib/sampling-params.tsnew file mode 100644index 00000000..6b431ee9--- /dev/null+++ b/server/src/lib/sampling-params.ts@@ -0,0 +1,183 @@+// Extended sampling / output-shape parameters forwarded to providers.+//+// Until now only temperature / max_tokens / top_p / stop reached upstream —+// everything else a client sent (seed, penalties, logit_bias, logprobs,+// response_format…) was validated away by the request schema and silently+// dropped, so "OpenAI-compatible" was quietly narrower than it claimed+// (structured outputs simply did not work). This module is the single source+// of truth for the extended set:+//+// - the zod fields each surface spreads into its request schema,+// - the pick helper that turns a parsed body into CompletionOptions fields,+// - the per-platform support policy (which params a provider is known to+// reject or ignore) used both by the adapters (drop before send) and by+// /v1/models `supported_parameters` (advertise per model).+//+// Forward-by-default: most OpenAI-compatible providers ignore unknown body+// fields, so the default is to send everything and let the per-platform+// droplist name the documented exceptions (Mistral 422s on unknown keys,+// Groq 400s on the logprobs family, Azure-backed GitHub Models rejects+// non-Azure knobs). A provider that still 400s fails over like any other+// provider-invalid request and shows up in the attempt trail — and its+// droplist entry is one line to add.++import { z } from 'zod';++// Every field is `.nullable()` because real clients serialize their whole+// request struct and send explicit nulls for unset knobs (#200); null is+// treated as absent and never forwarded.+export const samplingParamSchemaFields = {+ top_k: z.number().int().min(1).nullable().optional(),+ min_p: z.number().min(0).max(1).nullable().optional(),+ seed: z.number().int().nullable().optional(),+ presence_penalty: z.number().min(-2).max(2).nullable().optional(),+ frequency_penalty: z.number().min(-2).max(2).nullable().optional(),+ repetition_penalty: z.number().positive().nullable().optional(),+ logit_bias: z.record(z.string(), z.number()).nullable().optional(),+ logprobs: z.boolean().nullable().optional(),+ top_logprobs: z.number().int().min(0).max(20).nullable().optional(),+ response_format: z.object({+ type: z.enum(['text', 'json_object', 'json_schema']),+ json_schema: z.object({+ name: z.string().optional(),+ strict: z.boolean().nullable().optional(),+ schema: z.record(z.string(), z.unknown()).optional(),+ }).passthrough().optional(),+ }).passthrough().nullable().optional(),+ // OpenAI's newer alias for max_tokens; surfaces resolve it into max_tokens+ // themselves (it is not a forwarded param of its own).+ max_completion_tokens: z.number().int().nullable().optional(),+} as const;++export interface ResponseFormat {+ type: 'json_object' | 'json_schema';+ json_schema?: {+ name?: string;+ strict?: boolean | null;+ schema?: Record<string, unknown>;+ } & Record<string, unknown>;+}++export interface ExtendedSamplingOptions {+ top_k?: number;+ min_p?: number;+ seed?: number;+ presence_penalty?: number;+ frequency_penalty?: number;+ repetition_penalty?: number;+ logit_bias?: Record<string, number>;+ logprobs?: boolean;+ top_logprobs?: number;+ response_format?: ResponseFormat;+}++export const EXTENDED_SAMPLING_KEYS = [+ 'top_k', 'min_p', 'seed', 'presence_penalty', 'frequency_penalty',+ 'repetition_penalty', 'logit_bias', 'logprobs', 'top_logprobs',+ 'response_format',+] as const;+export type ExtendedSamplingKey = typeof EXTENDED_SAMPLING_KEYS[number];++type ParsedSamplingBody = {+ [K in ExtendedSamplingKey]?: unknown;+};++/**+ * Turn a schema-parsed request body into the extended CompletionOptions+ * fields: nulls dropped, `response_format: {type:'text'}` dropped (it is the+ * default and some providers 400 on receiving it explicitly), everything else+ * forwarded as-is.+ */+export function pickSamplingParams(body: ParsedSamplingBody): ExtendedSamplingOptions {+ const out: Record<string, unknown> = {};+ for (const key of EXTENDED_SAMPLING_KEYS) {+ const value = body[key];+ if (value === undefined || value === null) continue;+ if (key === 'response_format' && (value as { type?: string }).type === 'text') continue;+ out[key] = value;+ }+ return out as ExtendedSamplingOptions;+}++// ── Per-platform support policy ──────────────────────────────────────────────+// `drop` = params this platform is KNOWN to reject (or that would corrupt the+// request); stripped by the adapter before send and omitted from the model's+// advertised `supported_parameters`. `rename` = same param, different wire+// name. Platforms not listed forward everything.+//+// Deliberately conservative: only documented/observed rejections are listed,+// because wrongly dropping a working param is as bad as a 400. Findings from+// live sweeps go here, one line each.+export interface PlatformParamPolicy {+ drop?: readonly ExtendedSamplingKey[];+ rename?: Readonly<Partial<Record<ExtendedSamplingKey, string>>>;+}++export const PLATFORM_PARAM_POLICIES: Record<string, PlatformParamPolicy> = {+ // Mistral's API is strict (422 on unknown body keys) and names its seed+ // `random_seed`. It has no top_k/min_p/logit_bias/logprobs equivalents.+ mistral: {+ drop: ['top_k', 'min_p', 'repetition_penalty', 'logit_bias', 'logprobs', 'top_logprobs'],+ rename: { seed: 'random_seed' },+ },+ // Groq documents logprobs / top_logprobs / logit_bias as unsupported and+ // rejects requests that include them.+ groq: { drop: ['logprobs', 'top_logprobs', 'logit_bias'] },+ // GitHub Models sits on Azure OpenAI, which 400s "Unrecognized request+ // argument" for knobs outside the OpenAI set.+ github: { drop: ['top_k', 'min_p', 'repetition_penalty'] },+ // Gemini's generationConfig has no equivalents for these; the adapter+ // translates the rest natively (topK, seed, penalties, responseSchema).+ google: { drop: ['min_p', 'repetition_penalty', 'logit_bias', 'logprobs', 'top_logprobs'] },+ // Cohere's OpenAI-compat endpoint covers seed/penalties/response_format;+ // the rest have no mapping there.+ cohere: { drop: ['top_k', 'min_p', 'repetition_penalty', 'logit_bias', 'logprobs', 'top_logprobs'] },+ // Workers AI's OpenAI-compat endpoint parses a known subset; send only what+ // it understands.+ cloudflare: { drop: ['min_p', 'logit_bias', 'logprobs', 'top_logprobs'] },+ // AI Horde builds its own payload format; none of the extended set maps.+ aihorde: { drop: [...EXTENDED_SAMPLING_KEYS] },+};++/**+ * Build the extended wire-body fields for one platform: policy droplist+ * applied, renames applied, undefined skipped. Adapters spread the result+ * into their OpenAI-shaped request bodies.+ */+export function extendedBodyParams(platform: string, options: ExtendedSamplingOptions | undefined): Record<string, unknown> {+ if (!options) return {};+ const policy = PLATFORM_PARAM_POLICIES[platform];+ const dropped = new Set<string>(policy?.drop ?? []);+ const out: Record<string, unknown> = {};+ for (const key of EXTENDED_SAMPLING_KEYS) {+ if (dropped.has(key)) continue;+ const value = (options as Record<string, unknown>)[key];+ if (value === undefined) continue;+ out[policy?.rename?.[key] ?? key] = value;+ }+ return out;+}++/** The advertised parameter list for a model on `platform` — the base set+ * every surface supports, plus tools when the model does, minus the+ * platform's droplist. */+export function supportedParametersFor(platform: string, caps: { tools?: boolean } = {}): string[] {+ const policy = PLATFORM_PARAM_POLICIES[platform];+ const dropped = new Set<string>(policy?.drop ?? []);+ const params = [+ 'temperature', 'top_p', 'max_tokens', 'max_completion_tokens', 'stop', 'stream',+ ...EXTENDED_SAMPLING_KEYS.filter(k => !dropped.has(k)),+ ];+ if (caps.tools) params.push('tools', 'tool_choice', 'parallel_tool_calls');+ return params;+}++/** For a model served by several platforms (a unify group): the INTERSECTION+ * of the members' supported sets — a param is only advertised when every+ * platform the router might pick honors it. */+export function supportedParametersForPlatforms(platforms: string[], caps: { tools?: boolean } = {}): string[] {+ if (platforms.length === 0) return supportedParametersFor('', caps);+ const [first, ...rest] = platforms.map(p => supportedParametersFor(p, caps));+ const restSets = rest.map(list => new Set(list));+ return first.filter(param => restSets.every(s => s.has(param)));+}diff --git a/server/src/providers/base.ts b/server/src/providers/base.tsindex e8ab7efc..85f19ffd 100644--- a/server/src/providers/base.ts+++ b/server/src/providers/base.ts@@ -7,6 +7,7 @@ import type {Platform,} from '@freellmapi/shared/types.js';import type { QuotaObservationContext } from '../services/provider-quota.js';+import type { ExtendedSamplingOptions } from '../lib/sampling-params.js';import { proxyFetch } from '../lib/proxy.js';/** A provider HTTP error carrying the upstream status and, when the response@@ -39,7 +40,10 @@ export function providerHttpError(res: Response, message: string): ProviderHttpEreturn err;}-export interface CompletionOptions {+// Extended sampling knobs (top_k, seed, penalties, logit_bias, logprobs,+// response_format…) ride along via ExtendedSamplingOptions; adapters forward+// them per the platform policy in lib/sampling-params.ts.+export interface CompletionOptions extends ExtendedSamplingOptions {model?: string;temperature?: number;max_tokens?: number;diff --git a/server/src/providers/cloudflare.ts b/server/src/providers/cloudflare.tsindex 19d6cf43..7f940bf0 100644--- a/server/src/providers/cloudflare.ts+++ b/server/src/providers/cloudflare.ts@@ -4,6 +4,7 @@ import type {ChatCompletionChunk,} from '@freellmapi/shared/types.js';import { BaseProvider, providerHttpError, type CompletionOptions } from './base.js';+import { extendedBodyParams } from '../lib/sampling-params.js';import { contentToString } from '../lib/content.js';import { recordQuotaObservationsFromResponse, type QuotaObservationContext } from '../services/provider-quota.js';@@ -56,6 +57,7 @@ export class CloudflareProvider extends BaseProvider {tools: options?.tools,tool_choice: options?.tool_choice,parallel_tool_calls: options?.parallel_tool_calls,+ ...extendedBodyParams(this.platform, options),}),});recordQuotaObservationsFromResponse(res, {@@ -103,6 +105,7 @@ export class CloudflareProvider extends BaseProvider {tools: options?.tools,tool_choice: options?.tool_choice,parallel_tool_calls: options?.parallel_tool_calls,+ ...extendedBodyParams(this.platform, options),stream: true,}),});diff --git a/server/src/providers/cohere.ts b/server/src/providers/cohere.tsindex c2cc45fc..b5f8f68b 100644--- a/server/src/providers/cohere.ts+++ b/server/src/providers/cohere.ts@@ -5,6 +5,7 @@ import type {ChatToolDefinition,} from '@freellmapi/shared/types.js';import { BaseProvider, providerHttpError, type CompletionOptions } from './base.js';+import { extendedBodyParams } from '../lib/sampling-params.js';import { flattenMessageContent } from '../lib/content.js';import { recordQuotaObservationsFromResponse, type QuotaObservationContext } from '../services/provider-quota.js';import { stripSchemaKeys } from '../lib/tool-args.js';@@ -48,6 +49,7 @@ export class CohereProvider extends BaseProvider {stop: options?.stop,tools: sanitizeCohereTools(options?.tools),tool_choice: options?.tool_choice,+ ...extendedBodyParams(this.platform, options),};const res = await this.fetchWithTimeout(`${API_BASE}/chat/completions`, {@@ -93,6 +95,7 @@ export class CohereProvider extends BaseProvider {stop: options?.stop,tools: sanitizeCohereTools(options?.tools),tool_choice: options?.tool_choice,+ ...extendedBodyParams(this.platform, options),stream: true,};diff --git a/server/src/providers/google.ts b/server/src/providers/google.tsindex 3b6ae25e..c6cf7f39 100644--- a/server/src/providers/google.ts+++ b/server/src/providers/google.ts@@ -185,6 +185,32 @@ function sanitizeForGeminiSchema(schema: unknown, insidePropertiesMap: boolean):// declaration, and can ride alongside real function tools in the same array. (#59)const GROUNDING_TOOL_NAMES = new Set(['google_search', 'googlesearch', 'google_search_retrieval']);+/**+ * Extended generationConfig knobs translated from the OpenAI wire: topK,+ * seed, penalties, and structured output. JSON output conflicts with function+ * calling on Gemini ("Function calling with a response mime type:+ * 'application/json' is unsupported"), so response_format is only applied on+ * tool-free requests. Params Gemini has no equivalent for (min_p, logit_bias,+ * logprobs…) are dropped by the platform policy in lib/sampling-params.ts and+ * are ignored here. Exported for tests.+ */+export function toGeminiExtendedConfig(options?: CompletionOptions): Record<string, unknown> {+ const out: Record<string, unknown> = {+ topK: options?.top_k,+ seed: options?.seed,+ presencePenalty: options?.presence_penalty,+ frequencyPenalty: options?.frequency_penalty,+ };+ const rf = options?.response_format;+ const hasTools = (options?.tools?.length ?? 0) > 0;+ if (rf && !hasTools) {+ out.responseMimeType = 'application/json';+ const schema = rf.type === 'json_schema' ? rf.json_schema?.schema : undefined;+ if (schema) out.responseSchema = sanitizeForGemini(schema);+ }+ return out;+}+function toGeminiTools(tools?: ChatToolDefinition[]): Array<Record<string, unknown>> | undefined {if (!tools || tools.length === 0) return undefined;@@ -460,6 +486,7 @@ export class GoogleProvider extends BaseProvider {maxOutputTokens: options?.max_tokens,topP: options?.top_p,stopSequences: toGeminiStopSequences(options?.stop),+ ...toGeminiExtendedConfig(options),},tools,// functionCallingConfig is only valid when real function tools are present;@@ -537,6 +564,7 @@ export class GoogleProvider extends BaseProvider {maxOutputTokens: options?.max_tokens,topP: options?.top_p,stopSequences: toGeminiStopSequences(options?.stop),+ ...toGeminiExtendedConfig(options),},tools,toolConfig: hasFunctionDeclarations(tools) ? toGeminiToolConfig(options?.tool_choice) : undefined,diff --git a/server/src/providers/openai-compat.ts b/server/src/providers/openai-compat.tsindex e5d42e59..65149bf6 100644--- a/server/src/providers/openai-compat.ts+++ b/server/src/providers/openai-compat.ts@@ -6,6 +6,7 @@ import type {Platform,} from '@freellmapi/shared/types.js';import { BaseProvider, providerHttpError, type CompletionOptions } from './base.js';+import { extendedBodyParams } from '../lib/sampling-params.js';import { rescueInlineToolCalls } from '../lib/tool-call-rescue.js';import { repairToolArguments, toolSchemaMap } from '../lib/tool-args.js';import { recordQuotaObservationsFromResponse, type QuotaObservationContext } from '../services/provider-quota.js';@@ -115,6 +116,7 @@ export class OpenAICompatProvider extends BaseProvider {tools: options?.tools,tool_choice: options?.tool_choice,parallel_tool_calls: this.resolveParallelToolCalls(options),+ ...extendedBodyParams(this.platform, options),}),}, options?.timeoutMs ?? this.timeoutMs);@@ -226,6 +228,7 @@ export class OpenAICompatProvider extends BaseProvider {tools: options?.tools,tool_choice: options?.tool_choice,parallel_tool_calls: this.resolveParallelToolCalls(options),+ ...extendedBodyParams(this.platform, options),stream: true,}),}, this.timeoutMs);diff --git a/server/src/routes/anthropic.ts b/server/src/routes/anthropic.tsindex 4c57a1a3..89f26533 100644--- a/server/src/routes/anthropic.ts+++ b/server/src/routes/anthropic.ts@@ -79,6 +79,9 @@ const messagesSchema = z.object({system: z.union([z.string(), z.array(contentBlockSchema)]).optional(),temperature: z.number().min(0).max(2).optional(),top_p: z.number().min(0).max(1).optional(),+ // Anthropic's native top_k — forwarded to providers that support it via the+ // platform policy in lib/sampling-params.ts.+ top_k: z.number().int().min(1).nullable().optional(),stream: z.boolean().optional(),stop_sequences: z.array(z.string()).optional(),tools: z.array(anthropicToolSchema).optional(),@@ -370,7 +373,7 @@ anthropicRouter.post('/messages', async (req: Request, res: Response) => {const { temperature, top_p, stream } = body;const { messages, tools, tool_choice, hasImage, wantsTools } = convertRequest(body);- const completionOptions = { temperature, max_tokens, top_p, tools, tool_choice };+ const completionOptions = { temperature, max_tokens, top_p, top_k: body.top_k ?? undefined, tools, tool_choice };const estimatedInputTokens = estimateTokens(messages);const imageCount = messages.reduce((n, m) =>diff --git a/server/src/routes/proxy.ts b/server/src/routes/proxy.tsindex 52d23494..1018ced9 100644--- a/server/src/routes/proxy.ts+++ b/server/src/routes/proxy.ts@@ -19,6 +19,7 @@ import { logRequest } from '../lib/request-log.js';import { parseCacheDirective, cacheActive, isCacheableTemperature, computeCacheKey, getCachedResponse, storeCachedResponse } from '../services/cache.js';import { runFallbackLoop, newFallbackState, recordUpstreamSuccess, exhaustedRetryError } from '../lib/fallback-loop.js';import { applyTokenBudget, tokenBudgetMessage } from '../lib/guardrails.js';+import { samplingParamSchemaFields, pickSamplingParams, supportedParametersForPlatforms } from '../lib/sampling-params.js';import type { Platform } from '@freellmapi/shared/types.js';import { inferQuotaPoolKey, type QuotaObservationContext } from '../services/provider-quota.js';import { isUnifyEnabled, getModelGroups, resolveRequestedIdToMembers } from '../services/model-groups.js';@@ -260,6 +261,11 @@ proxyRouter.get('/models', (req: Request, res: Response) => {// Non-standard but additive: OpenAI clients ignore unknown fields.available: m.available === 1,unavailable_reason: m.available === 1 ? null : (m.enabled === 1 ? 'no_key' : 'disabled'),+ // OpenRouter's field name; agents use it to pick knobs per model. For+ // a unify group this is the intersection over member platforms — a+ // param is only advertised when every platform the router might pick+ // honors it.+ supported_parameters: supportedParametersForPlatforms(m.platforms, { tools: m.supportsTools }),})),],});@@ -415,6 +421,10 @@ const chatCompletionSchema = z.object({// Fusion config — only meaningful when `model` is the virtual "fusion" id.// Ignored for every other model. See services/fusion.ts.fusion: fusionConfigSchema.optional(),+ // Extended sampling + structured-output params (top_k, seed, penalties,+ // logit_bias, logprobs, response_format, max_completion_tokens…), forwarded+ // per the platform policy in lib/sampling-params.ts.+ ...samplingParamSchemaFields,});// Upstream-error classifiers live in lib/error-classify.ts so the fusion@@ -976,10 +986,16 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {// Agent-tolerant knob normalization (#200): max_tokens <= 0 means "no// limit" in several clients → unset; tool_choice 'any' is OpenAI's// 'required'; tool definitions get their 'function' type re-defaulted.- // `let`: the token-budget guardrail below may cap an absent max_tokens to- // the budget remainder before the options objects are built from it.- let max_tokens = parsed.data.max_tokens != null && parsed.data.max_tokens > 0- ? parsed.data.max_tokens : undefined;+ // `max_completion_tokens` is OpenAI's newer alias — honored when max_tokens+ // itself is absent. `let`: the token-budget guardrail below may cap an+ // absent max_tokens to the budget remainder before the options objects are+ // built from it.+ const requestedMaxTokens = parsed.data.max_tokens ?? parsed.data.max_completion_tokens;+ let max_tokens = requestedMaxTokens != null && requestedMaxTokens > 0+ ? requestedMaxTokens : undefined;+ // Extended sampling/output params (seed, penalties, response_format…),+ // spread into every options object below — including fusion fan-out.+ const samplingParams = pickSamplingParams(parsed.data);const stop = providerSafeStop(parsed.data.stop);const tool_choice = parsed.data.tool_choice === 'any' ? 'required' as const : parsed.data.tool_choice ?? undefined;const tools = parsed.data.tools?.map(t => ({ ...t, type: 'function' as const }));@@ -1149,7 +1165,7 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {res.status(422).json({ error: { message: 'Fusion does not support image input yet. Use a vision model directly.', type: 'invalid_request_error', code: 'fusion_no_vision' } });return;}- const fusionOptions = { temperature, max_tokens, top_p, stop, tools, tool_choice, parallel_tool_calls };+ const fusionOptions = { temperature, max_tokens, top_p, stop, tools, tool_choice, parallel_tool_calls, ...samplingParams };const fusionConfig = parsed.data.fusion ?? {};if (stream) {@@ -1483,7 +1499,7 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {try {const gen = route.provider.streamChatCompletion(route.apiKey, outboundMessages, route.modelId,- { temperature, max_tokens, top_p, stop, tools, tool_choice, parallel_tool_calls },+ { temperature, max_tokens, top_p, stop, tools, tool_choice, parallel_tool_calls, ...samplingParams },quotaContextForRoute(route, 'chat/completions'),);@@ -1685,7 +1701,7 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {} else {const result = await route.provider.chatCompletion(route.apiKey, outboundMessages, route.modelId,- { temperature, max_tokens, top_p, stop, tools, tool_choice, parallel_tool_calls },+ { temperature, max_tokens, top_p, stop, tools, tool_choice, parallel_tool_calls, ...samplingParams },quotaContextForRoute(route, 'chat/completions'),);diff --git a/server/src/routes/responses.ts b/server/src/routes/responses.tsindex 9e4ff7ea..55c75a0e 100644--- a/server/src/routes/responses.ts+++ b/server/src/routes/responses.ts@@ -25,6 +25,7 @@ import {} from './proxy.js';import { runFallbackLoop, newFallbackState, recordUpstreamSuccess } from '../lib/fallback-loop.js';import { applyTokenBudget, tokenBudgetMessage } from '../lib/guardrails.js';+import { samplingParamSchemaFields, pickSamplingParams, type ResponseFormat } from '../lib/sampling-params.js';import { sanitizeProviderErrorMessage } from '../lib/error-redaction.js';import { inferQuotaPoolKey, type QuotaObservationContext } from '../services/provider-quota.js';@@ -119,6 +120,18 @@ const responsesRequestSchema = z.object({z.object({ type: z.literal('function'), name: z.string() }).passthrough(),]).optional(),parallel_tool_calls: z.boolean().nullable().optional(),+ // Extended sampling params, validated the same way as /chat/completions.+ // Responses clients express structured output as `text.format` rather than+ // `response_format` — mapped where completionOpts is built.+ ...samplingParamSchemaFields,+ text: z.object({+ format: z.object({+ type: z.enum(['text', 'json_object', 'json_schema']),+ name: z.string().optional(),+ strict: z.boolean().nullable().optional(),+ schema: z.record(z.string(), z.unknown()).optional(),+ }).passthrough().optional(),+ }).passthrough().nullable().optional(),}).passthrough();type ResponsesRequest = z.infer<typeof responsesRequestSchema>;@@ -327,6 +340,17 @@ responsesRouter.post('/responses', async (req: Request, res: Response) => {// the way back out (see lib/tool-args.ts).const toolSchemas = toolSchemaMap(tools);const tool_choice = tools?.length ? toChatToolChoice(reqData.tool_choice) : undefined;+ // Responses-API structured output arrives as `text.format`; translate it to+ // the internal response_format shape (an explicit response_format on the+ // body, unusual for this surface but valid, wins).+ const samplingParams = pickSamplingParams(reqData);+ const textFormat = reqData.text?.format;+ if (!samplingParams.response_format && textFormat && textFormat.type !== 'text') {+ samplingParams.response_format = textFormat.type === 'json_schema'+ ? { type: 'json_schema', json_schema: { name: textFormat.name, strict: textFormat.strict, schema: textFormat.schema } }+ : { type: 'json_object' } as ResponseFormat;+ }+const completionOpts = {temperature: reqData.temperature ?? undefined,max_tokens: reqData.max_output_tokens ?? undefined,@@ -334,6 +358,7 @@ responsesRouter.post('/responses', async (req: Request, res: Response) => {tools,tool_choice,parallel_tool_calls: reqData.parallel_tool_calls ?? undefined,+ ...samplingParams,};const estimatedInputTokens = messages.reduce(diff --git a/server/src/services/model-listing.ts b/server/src/services/model-listing.tsindex 5aeeef2d..30ce7896 100644--- a/server/src/services/model-listing.ts+++ b/server/src/services/model-listing.ts@@ -15,6 +15,11 @@ export interface NormalizedModel {enabled: number;contextWindow: number | null;intel: number;+ // Platforms that can serve this entry (group members under unify, a single+ // platform otherwise) + tool capability — feeds /v1/models+ // `supported_parameters` so agents can pick knobs per model.+ platforms: string[];+ supportsTools: boolean;}export interface ModelListing {@@ -41,9 +46,9 @@ export function buildModelListing(): ModelListing {if (isUnifyEnabled()) {// Unify ON: one entry per logical model group. Pull per-row availability +// context keyed by db id, then aggregate over each group's members.- type AvailRow = { id: number; platform: string; intelligence_rank: number; context_window: number | null; enabled: number; available: number };+ type AvailRow = { id: number; platform: string; intelligence_rank: number; context_window: number | null; enabled: number; available: number; supports_tools: number };const rows = db.prepare(`- SELECT m.id, m.platform, m.intelligence_rank, m.context_window,+ SELECT m.id, m.platform, m.intelligence_rank, m.context_window, m.supports_tools,m.enabled AS enabled, ${availableExpr} AS availableFROM models m`).all() as AvailRow[];@@ -59,15 +64,17 @@ export function buildModelListing(): ModelListing {enabled: infos.some(i => i.enabled === 1) ? 1 : 0,contextWindow: ctxs.length ? Math.max(...ctxs) : null,intel: infos.length ? Math.min(...infos.map(i => i.intelligence_rank)) : Number.MAX_SAFE_INTEGER,+ platforms: [...new Set(infos.map(i => i.platform))],+ supportsTools: infos.some(i => i.supports_tools === 1),};});} else {// Unify OFF: one entry per model_id (dedup picks the available, smartest// representative row).const models = db.prepare(`- SELECT platform, model_id, display_name, context_window, enabled, available, intelligence_rank, id+ SELECT platform, model_id, display_name, context_window, enabled, available, intelligence_rank, id, supports_toolsFROM (- SELECT m.platform, m.model_id, m.display_name, m.context_window, m.intelligence_rank, m.id,+ SELECT m.platform, m.model_id, m.display_name, m.context_window, m.intelligence_rank, m.id, m.supports_tools,m.enabled AS enabled,${availableExpr} AS available,ROW_NUMBER() OVER (@@ -77,11 +84,13 @@ export function buildModelListing(): ModelListing {FROM models m)WHERE rn = 1- `).all() as (ModelListRow & { intelligence_rank: number; id: number })[];+ `).all() as (ModelListRow & { intelligence_rank: number; id: number; supports_tools: number })[];allListed = models.map(m => ({id: m.model_id, name: m.display_name, ownedBy: m.platform,available: m.available, enabled: m.enabled, contextWindow: m.context_window,intel: m.intelligence_rank,+ platforms: [m.platform],+ supportsTools: m.supports_tools === 1,}));}diff --git a/shared/types.ts b/shared/types.tsindex ed5c3326..4742d8a4 100644--- a/shared/types.ts+++ b/shared/types.ts@@ -298,6 +298,9 @@ export interface ChatCompletionChoice {index: number;message: ChatMessage;finish_reason: string | null;+ // Present when the client requested logprobs and the provider returned+ // them; passed through verbatim (provider shapes vary slightly).+ logprobs?: unknown;}export interface TokenUsage {