7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.
TypeScript
import type {
ChatMessage,
ChatCompletionResponse,
ChatCompletionChunk,
ChatToolCall,
Platform,
} from '@freellmapi/shared/types.js';
import { BaseProvider, providerHttpError, type CompletionOptions, type KeyValidationResult } from './base.js';
import { extendedBodyParams, resolveMaxTokens } from '../lib/sampling-params.js';
import { rescueInlineToolCalls } from '../lib/tool-call-rescue.js';
import { extractThinkFromMessage } from '../lib/think-tags.js';
import { repairToolArguments, toolSchemaMap } from '../lib/tool-args.js';
import { invalidToolCallReasons, isToolArgumentValidationEnabled } from '../lib/tool-validate.js';
import { recordQuotaObservation, recordQuotaObservationsFromResponse, type QuotaObservationContext } from '../services/provider-quota.js';
import type { QuotaMetric } from '@freellmapi/shared/types.js';
import { providerTimeoutMs } from '../lib/provider-timeout.js';
import { isAbortLikeError } from '../lib/error-classify.js';
import { contentToString } from '../lib/content.js';
import { bearerAuthHeader } from '../lib/credential.js';
// Shared with the media/embeddings custom-endpoint paths; re-exported so
// existing imports keep working (#1331).
export { isAnonymousCredential } from '../lib/credential.js';
/** Hosts that ARE Moonshot's OpenAI-compatible API (api.moonshot.ai,
* api.moonshot.cn, api.kimi.com and their subdomains). */
const MOONSHOT_HOST_SUFFIXES = ['moonshot.ai', 'moonshot.cn', 'kimi.com'];
/**
* Whether a base URL points at Moonshot's own API. Moonshot documents an
* assistant-message `partial: true` prefill flag that no other OpenAI-compatible
* upstream understands, and there is no built-in moonshot platform — Kimi
* models are otherwise served by Groq, Cloudflare, OpenRouter, Hugging Face,
* Ollama, ... — so the flag is gated on the endpoint host, never on the model
* id. Returns false for anything that does not parse as a URL. (#1038)
*/
export function isMoonshotEndpoint(baseUrl: string): boolean {
let host: string;
try {
host = new URL(baseUrl).hostname.toLowerCase();
} catch {
return false;
}
return MOONSHOT_HOST_SUFFIXES.some((d) => host === d || host.endsWith(`.${d}`));
}
/**
* Some free-tier upstreams (notably Pollinations) answer HTTP 200 but put an
* out-of-credits / top-up notice in the assistant message instead of returning
* a 402. That reads as a successful completion, so the fallback loop never
* rotates off the dead key (#pollinations-inband). Detect the notice so callers
* can throw a payment-required error and fail over. Kept deliberately tight —
* requires the credits phrase AND a top-up/quest/Pollinations marker — so a
* genuine reply that merely discusses credits does not trip it.
*/
export function inBandCreditsError(text: string | null | undefined): string | null {
if (!text) return null;
const t = text.toLowerCase();
const mentionsCredits = t.includes('enough credits') || t.includes('insufficient credit');
if (!mentionsCredits) return null;
const topUpMarker =
t.includes('top up') || t.includes('top-up') ||
t.includes('complete a quest') || t.includes('pollinations');
if (!topUpMarker) return null;
return text.trim().slice(0, 200);
}
/**
* The retry options for a custom endpoint that answered 400 to a tool_choice
* pinning one function. Some local servers (LM Studio 0.4.26) reject
* {"type":"function","function":{"name":...}} but accept "required", and
* "required" with the tool list narrowed to the pinned function asks for the
* same call. Returns null when the request does not pin a function present in
* tools, so a retry never changes what the client asked for. (#1548)
*/
function pinnedToolChoiceRetry(options?: CompletionOptions): CompletionOptions | null {
const choice = options?.tool_choice;
if (!choice || typeof choice === 'string') return null;
const tools = options?.tools?.filter(t => t.function.name === choice.function.name);
if (!tools?.length) return null;
return { ...options, tools, tool_choice: 'required' };
}
/**
* Generic provider for platforms that use an OpenAI-compatible API.
* Covers: Groq, Cerebras, NVIDIA NIM, Mistral, OpenRouter,
* GitHub Models, Fireworks AI.
*/
/** A provider's key-info/quota endpoint (#1403): the GET URL and the JSON
* field names carrying the limit and the remaining balance. Fields are looked
* up first under a `data` wrapper (OpenRouter style), then at the top level.
* `credits` are in the provider's own currency (OpenRouter USD, SiliconFlow
* CNY); `notes` names it. Only a reading with a limit feeds routing headroom. */
export interface QuotaProbeSpec {
url: string;
metric: QuotaMetric;
limitFields: string[];
remainingFields: string[];
notes?: string;
}
export class OpenAICompatProvider extends BaseProvider {
readonly platform: Platform;
readonly name: string;
private readonly baseUrl: string;
private readonly extraHeaders: Record<string, string>;
private readonly validateUrl?: string;
/** Per-provider HTTP timeout override. OpenAI-compatible gateways often buffer
* non-streaming responses until generation completes, and reasoning models can
* take >15s before first byte. Default 60000. */
private readonly timeoutMs: number;
/** NVIDIA NIM models reject any request that permits parallel tool calls with
* `400 This model only supports single tool-calls at once!`. When set, pin
* parallel_tool_calls to false whenever tools are in play. See issue #255. */
private readonly forceSingleToolCall: boolean;
/** True only for a custom endpoint whose host is Moonshot's own API; see
* isMoonshotEndpoint(). Gates the assistant `partial` prefill flag (#1038). */
private readonly forwardsPartial: boolean;
/** Optional provider-reported quota endpoint (#1403). */
private readonly quotaProbe?: QuotaProbeSpec;
constructor(opts: {
platform: Platform;
name: string;
baseUrl: string;
extraHeaders?: Record<string, string>;
validateUrl?: string;
timeoutMs?: number;
keyless?: boolean;
forceSingleToolCall?: boolean;
quotaProbe?: QuotaProbeSpec;
}) {
super();
this.platform = opts.platform;
this.name = opts.name;
this.baseUrl = opts.baseUrl;
this.extraHeaders = opts.extraHeaders ?? {};
this.validateUrl = opts.validateUrl;
// PROVIDER_TIMEOUT_<PLATFORM> wins over the registration default (#547).
this.timeoutMs = providerTimeoutMs(opts.platform, opts.timeoutMs ?? 60_000);
this.keyless = opts.keyless ?? false;
this.forceSingleToolCall = opts.forceSingleToolCall ?? false;
this.forwardsPartial = opts.platform === 'custom' && isMoonshotEndpoint(opts.baseUrl);
this.quotaProbe = opts.quotaProbe;
}
/** Resolve the parallel_tool_calls flag to send upstream. For providers that
* only accept single tool calls (NVIDIA NIM), force `false` whenever tools are
* present so the model never tries to emit two at once and 400s; otherwise pass
* the caller's value through unchanged. See issue #255. */
private resolveParallelToolCalls(options?: CompletionOptions): boolean | undefined {
if (this.forceSingleToolCall && options?.tools && options.tools.length > 0) return false;
return options?.parallel_tool_calls;
}
/** Some providers (Groq especially) reject a model's tool call with a 400
* `tool_use_failed` when the model emitted it as inline DIALECT TEXT
* (`<function=NAME{...}</function>`, Hermes/Qwen XML, etc.) that the provider's
* own parser couldn't convert — but they hand back the raw text in
* `error.failed_generation`. Weaker tool models (e.g. groq llama-3.3-70b) hit
* this constantly, dead-ending an agent's whole turn even though the call is
* perfectly recoverable. Reuse the same inline-dialect rescue the proxy already
* applies to streamed text: parse `failed_generation` into structured
* tool_calls so the turn succeeds instead of failing over (or exhausting the
* chain when every enabled tool model behaves the same way). See issue #264. */
private rescueFailedGeneration(errBody: unknown, options?: CompletionOptions): ChatToolCall[] | null {
const failed = (errBody as { error?: { failed_generation?: unknown } })?.error?.failed_generation;
if (typeof failed !== 'string' || failed.length === 0) return null;
const toolNames = new Set((options?.tools ?? []).map(t => t.function.name));
if (toolNames.size === 0) return null;
const rescue = rescueInlineToolCalls(failed, toolNames);
if (!rescue.detected || !rescue.calls?.length) return null;
const schemas = toolSchemaMap(options?.tools);
const rescued = rescue.calls.map((c, i) => ({
id: `call_rescued_${i + 1}`,
type: 'function' as const,
function: { name: c.name, arguments: repairToolArguments(c.arguments, schemas.get(c.name)) },
}));
// Opt-in schema verdict. A rescue that produces schema-invalid arguments
// has turned the provider's 400 into a "success" the client cannot use, so
// decline it instead: the original upstream error propagates and the loop
// fails over exactly as it did before the rescue existed. Declining is the
// right shape here rather than throwing our own error — we are inside the
// provider's error path already.
if (isToolArgumentValidationEnabled() && invalidToolCallReasons(rescued, schemas).length > 0) {
return null;
}
return rescued;
}
/** Options for one retry of a custom endpoint's 400 to a pinned tool_choice,
* or null when no retry applies. The retried request uses "required", so it
* can never qualify for a second retry. See pinnedToolChoiceRetry (#1548). */
private pinnedToolChoiceRetryFor(status: number, options?: CompletionOptions): CompletionOptions | null {
if (this.platform !== 'custom' || status !== 400) return null;
const retry = pinnedToolChoiceRetry(options);
if (retry) console.log(`[${this.name}] 400 on a pinned tool_choice; retrying with "required" over the pinned tool (#1548)`);
return retry;
}
/** Extract the useful text from an upstream error body. Most providers put it
* at error.message, but NVIDIA NIM answers RFC7807-style ({"title": ...,
* "detail": "Function id '...': DEGRADED function cannot be invoked"}) — the
* old error.message-only read collapsed that to "Bad Request", so neither the
* logs nor the error classifier could ever see the DEGRADED marker (#522). */
private upstreamErrorText(errBody: unknown, res: Response): string {
const e = errBody as { error?: { message?: unknown }; detail?: unknown; title?: unknown };
if (typeof e?.error?.message === 'string' && e.error.message) return e.error.message;
if (typeof e?.detail === 'string' && e.detail) return e.detail;
if (typeof e?.title === 'string' && e.title) return e.title;
return res.statusText;
}
/** Anonymous access sends NO Authorization header: keyless providers
* (Kilo's anonymous free tier) holding the stored `no-key` sentinel, and
* custom endpoints whose stored credential is the same sentinel with auth
* off. Sending `Bearer no-key` upstream is never right — upstreams read it
* as an invalid key. A REAL key saved on a keyless platform is used instead
* of the anonymous path, and a real key on a custom endpoint still gets its
* bearer (#1331): the presence of a credential decides at request time —
* Kilo, OVH and AI Horde all accept both modes per their docs. */
private authHeader(apiKey: string): Record<string, string> {
return bearerAuthHeader(apiKey);
}
/** Requesty's Leanstral route rejects greedy sampling when temperature=0.
* Omitting that value and supplying a neutral top_p keeps the caller's intent
* deterministic enough while using the provider's supported sampling path. */
protected samplingForModel(modelId: string, options?: CompletionOptions): {
temperature: number | undefined;
topP: number | undefined;
} {
if (
this.platform === 'requesty' &&
modelId === 'mistral/leanstral-1-5' &&
options?.temperature === 0
) {
return { temperature: undefined, topP: options.top_p ?? 1 };
}
return { temperature: options?.temperature, topP: options?.top_p };
}
/**
* OpenAI-compatible endpoints that are strict about unknown nested fields:
* Mistral returns 422 for provider-private replay fields, Groq rejects
* assistant `reasoning_content` with 400 (verified in production, #1070),
* and Cerebras rejects it too — `property 'messages.N.assistant.
* reasoning_content' is unsupported` (confirmed via vercel/ai#15042 and
* opencode#26762, and by Cerebras' own docs, which use a `reasoning` field
* instead). Other gateways ignore the fields, so keep the OpenAI wire
* shape but strip our internal reasoning / thought-signature extensions
* before sending to these platforms.
*/
private static readonly STRICT_PLATFORMS = new Set(['mistral', 'groq', 'cerebras']);
private messagesForPlatform(messages: ChatMessage[], modelId: string): ChatMessage[] {
if (OpenAICompatProvider.STRICT_PLATFORMS.has(this.platform)) {
// Rebuild every message from a whitelist of keys, so `partial` and the
// reasoning extensions never reach these platforms.
return messages.map((m) => {
if (m.role === 'assistant') {
return {
role: m.role,
content: m.content,
...(m.name ? { name: m.name } : {}),
...(m.tool_calls && m.tool_calls.length > 0 ? {
tool_calls: m.tool_calls.map((tc) => ({
id: tc.id,
type: tc.type,
function: {
name: tc.function.name,
arguments: tc.function.arguments,
},
})),
} : {}),
};
}
if (m.role === 'tool') {
return {
role: m.role,
content: m.content,
...(m.tool_call_id ? { tool_call_id: m.tool_call_id } : {}),
...(m.name ? { name: m.name } : {}),
};
}
return {
role: m.role,
content: m.content,
...(m.name ? { name: m.name } : {}),
};
});
}
// Moonshot's assistant `partial` prefill flag only survives to the wire
// when this provider IS Moonshot's own API (a custom endpoint on a
// Moonshot/Kimi host). Every other OpenAI-compatible gateway — including
// the ones that serve Kimi models, like Groq, Cloudflare or OpenRouter —
// gets it stripped, since strict upstreams 400/422 on unknown message keys
// and the rest would ignore it anyway. (#1038)
const sanitized = this.forwardsPartial ? messages : messages.map((m) => {
if (m.role === 'assistant' && m.partial !== undefined) {
const { partial: _partial, ...rest } = m;
return rest;
}
return m;
});
// Qwen3.8-Flash-Next accepts exactly one system message, and only at index
// zero. Profiles can prepend a gateway system prompt to a client's own
// system message, so coalesce every system instruction before dispatch
// instead of letting an otherwise valid request fail upstream.
if (this.platform === 'radeon' && modelId === 'Qwen3.8-Flash-Next') {
const systems = sanitized.filter(m => m.role === 'system');
if (systems.length === 1 && sanitized[0]?.role === 'system') return sanitized;
const systemText = systems.map(m => contentToString(m.content)).filter(Boolean).join('\n\n');
const rest = sanitized.filter(m => m.role !== 'system');
return systemText ? [{ role: 'system', content: systemText }, ...rest] : rest;
}
return sanitized;
}
async chatCompletion(
apiKey: string,
messages: ChatMessage[],
modelId: string,
options?: CompletionOptions,
quotaContext?: QuotaObservationContext,
): Promise<ChatCompletionResponse> {
const sampling = this.samplingForModel(modelId, options);
const res = await this.fetchWithTimeout(`${this.baseUrl}/chat/completions`, {
method: 'POST',
headers: {
...this.authHeader(apiKey),
'Content-Type': 'application/json',
...this.extraHeaders,
},
body: JSON.stringify({
model: modelId,
messages: this.messagesForPlatform(messages, modelId),
temperature: sampling.temperature,
max_tokens: resolveMaxTokens(this.platform, options?.max_tokens, options?.contextBudget),
top_p: sampling.topP,
stop: options?.stop,
tools: options?.tools,
tool_choice: options?.tool_choice,
parallel_tool_calls: this.resolveParallelToolCalls(options),
...extendedBodyParams(this.platform, options),
}),
// 'request' bounds: the deadline covers the body read too, so a 200
// whose body hangs aborts instead of stalling res.json() forever.
}, options?.timeoutMs ?? this.timeoutMs, { signal: options?.signal, timeoutBounds: 'request' });
recordQuotaObservationsFromResponse(res, {
platform: this.platform,
keyId: quotaContext?.keyId,
providerAccountId: quotaContext?.providerAccountId,
modelId,
quotaPoolKey: quotaContext?.quotaPoolKey,
endpoint: 'chat/completions',
});
if (!res.ok) {
const err = await res.json().catch(() => ({}));
const rescued = this.rescueFailedGeneration(err, options);
if (rescued) {
console.log(`[${this.name}] Rescued ${rescued.length} inline tool call(s) from a ${res.status} tool_use_failed (#264)`);
const out: ChatCompletionResponse = {
id: `chatcmpl-rescued-${Date.now()}`,
object: 'chat.completion',
created: Math.floor(Date.now() / 1000),
model: modelId,
choices: [{ index: 0, message: { role: 'assistant', content: null as unknown as string, tool_calls: rescued }, finish_reason: 'tool_calls' }],
usage: { prompt_tokens: 0, completion_tokens: 0, total_tokens: 0 },
};
out._routed_via = { platform: this.platform, model: modelId };
return out;
}
const retry = this.pinnedToolChoiceRetryFor(res.status, options);
if (retry) return this.chatCompletion(apiKey, messages, modelId, retry, quotaContext);
throw providerHttpError(res, `${this.name} API error ${res.status}: ${this.upstreamErrorText(err, res)}`, err);
}
let data: ChatCompletionResponse;
let parseErr: unknown;
try {
data = await res.json() as ChatCompletionResponse;
} catch (err) {
// An aborted body read (per-attempt deadline, client disconnect) is not
// a malformed body — rethrow so it keeps its abort classification
// instead of reading as a "non-OpenAI-compatible endpoint" below.
if (isAbortLikeError(err)) throw err;
parseErr = err;
data = undefined as unknown as ChatCompletionResponse;
}
if (!data) {
// A 200 whose body isn't a single JSON document. Two distinct causes:
// (1) base URL points at a non-OpenAI-compatible API (Ollama's native
// NDJSON /api endpoints, llama.cpp's non-/v1 server, etc., #189).
// Typical signals: Content-Type is application/x-ndjson or
// text/event-stream; parser sees a SECOND JSON object after the
// first one ends ("Unexpected non-whitespace character after JSON
// at position <n> (line <n> column <n>)") where <n> sits inside
// the whitespace between two valid JSON documents.
// (2) the upstream connection was cut short mid-response — most often
// Cloudflare's 600-second edge idle keepalive dropping a slow
// free-tier queue (Kilo provider, NVIDIA nemotron / Poolside
// Laguna models on Cloudflare-fronted upstreams). Signals: body
// ends inside a string or mid-token, parser sees
// "Unexpected end of JSON input"; Content-Type is application/json
// (CF proxies it transparently); latency_ms ≈ 600000.
// Without this split every Cloudflare-truncated request was logged as
// "endpoint is not OpenAI-compatible", which sent operators chasing a
// base-URL config bug that doesn't exist (#430).
const msg = parseErr instanceof Error ? parseErr.message : String(parseErr);
const contentType = (res.headers?.get?.('content-type') ?? '').toLowerCase();
const looksLikeNdjson = /ndjson|text\/event-stream|x-ndjson/.test(contentType);
const looksLikeJson = /application\/json/.test(contentType);
// Truncation = body parsed cleanly until mid-stream EOF/mid-token garbage.
// Only attribute it to a CDN keepalive when the upstream claims to be
// sending JSON (Content-Type: application/json). Without that hint,
// the parser is more likely choking on NDJSON, native API output, or
// HTML — all "wrong endpoint" cases. This is the safe default.
const looksTruncated =
/Unexpected end of JSON input/.test(msg) ||
(/Unexpected non-whitespace character after JSON at position/.test(msg) && looksLikeJson && !looksLikeNdjson);
if (looksTruncated) {
throw new Error(
`${this.name} returned 200 but the response body was truncated mid-stream ` +
`(likely an idle-keepalive timeout at an upstream proxy or CDN, e.g. Cloudflare's ` +
`600s edge limit). Retry, or switch to a faster model/upstream.`,
);
}
throw new Error(
`${this.name} returned 200 with a non-JSON body — the endpoint is not OpenAI-compatible. ` +
`Check the base URL (for Ollama use http://host:11434/v1, for llama.cpp/vLLM/LM Studio the /v1 path).`,
);
}
normalizeChoices(data);
// #pollinations-inband: a 200 whose body is really an out-of-credits notice
// must fail over, not be returned as the answer. The "402 ... insufficient
// credit" wording makes isPaymentRequiredError classify it as retryable and
// bench the key with the day-long payment-required cooldown.
const creditsNotice = inBandCreditsError(
(data.choices ?? []).map(c => contentToString((c.message as ChatMessage)?.content)).join('\n'),
);
if (creditsNotice) {
throw new Error(`${this.name} API error 402: insufficient credit (upstream returned 200 with an out-of-credits notice): ${creditsNotice}`);
}
data._routed_via = { platform: this.platform, model: modelId };
return data;
}
async *streamChatCompletion(
apiKey: string,
messages: ChatMessage[],
modelId: string,
options?: CompletionOptions,
quotaContext?: QuotaObservationContext,
): AsyncGenerator<ChatCompletionChunk> {
const sampling = this.samplingForModel(modelId, options);
const res = await this.fetchWithTimeout(`${this.baseUrl}/chat/completions`, {
method: 'POST',
headers: {
...this.authHeader(apiKey),
'Content-Type': 'application/json',
...this.extraHeaders,
},
body: JSON.stringify({
model: modelId,
messages: this.messagesForPlatform(messages, modelId),
temperature: sampling.temperature,
max_tokens: resolveMaxTokens(this.platform, options?.max_tokens, options?.contextBudget),
top_p: sampling.topP,
stop: options?.stop,
tools: options?.tools,
tool_choice: options?.tool_choice,
parallel_tool_calls: this.resolveParallelToolCalls(options),
...extendedBodyParams(this.platform, options),
stream: true,
stream_options: options?.stream_options,
}),
// Default 'headers' bounds: the deadline dies at response headers, and
// the client signal + stall watchdog own the stream from there.
}, options?.timeoutMs ?? this.timeoutMs, { signal: options?.signal });
recordQuotaObservationsFromResponse(res, {
platform: this.platform,
keyId: quotaContext?.keyId,
providerAccountId: quotaContext?.providerAccountId,
modelId,
quotaPoolKey: quotaContext?.quotaPoolKey,
endpoint: 'chat/completions',
});
if (!res.ok) {
const err = await res.json().catch(() => ({}));
const rescued = this.rescueFailedGeneration(err, options);
if (rescued) {
console.log(`[${this.name}] Rescued ${rescued.length} inline tool call(s) from a ${res.status} tool_use_failed (stream, #264)`);
const base = { id: `chatcmpl-rescued-${Date.now()}`, object: 'chat.completion.chunk' as const, created: Math.floor(Date.now() / 1000), model: modelId };
yield { ...base, choices: [{ index: 0, delta: { role: 'assistant' }, finish_reason: null }] };
yield { ...base, choices: [{ index: 0, delta: { tool_calls: rescued.map((c, i) => ({ index: i, ...c })) as unknown as ChatToolCall[] }, finish_reason: null }] };
yield { ...base, choices: [{ index: 0, delta: {}, finish_reason: 'tool_calls' }] };
return;
}
const retry = this.pinnedToolChoiceRetryFor(res.status, options);
if (retry) {
yield* this.streamChatCompletion(apiKey, messages, modelId, retry, quotaContext);
return;
}
throw providerHttpError(res, `${this.name} API error ${res.status}: ${this.upstreamErrorText(err, res)}`, err);
}
// First-byte grace (#584): the same chat timeout that bounded the headers
// also budgets the first stream read — NIM-style providers send SSE
// headers instantly, then prefill long prompts for minutes.
yield* this.guardInBandCreditsError(
this.readSseStream(res, { firstByteTimeoutMs: options?.timeoutMs ?? this.timeoutMs }),
);
}
/**
* #pollinations-inband: wrap a chat stream so an in-band out-of-credits notice
* (a 200 that streams the top-up message as content) fails over instead of
* being shown as the answer. Only the FIRST content-bearing chunk is inspected
* — a Pollinations credits notice arrives as a single canned message — so the
* stream is otherwise byte-for-byte unchanged: reasoning/role chunks pass
* straight through (first-token ttfb intact) and every chunk after the first
* content one is untouched. Throwing on that first content chunk happens before
* it reaches the proxy's commit point, so the fallback loop can still rotate.
*/
private async *guardInBandCreditsError(
src: AsyncGenerator<ChatCompletionChunk>,
): AsyncGenerator<ChatCompletionChunk> {
let sawContent = false;
for await (const chunk of src) {
if (!sawContent) {
const content = (chunk.choices ?? [])
.map(c => (c.delta as { content?: unknown } | undefined)?.content)
.filter((c): c is string => typeof c === 'string')
.join('');
if (content) {
sawContent = true;
const notice = inBandCreditsError(content);
if (notice) {
throw new Error(`${this.name} API error 402: insufficient credit (upstream streamed an out-of-credits notice): ${notice}`);
}
}
}
yield chunk;
}
}
/** This provider's OpenAI-style model catalog URL. */
get modelsUrl(): string {
return `${this.baseUrl}/models`;
}
/**
* GET a catalog-style endpoint with this provider's auth header, extra
* headers, proxy routing, timeout policy and quota bookkeeping. Shared by
* validateKey (which only reads the status) and custom-endpoint model
* discovery (#488), which also reads the body — one place owns how we talk
* to a provider's /models route.
*
* Note: transport errors (DNS / timeout / TLS) propagate to the caller.
* health.ts catches them and marks status='error' WITHOUT incrementing the
* consecutive-failure counter — only confirmed 401/403 disables a key.
*/
protected fetchCatalogEndpoint(url: string, apiKey: string, quotaContext?: QuotaObservationContext): Promise<Response> {
// 30s (not 10s): some upstreams return a large /v1/models catalog that
// takes >10s from high-latency regions (e.g. NVIDIA NIM measured ~11.2s
// from India). A 10s cap aborted those calls and health.ts marked a
// perfectly good key status='error'. 30s aligns with chatCompletion's
// own slow-upstream allowance and costs nothing for fast providers.
return this.fetchWithTimeout(url, {
method: 'GET',
headers: {
...this.authHeader(apiKey),
...this.extraHeaders,
},
// 'request' bounds: a catalog body that hangs mid-transfer must not
// stall the health cycle past the deadline.
}, 30000, { timeoutBounds: 'request' }).then(res => {
recordQuotaObservationsFromResponse(res, {
platform: this.platform,
keyId: quotaContext?.keyId,
providerAccountId: quotaContext?.providerAccountId,
quotaPoolKey: quotaContext?.quotaPoolKey,
endpoint: 'models',
});
return res;
});
}
/** The raw `${baseUrl}/models` response, body unread. Used by custom-endpoint
* model discovery (#488); unlike validateKey it always hits /models, never a
* provider-specific validateUrl, because the caller wants the catalog. */
fetchModelCatalog(apiKey: string, quotaContext?: QuotaObservationContext): Promise<Response> {
return this.fetchCatalogEndpoint(this.modelsUrl, apiKey, quotaContext);
}
async validateKey(apiKey: string, quotaContext?: QuotaObservationContext): Promise<KeyValidationResult> {
const res = await this.fetchCatalogEndpoint(this.validateUrl ?? this.modelsUrl, apiKey, quotaContext);
// An OpenAI-compatible /models answers JSON. A 200 with anything else is a
// shut-down or parked service, not a working key: retired GitHub Models
// answered every path with 200 "OK" in plain text and every key looked
// healthy (#1557). Thrown, so health records an error without counting it
// toward auto-disable. Provider-specific validate URLs keep their own shape.
if (res.ok && !this.validateUrl && typeof res.text === 'function') {
const text = await res.text().catch(() => '');
try {
JSON.parse(text);
} catch {
throw providerHttpError(res, `${this.name} /models did not answer JSON (got ${JSON.stringify(text.trim().slice(0, 40))}); the service may be retired or down`);
}
return true;
}
return this.validationResult(res);
}
/**
* Provider-reported balance via the registered quotaProbe endpoint (#1403).
* GETs the key-info URL (e.g. OpenRouter's `GET /api/v1/key`), pulls the
* limit/remaining numbers out of the JSON (checking the `data` wrapper and
* the top level), and records them as a `source: 'quota_api'` observation.
* Never throws: a quota probe that fails says nothing about the key, and
* the caller's health verdict must stay about the key.
*/
get hasQuotaProbe(): boolean {
return this.quotaProbe !== undefined;
}
async fetchQuota(apiKey: string, quotaContext?: QuotaObservationContext): Promise<boolean> {
const spec = this.quotaProbe;
if (!spec) return false;
try {
const res = await this.fetchWithTimeout(spec.url, {
method: 'GET',
headers: {
...this.authHeader(apiKey),
...this.extraHeaders,
},
}, 10_000, { timeoutBounds: 'request' });
if (!res.ok) return false;
const body: any = await res.json().catch(() => null);
const scope = (body && typeof body === 'object' && body.data && typeof body.data === 'object')
? [body.data, body]
: [body];
const pick = (fields: string[]): number | null => {
for (const obj of scope) {
if (!obj || typeof obj !== 'object') continue;
for (const f of fields) {
const v = (obj as Record<string, unknown>)[f];
if (typeof v === 'number' && Number.isFinite(v)) return v;
// Some providers serialize balances as strings ("88.88", SiliconFlow
// /v1/user/info). Accept a numeric string; "", null and prose never
// parse to a finite number.
if (typeof v === 'string' && v.trim() !== '') {
const n = Number(v);
if (Number.isFinite(n)) return n;
}
}
}
return null;
};
const limit = pick(spec.limitFields);
const remaining = pick(spec.remainingFields);
if (limit === null && remaining === null) return false;
const record = recordQuotaObservation({
platform: this.platform,
keyId: quotaContext?.keyId,
providerAccountId: quotaContext?.providerAccountId,
quotaPoolKey: quotaContext?.quotaPoolKey,
endpoint: 'quota_api',
metric: spec.metric,
limit,
remaining,
resetStrategy: 'provider_reported',
source: 'quota_api',
statusCode: res.status,
notes: spec.notes ?? null,
});
return record !== null;
} catch {
// Transport/parse failure: inconclusive, never a key verdict (#1403).
return false;
}
}
}
/**
* Some providers (Z.ai glm-4.5-flash, Cloudflare DeepSeek-R1-distill, others)
* return reasoning models' actual answer in `message.reasoning_content` with
* `message.content === ""`. Fold reasoning_content into content so OpenAI-
* compatible clients see a non-empty assistant message.
*
* Other providers (Mistral magistral-medium) return `message.content` as an
* array of text segments instead of a string. Flatten to string.
*/
function normalizeChoices(data: ChatCompletionResponse): void {
for (const choice of data.choices ?? []) {
const msg = choice.message as ChatMessage & {
reasoning_content?: string;
reasoning?: string;
content: unknown;
};
// Flatten array content (Mistral magistral) → join text segments.
if (Array.isArray(msg.content)) {
msg.content = (msg.content as Array<{ text?: string; type?: string }>)
.map(seg => (typeof seg === 'string' ? seg : (seg.text ?? '')))
.join('');
}
// Inline `<think>…</think>` extraction (DeepSeek-style) BEFORE the fold
// below: a leading think block moves out of content into
// reasoning_content. Runs before the fold so a think-only message (no
// answer after the block) still folds back into content and is never
// returned as an empty assistant message.
extractThinkFromMessage(msg);
// Fold reasoning into content if content is empty AND there are no
// tool_calls. With tool_calls present, content=null is the correct OpenAI
// shape; folding reasoning would confuse clients that branch on content.
// Field naming varies by provider: Z.ai uses `reasoning_content`, Ollama
// uses `reasoning`. Prefer `reasoning_content` when both are set.
const hasToolCalls = Array.isArray(msg.tool_calls) && msg.tool_calls.length > 0;
if (!hasToolCalls && (msg.content === '' || msg.content == null)) {
const fold = (typeof msg.reasoning_content === 'string' && msg.reasoning_content.length > 0)
? msg.reasoning_content
: (typeof msg.reasoning === 'string' && msg.reasoning.length > 0 ? msg.reasoning : null);
if (fold !== null) msg.content = fold;
}
}
}