7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.
TypeScript
import type {
ChatMessage,
ChatContent,
ChatCompletionResponse,
ChatCompletionChunk,
Platform,
} from '@freellmapi/shared/types.js';
import { BaseProvider, providerHttpError, type CompletionOptions, type KeyValidationResult } from './base.js';
import { recordQuotaObservationsFromResponse, type QuotaObservationContext } from '../services/provider-quota.js';
import { providerTimeoutMs } from '../lib/provider-timeout.js';
import { resolveMaxTokens } from '../lib/sampling-params.js';
/**
* AI Horde — free, community-powered inference served by volunteer workers and
* exposed through an OpenAI-compatible proxy at https://oai.aihorde.net/v1
* (issue #345).
*
* It is deliberately NOT a plain OpenAICompatProvider: the proxy diverges from
* the OpenAI contract in ways that would otherwise 422 or corrupt analytics,
* all handled here:
*
* - QUEUED execution. A request waits for whatever volunteer workers are
* online, so a single call can take tens of seconds to minutes. Hence the
* 120s timeout and the no-upstream-streaming design — one queued generation,
* surfaced whole (streamChatCompletion emits it as a single delta).
* - `max_tokens` must be >= 16 or the proxy 422s. We floor it, and default it
* when the caller omits it so a worker doesn't run to its own (often large)
* internal cap on every call.
* - `stop` must be an ARRAY; a bare string 422s. We wrap it.
* - No tool / function calling. tools, tool_choice and parallel_tool_calls are
* dropped so a tool-using caller doesn't 422 the whole request.
* - `usage` comes back as `{"kudos": N}` with no token counts. We synthesize
* prompt/completion/total token estimates (chars/4) so analytics, savings
* math and per-model usage charts aren't uniformly zero.
* - Auth: anonymous access works with the documented sentinel key `0000000000`
* (lowest queue priority). Registered keyless so the gateway auto-configures
* it; a real aihorde.net key stored on the api_keys row is forwarded instead
* for higher priority (see resolveBearer).
*
* Quality caveat (not fixable here): some workers append template/instruction
* text after the answer. Surfaced verbatim; documented as a catalog quirk.
*/
const ANON_KEY = '0000000000';
const MIN_MAX_TOKENS = 16;
const DEFAULT_MAX_TOKENS = 512;
// PROVIDER_TIMEOUT_AIHORDE overrides (#547).
const HORDE_TIMEOUT_MS = providerTimeoutMs('aihorde', 120000);
/** Rough token estimate (~4 chars/token) used only to fill usage when the proxy
* returns kudos instead of token counts. Good enough for analytics, never
* billed against anything. Handles string | null | multimodal-array content. */
function estimateTokens(content: ChatContent | undefined): number {
if (content == null) return 0;
let text: string;
if (typeof content === 'string') {
text = content;
} else {
text = content
.map(block => (typeof block === 'string' ? block : (block?.text ?? '')))
.join(' ');
}
return Math.ceil(text.length / 4);
}
export class AIHordeProvider extends BaseProvider {
readonly platform: Platform = 'aihorde';
readonly name = 'AI Horde';
/** Works out of the box via the anonymous key: the gateway stores a sentinel
* row (so routing treats the platform as configured) and we send the anon
* bearer. A stored real horde key replaces the sentinel for higher priority. */
keyless = true;
private readonly baseUrl = 'https://oai.aihorde.net/v1';
/** Map the stored credential to the bearer we send upstream. The keyless flow
* stores `'no-key'` (or nothing) for the anonymous case → send AI Horde's
* documented anonymous key. Any other stored value is treated as a registered
* key and forwarded verbatim for higher queue priority. */
private resolveBearer(apiKey: string): string {
const k = apiKey?.trim();
if (!k || k === 'no-key' || k === ANON_KEY) return ANON_KEY;
return k;
}
/** Build the upstream body, normalizing the OpenAI params the proxy rejects
* (see class doc). No `stream` — we never stream upstream. */
private buildBody(messages: ChatMessage[], modelId: string, options?: CompletionOptions): Record<string, unknown> {
const body: Record<string, unknown> = {
model: modelId,
messages,
// Floor at 16 (proxy 422s below it); default when omitted. Our own
// default goes through resolveMaxTokens too, so the unified output cap
// applies to it as well — it can only ever lower what we send.
max_tokens: Math.max(
MIN_MAX_TOKENS,
resolveMaxTokens(this.platform, options?.max_tokens ?? DEFAULT_MAX_TOKENS, options?.contextBudget) ?? DEFAULT_MAX_TOKENS,
),
};
if (options?.temperature != null) body.temperature = options.temperature;
if (options?.top_p != null) body.top_p = options.top_p;
// `stop` must be an array; wrap a bare string, pass arrays through.
if (options?.stop != null) {
body.stop = Array.isArray(options.stop) ? options.stop : [options.stop];
}
// tools / tool_choice / parallel_tool_calls intentionally dropped: no tool support.
return body;
}
/** Replace the proxy's `{"kudos": N}` usage with synthesized token counts so
* downstream analytics aren't all zero. Prompt from the input messages,
* completion from the returned content. */
private synthesizeUsage(messages: ChatMessage[], data: ChatCompletionResponse): void {
const prompt = messages.reduce((sum, m) => sum + estimateTokens(m.content), 0);
const completion = (data.choices ?? []).reduce(
(sum, c) => sum + estimateTokens(c.message?.content),
0,
);
data.usage = {
prompt_tokens: prompt,
completion_tokens: completion,
total_tokens: prompt + completion,
};
}
private parseError(err: unknown, status: number, statusText: string): string {
// AI Horde errors are `{"detail": "..."}`; fall back to OpenAI's
// `{error:{message}}` shape and then the status line.
const detail = (err as { detail?: unknown })?.detail;
if (typeof detail === 'string' && detail.length > 0) return detail;
const msg = (err as { error?: { message?: unknown } })?.error?.message;
if (typeof msg === 'string' && msg.length > 0) return msg;
return statusText || `HTTP ${status}`;
}
async chatCompletion(
apiKey: string,
messages: ChatMessage[],
modelId: string,
options?: CompletionOptions,
quotaContext?: QuotaObservationContext,
): Promise<ChatCompletionResponse> {
const res = await this.fetchWithTimeout(`${this.baseUrl}/chat/completions`, {
method: 'POST',
headers: {
'Authorization': `Bearer ${this.resolveBearer(apiKey)}`,
'Content-Type': 'application/json',
},
body: JSON.stringify(this.buildBody(messages, modelId, options)),
// 'request' bounds: the queued generation is one blocking body read, so
// the deadline must cover it too (a hung body used to stall forever).
}, options?.timeoutMs ?? HORDE_TIMEOUT_MS, { signal: options?.signal, timeoutBounds: 'request' });
recordQuotaObservationsFromResponse(res, {
platform: this.platform,
keyId: quotaContext?.keyId,
providerAccountId: quotaContext?.providerAccountId,
modelId,
quotaPoolKey: quotaContext?.quotaPoolKey,
endpoint: 'chat/completions',
});
if (!res.ok) {
const err = await res.json().catch(() => ({}));
throw providerHttpError(res, `${this.name} API error ${res.status}: ${this.parseError(err, res.status, res.statusText)}`, err);
}
const data = await res.json() as ChatCompletionResponse;
this.synthesizeUsage(messages, data);
data._routed_via = { platform: this.platform, model: modelId };
return data;
}
/**
* AI Horde's proxy returns a queued generation as one response, so there is no
* meaningful token-by-token stream. We run the same blocking call and emit the
* result as a minimal SSE sequence (role → content → finish) so streaming
* clients still work.
*/
async *streamChatCompletion(
apiKey: string,
messages: ChatMessage[],
modelId: string,
options?: CompletionOptions,
quotaContext?: QuotaObservationContext,
): AsyncGenerator<ChatCompletionChunk> {
const data = await this.chatCompletion(apiKey, messages, modelId, options, quotaContext);
const choice = data.choices?.[0];
const content = typeof choice?.message?.content === 'string' ? choice.message.content : '';
const base = {
id: data.id ?? this.makeId(),
object: 'chat.completion.chunk' as const,
created: data.created ?? Math.floor(Date.now() / 1000),
model: modelId,
};
yield { ...base, choices: [{ index: 0, delta: { role: 'assistant' }, finish_reason: null }] };
if (content) {
yield { ...base, choices: [{ index: 0, delta: { content }, finish_reason: null }] };
}
yield { ...base, choices: [{ index: 0, delta: {}, finish_reason: choice?.finish_reason ?? 'stop' }] };
}
/**
* The OpenAI proxy's GET /v1/models answers 200 for ANY bearer (it does not
* validate the key), and the anonymous key is always usable, so a reachable
* endpoint means the platform is healthy. Mirrors keyless providers: only a
* confirmed 401/403 is treated as an invalid key. Transport errors propagate
* to health.ts (marked status='error' without counting a failure).
*/
async validateKey(apiKey: string, quotaContext?: QuotaObservationContext): Promise<KeyValidationResult> {
const res = await this.fetchWithTimeout(`${this.baseUrl}/models`, {
method: 'GET',
headers: { 'Authorization': `Bearer ${this.resolveBearer(apiKey)}` },
}, 30000, { timeoutBounds: 'request' });
recordQuotaObservationsFromResponse(res, {
platform: this.platform,
keyId: quotaContext?.keyId,
providerAccountId: quotaContext?.providerAccountId,
quotaPoolKey: quotaContext?.quotaPoolKey,
endpoint: 'models',
});
return this.validationResult(res);
}
}