Skip to content

7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.

TypeScript
11.7 KB
Raw Blame
import { createHash } from 'crypto';
import type { ChatMessage, ChatContent } from '@freellmapi/shared/types.js';
import { getSetting, setSetting } from '../db/index.js';
import { setRequestRoutingStrategy } from '../lib/client-context.js';
import { DecisionError, listDecisionModels, runDecision } from './decision.js';
import { getRoutingStrategy } from './router.js';
import type { PresetStrategy } from './scoring.js';

// Decision routing (#1354). With the 'decision' strategy on, a decision model
// reads the request and picks which of the four presets (Smartest, Fastest,
// Balanced, Most reliable) this request routes with. The pick only re-weights
// the bandit for this one request: every capability gate and the keep-trying
// fallback chain stay exactly as they are, and any doubt routes as Balanced:
//   - the top probability is below the configured minimum,
//   - the decision takes longer than DECISION_TIMEOUT_MS,
//   - no decision model or key is available, or the call fails.
// One decision per user turn: the answer is cached on the text it was asked
// about, so an agent's tool-call loop inside one turn asks once.

export const DECISION_PRESETS: readonly PresetStrategy[] = ['smartest', 'fastest', 'balanced', 'reliable'];

const DECISION_MODEL_KEY = 'routing_decision_model';
const MIN_PROBABILITY_KEY = 'routing_decision_min_probability';
export const DEFAULT_MIN_PROBABILITY = 0.5;
export const DECISION_TIMEOUT_MS = 1500;
/** Logged as the decision's client so Models → Decision shows routing usage. */
export const ROUTER_CLIENT = 'freellmapi-router';
// BeatAPI's free routes allow one successful call a minute per account: fine
// for /v1/decisions, far too slow to sit in front of every chat turn.
const AUTO_SKIP_PLATFORMS: ReadonlySet<string> = new Set(['beatapi']);

const SYSTEM_CHARS = 400;
const USER_CHARS = 1600;
const CACHE_TTL_MS = 30 * 60_000;
const CACHE_MAX = 500;

const QUESTION = {
  type: 'choice' as const,
  instructions:
    'You route requests for an LLM gateway. Read the request and pick the routing strategy that serves it best. ' +
    'Judge by how hard the request is and what the person needs from the answer, not by its topic.',
  criteria: {
    smartest: 'Hard or high-stakes: answer quality matters most and waiting a bit longer is fine.',
    fastest: 'Simple or short: a quick answer matters most and a smaller model will do.',
    balanced: 'Ordinary request: a sensible mix of quality and speed.',
    reliable: 'Must work first time: strict formats, structured output or an automated step that cannot be retried by hand.',
  } satisfies Record<PresetStrategy, string>,
};

export interface DecisionRoutingSettings {
  /** A decision model id, or 'auto' (first available, BeatAPI skipped). */
  model: string;
  /** 0..1: below this top probability the request routes as Balanced. */
  minProbability: number;
}

export function getDecisionRoutingSettings(): DecisionRoutingSettings {
  const model = getSetting(DECISION_MODEL_KEY)?.trim() || 'auto';
  const raw = getSetting(MIN_PROBABILITY_KEY);
  const n = raw === undefined || raw.trim() === '' ? NaN : Number(raw);
  return { model, minProbability: Number.isFinite(n) && n >= 0 && n <= 1 ? n : DEFAULT_MIN_PROBABILITY };
}

export function setDecisionRoutingSettings(patch: Partial<DecisionRoutingSettings>): void {
  if (patch.model !== undefined) {
    const model = patch.model.trim();
    if (!model) throw new Error('decision model must be a model id or "auto"');
    setSetting(DECISION_MODEL_KEY, model);
  }
  if (patch.minProbability !== undefined) {
    if (!Number.isFinite(patch.minProbability) || patch.minProbability < 0 || patch.minProbability > 1) {
      throw new Error('minProbability must be between 0 and 1');
    }
    setSetting(MIN_PROBABILITY_KEY, String(patch.minProbability));
  }
}

export type DecisionPickSource = 'decision' | 'cache' | 'low_confidence' | 'timeout' | 'unavailable' | 'error';

export interface DecisionPick {
  strategy: PresetStrategy;
  source: DecisionPickSource;
  /** The top probability the decision model gave (absent on fallback). */
  probability?: number;
  /** The decision model that answered. */
  model?: string;
}

// ── Per-turn cache ──────────────────────────────────────────────────────────
const cache = new Map<string, { pick: DecisionPick; at: number }>();

function cacheGet(key: string): DecisionPick | undefined {
  const hit = cache.get(key);
  if (!hit) return undefined;
  if (Date.now() - hit.at > CACHE_TTL_MS) {
    cache.delete(key);
    return undefined;
  }
  return hit.pick;
}

function cachePut(key: string, pick: DecisionPick): void {
  if (cache.size >= CACHE_MAX) cache.delete(cache.keys().next().value!);
  cache.set(key, { pick, at: Date.now() });
}

export function resetDecisionRoutingCache(): void {
  cache.clear();
  stats.counts = { smartest: 0, fastest: 0, balanced: 0, reliable: 0 };
  stats.fallbacks = {};
  stats.recent = [];
}

// ── Stats for the dashboard (in-memory, since boot) ─────────────────────────
const RECENT_MAX = 20;
const stats: {
  counts: Record<PresetStrategy, number>;
  fallbacks: Partial<Record<DecisionPickSource, number>>;
  recent: Array<DecisionPick & { at: number }>;
} = { counts: { smartest: 0, fastest: 0, balanced: 0, reliable: 0 }, fallbacks: {}, recent: [] };

function record(pick: DecisionPick): void {
  stats.counts[pick.strategy]++;
  if (pick.source !== 'decision' && pick.source !== 'cache') {
    stats.fallbacks[pick.source] = (stats.fallbacks[pick.source] ?? 0) + 1;
  }
  stats.recent.unshift({ ...pick, at: Date.now() });
  if (stats.recent.length > RECENT_MAX) stats.recent.length = RECENT_MAX;
}

export function getDecisionRoutingStats() {
  return { counts: { ...stats.counts }, fallbacks: { ...stats.fallbacks }, recent: stats.recent.slice() };
}

// ── What the decision model reads ───────────────────────────────────────────
function contentText(content: ChatContent): string {
  if (typeof content === 'string') return content;
  if (!Array.isArray(content)) return '';
  return content
    .map(block => (typeof block === 'string' ? block : typeof block?.text === 'string' ? block.text : ''))
    .filter(Boolean)
    .join('\n');
}

/** The system prompt's opening plus the latest user message, bounded so a huge
 *  conversation never turns into a huge decision prompt. Empty when there is
 *  no user text to judge (nothing to decide on: route as Balanced). */
export function decisionStateFor(messages: readonly ChatMessage[]): string {
  const system = messages.filter(m => m.role === 'system').map(m => contentText(m.content)).join('\n').trim();
  let user = '';
  for (let i = messages.length - 1; i >= 0; i--) {
    if (messages[i].role === 'user') {
      user = contentText(messages[i].content).trim();
      if (user) break;
    }
  }
  if (!user) return '';
  const parts: string[] = [];
  if (system) parts.push(`System prompt (start): ${system.slice(0, SYSTEM_CHARS)}`);
  parts.push(`Request: ${user.slice(0, USER_CHARS)}`);
  return parts.join('\n\n');
}

function topProbability(answer: unknown, chosen: string): number | undefined {
  const probs = (answer as { probabilities?: unknown } | null)?.probabilities;
  if (!probs || typeof probs !== 'object') return undefined;
  const p = (probs as Record<string, unknown>)[chosen];
  return typeof p === 'number' && Number.isFinite(p) ? p : undefined;
}

/** Ask the decision model which preset fits this request. Never throws: every
 *  failure is a Balanced pick with the reason in `source`. */
export async function chooseRoutingPreset(messages: readonly ChatMessage[], signal?: AbortSignal): Promise<DecisionPick> {
  const state = decisionStateFor(messages);
  if (!state) return { strategy: 'balanced', source: 'unavailable' };

  const settings = getDecisionRoutingSettings();
  // minProbability is part of the key: a pick cached under one threshold must
  // not answer a request judged under another (a low_confidence Balanced held
  // for the TTL would silently undo a threshold lowering — the whole point of
  // which is to let the model's own pick through).
  const key = createHash('sha256')
    .update(`${settings.model}\0${settings.minProbability}\0${state}`)
    .digest('hex');
  const cached = cacheGet(key);
  if (cached) return { ...cached, source: 'cache' };

  const timeout = AbortSignal.timeout(DECISION_TIMEOUT_MS);
  try {
    const out = await runDecision(
      { state, questions: { strategy: QUESTION } },
      {
        model: settings.model,
        signal: signal ? AbortSignal.any([timeout, signal]) : timeout,
        clientAgent: ROUTER_CLIENT,
        skipPlatforms: settings.model === 'auto' ? AUTO_SKIP_PLATFORMS : undefined,
      },
    );
    const answer = out.answers.strategy as { choice?: unknown } | undefined;
    const choice = answer?.choice;
    if (typeof choice !== 'string' || !DECISION_PRESETS.includes(choice as PresetStrategy)) {
      return { strategy: 'balanced', source: 'error', model: out.model };
    }
    const probability = topProbability(answer, choice);
    const pick: DecisionPick = (probability ?? 0) >= settings.minProbability
      ? { strategy: choice as PresetStrategy, source: 'decision', probability, model: out.model }
      : { strategy: 'balanced', source: 'low_confidence', probability, model: out.model };
    cachePut(key, pick);
    return pick;
  } catch (err) {
    if (timeout.aborted) return { strategy: 'balanced', source: 'timeout' };
    const status = err instanceof DecisionError ? err.status : undefined;
    return { strategy: 'balanced', source: status === 503 || status === 429 ? 'unavailable' : 'error' };
  }
}

/**
 * Called by the chat surfaces before routing. When the 'decision' strategy is
 * active and the request did not pin a model, picks the preset for this
 * request and stores it on the request context for the router. Returns the
 * pick (for the X-FreeLLM-Routing-Strategy header), or null when the strategy
 * is something else or the request pins a model.
 */
export async function applyDecisionRouting(
  messages: readonly ChatMessage[],
  opts: { pinned?: boolean; signal?: AbortSignal } = {},
): Promise<DecisionPick | null> {
  if (getRoutingStrategy() !== 'decision' || opts.pinned) return null;
  const pick = await chooseRoutingPreset(messages, opts.signal);
  setRequestRoutingStrategy(pick.strategy);
  record(pick);
  return pick;
}

/** Header value: the preset, plus why when it is a fallback. */
export function routingStrategyHeader(pick: DecisionPick): string {
  return pick.source === 'decision' || pick.source === 'cache' ? pick.strategy : `${pick.strategy}; fallback=${pick.source}`;
}

/** Settings + since-boot stats for the dashboard, plus what the setup guide
 *  needs. Mirrors the chain routing would use: the pinned model, or with
 *  'auto' every enabled decision model except the skipped platforms.
 *  - `ready`: one of those has a usable key (else every request is Balanced).
 *  - `hasModels`: there is at least one such model to add a key for.
 *  - `keyPlatforms`: providers whose key would make it ready, in priority order. */
export function getDecisionRoutingStatus() {
  const settings = getDecisionRoutingSettings();
  const candidates = listDecisionModels()
    .filter(m => m.enabled)
    .filter(m => settings.model === 'auto' ? !AUTO_SKIP_PLATFORMS.has(m.platform) : m.modelId === settings.model)
    .sort((a, b) => a.priority - b.priority || a.id - b.id);
  return {
    ...settings,
    ready: candidates.some(m => m.healthyKeyCount > 0),
    hasModels: candidates.length > 0,
    keyPlatforms: [...new Set(candidates.map(m => m.platform))],
    ...getDecisionRoutingStats(),
  };
}