Skip to content

7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.

TypeScript
20.7 KB
Raw Blame
import { getDb } from '../db/index.js';
import { BEATAPI_BASE_URL } from '../providers/beatapi.js';
import { APINEX_BASE_URL } from '../providers/apinex.js';
import { parseRetryAfterMs, parseStatedRetryMs } from '../providers/base.js';
import { RetryHintTracker } from '../lib/retry-hint.js';
import { getClientContext } from '../lib/client-context.js';
import { proxyFetch } from '../lib/proxy.js';
import { recordMonthlyUsage, secondsUntilNextMonth } from './key-budget.js';
import { noteBudgetAlerts } from './budget-alerts.js';
import { reserveProviderCredential, type ReservedProviderCredential } from './provider-credential.js';
import { isOnCooldown, setCooldown } from './ratelimit.js';
import { markKeyHealthyFromRequest } from './health.js';

// Decision models over the key pool.
//
// A decision model answers typed questions about a piece of state instead of
// generating text: pick one option (choice), yes/no (noul) or a rating on a
// rubric (score), each with probabilities and a confidence. They speak the
// "System One" request shape: { state, questions: { id: { type, instructions,
// criteria } } }.
//
// Decision models are media_models rows with modality = 'decision', maintained
// only by the catalog's `decisionModels` registry (see catalog-sync). They ride
// on keys the user already has for the hosting platform (Cloudflare Workers AI,
// OpenRouter), so there is no separate key to add. They are exposed on
// POST /v1/decisions and are NOT used by the chat router: routing never
// consults them.
//
// Routing within decisions: the requested model, or every enabled decision
// model in priority order for 'auto', then every healthy enabled key of that
// model's platform. A rate-limited or rejected key cools down for that model
// and the chain moves on. Every attempt is logged to decision_requests.

export const DECISION_MODALITY = 'decision';

const FETCH_TIMEOUT_MS = 30_000;
const RATE_LIMIT_COOLDOWN_MS = 60_000;
const BAD_KEY_COOLDOWN_MS = 60 * 60_000;
const NO_CREDIT_COOLDOWN_MS = 12 * 60 * 60_000;
const ESTIMATED_TOKENS = 500;

export const DECISION_QUESTION_TYPES = ['choice', 'noul', 'score'] as const;
export type DecisionQuestionType = (typeof DECISION_QUESTION_TYPES)[number];

export interface DecisionQuestion {
  type: DecisionQuestionType;
  instructions: unknown;
  criteria?: unknown;
}

export interface DecisionRequest {
  state: unknown;
  questions: Record<string, DecisionQuestion>;
}

export interface DecisionResponse {
  /** The decision model that answered (its catalog id). */
  model: string;
  provider: string;
  answers: Record<string, unknown>;
  usage: { input_tokens: number; output_tokens: number };
}

export class DecisionError extends Error {
  constructor(
    message: string,
    public status: number,
    public code?: string,
    public retryAfterMs?: number,
  ) {
    super(message);
    this.name = 'DecisionError';
  }
}

export interface DecisionOptions {
  /** A decision model id, or 'auto' / undefined for the first one that works. */
  model?: string;
  signal?: AbortSignal;
  /** Overrides the request-scoped client agent (tests, internal callers). */
  clientAgent?: string | null;
  /** With 'auto', leave these platforms out of the chain. Decision routing
   *  (#1354) asks once per chat turn, which BeatAPI's free 1 request/minute
   *  cannot keep up with. A pinned model is never skipped. */
  skipPlatforms?: ReadonlySet<string>;
}

interface DecisionModelRow {
  id: number;
  platform: string;
  model_id: string;
  display_name: string;
  priority: number;
  enabled: number;
  quota_label: string;
  meta_json: string | null;
}

interface DecisionModelMeta {
  contextWindow?: number;
  premiumSince?: string;
}

function parseMeta(raw: string | null): DecisionModelMeta {
  if (!raw) return {};
  try {
    const parsed = JSON.parse(raw) as DecisionModelMeta;
    return parsed && typeof parsed === 'object' ? parsed : {};
  } catch {
    return {};
  }
}

function allDecisionRows(): DecisionModelRow[] {
  return getDb()
    .prepare(`SELECT id, platform, model_id, display_name, priority, enabled, quota_label, meta_json
                FROM media_models WHERE modality = ? ORDER BY priority, id`)
    .all(DECISION_MODALITY) as DecisionModelRow[];
}

// ── Adapters ────────────────────────────────────────────────────────────────

interface AdapterResult {
  answers: Record<string, unknown>;
  usage: { input_tokens: number; output_tokens: number };
}

type Adapter = (cred: ReservedProviderCredential, modelId: string, body: DecisionRequest, signal?: AbortSignal) => Promise<AdapterResult>;

/** A back-off stated inside an error body: JSON first (RetryInfo-shaped), the
 *  raw text as prose otherwise. Mirrors buildUpstreamError in providers/base.ts. */
function statedBodyRetryMs(text: string): number | undefined {
  try {
    return parseStatedRetryMs(JSON.parse(text));
  } catch {
    return parseStatedRetryMs(text);
  }
}

async function upstream(url: string, platform: string, init: RequestInit, signal?: AbortSignal): Promise<unknown> {
  const timeoutSignal = AbortSignal.timeout(FETCH_TIMEOUT_MS);
  const res = await proxyFetch(
    url,
    { ...init, signal: signal ? AbortSignal.any([timeoutSignal, signal]) : timeoutSignal },
    platform,
    'decision',
    FETCH_TIMEOUT_MS,
  );
  if (!res.ok) {
    const text = await res.text().catch(() => '');
    // Same rule as the chat path (providers/base.ts buildUpstreamError): a
    // header Retry-After wins, then a back-off stated in the error BODY
    // (Gemini RetryInfo, prose "try again in 45 seconds"). Dropping the body
    // hint demotes an exact promise to the 60s heuristic and to no hint.
    throw new DecisionError(
      `${platform} ${res.status}: ${text.slice(0, 200)}`,
      res.status,
      undefined,
      parseRetryAfterMs(res.headers?.get('retry-after') ?? null) ?? statedBodyRetryMs(text),
    );
  }
  return res.json();
}

function usageOf(raw: unknown): AdapterResult['usage'] {
  const u = (raw ?? {}) as { input_tokens?: unknown; output_tokens?: unknown };
  return {
    input_tokens: typeof u.input_tokens === 'number' ? u.input_tokens : 0,
    output_tokens: typeof u.output_tokens === 'number' ? u.output_tokens : 0,
  };
}

function answersOf(raw: unknown, platform: string): Record<string, unknown> {
  if (!raw || typeof raw !== 'object' || Array.isArray(raw)) {
    throw new DecisionError(`${platform} returned no answers`, 502);
  }
  return raw as Record<string, unknown>;
}

/** Cloudflare Workers AI keys are stored as "account_id:api_token". */
function cloudflareKey(key: string): { accountId: string; token: string } {
  const sep = key.indexOf(':');
  if (sep === -1) throw new DecisionError('Cloudflare key must be in format "account_id:api_token"', 401);
  return { accountId: key.slice(0, sep), token: key.slice(sep + 1) };
}

const ADAPTERS: Record<string, Adapter> = {
  apinex: async (cred, modelId, body, signal) => {
    const json = await upstream(`${APINEX_BASE_URL}/decisions`, 'apinex', {
      method: 'POST', redirect: 'error',
      headers: { Authorization: `Bearer ${cred.key}`, 'Content-Type': 'application/json' },
      body: JSON.stringify({ model: modelId, state: body.state, questions: body.questions }),
    }, signal) as { model?: string; answers?: unknown; usage?: unknown; error?: unknown } | null;
    if (!json || json.error || json.model !== modelId) {
      throw new DecisionError('apinex returned an error or a different decision model', 502);
    }
    const answers = answersOf(json.answers, 'apinex');
    for (const [id, question] of Object.entries(body.questions)) {
      const answer = answers[id] as { type?: unknown; choice?: unknown; noul?: unknown; score?: unknown } | null;
      const value = answer?.[question.type];
      if (!answer || answer.type !== question.type || (question.type === 'choice'
        ? typeof value !== 'string'
        : typeof value !== 'number' || !Number.isFinite(value))) {
        throw new DecisionError('apinex returned a missing or invalid typed answer', 502);
      }
    }
    return { answers, usage: usageOf(json.usage) };
  },
  beatapi: async (cred, modelId, body, signal) => {
    const json = await upstream(`${BEATAPI_BASE_URL}/systemone`, 'beatapi', {
      method: 'POST', redirect: 'error',
      headers: { Authorization: `Bearer ${cred.key}`, 'Content-Type': 'application/json' },
      body: JSON.stringify({ model: modelId, state: body.state, questions: body.questions }),
    }, signal) as { model?: string; answers?: unknown; usage?: unknown; error?: unknown } | null;
    if (!json || json.error || json.model !== modelId) {
      throw new DecisionError('beatapi returned an error or a different decision model', 502);
    }
    const answers = answersOf(json.answers, 'beatapi');
    for (const [id, question] of Object.entries(body.questions)) {
      const answer = answers[id] as { type?: unknown; choice?: unknown; noul?: unknown; score?: unknown } | null;
      const value = answer?.[question.type];
      if (!answer || answer.type !== question.type || (question.type === 'choice'
        ? typeof value !== 'string'
        : typeof value !== 'number' || !Number.isFinite(value))) {
        throw new DecisionError('beatapi returned a missing or invalid typed answer', 502);
      }
    }
    return { answers, usage: usageOf(json.usage) };
  },
  // Workers AI runs Clef through ai/run, with a short `model` selector
  // ("clef-flash") inside the body. Response: { result: { answers, usage } }.
  cloudflare: async (cred, modelId, body, signal) => {
    const { accountId, token } = cloudflareKey(cred.key);
    const selector = modelId.split('/').pop() ?? modelId;
    const json = await upstream(
      `https://api.cloudflare.com/client/v4/accounts/${encodeURIComponent(accountId)}/ai/run/${modelId}`,
      'cloudflare',
      {
        method: 'POST',
        headers: { Authorization: `Bearer ${token}`, 'Content-Type': 'application/json' },
        body: JSON.stringify({ model: selector, state: body.state, questions: body.questions }),
      },
      signal,
    ) as { success?: boolean; result?: { answers?: unknown; usage?: unknown }; errors?: Array<{ message?: string }> };
    if (json?.success === false || !json?.result) {
      throw new DecisionError(`cloudflare: ${json?.errors?.[0]?.message ?? 'no result'}`, 502);
    }
    return { answers: answersOf(json.result.answers, 'cloudflare'), usage: usageOf(json.result.usage) };
  },
  // OpenRouter serves decision models on its alpha decisions endpoint (not
  // under /api/v1), same System One body with the full model id.
  openrouter: async (cred, modelId, body, signal) => {
    const json = await upstream(
      'https://openrouter.ai/api/alpha/decisions',
      'openrouter',
      {
        method: 'POST',
        headers: {
          Authorization: `Bearer ${cred.key}`,
          'Content-Type': 'application/json',
          'HTTP-Referer': 'http://localhost:3001',
          'X-Title': 'FreeLLMAPI',
        },
        body: JSON.stringify({ model: modelId, state: body.state, questions: body.questions }),
      },
      signal,
    ) as { answers?: unknown; usage?: unknown; error?: { message?: string } };
    if (json?.error) throw new DecisionError(`openrouter: ${json.error.message ?? 'error'}`, 502);
    return { answers: answersOf(json?.answers, 'openrouter'), usage: usageOf(json?.usage) };
  },
};

/** Platforms with a decision adapter. catalog-sync gates decisionModels rows
 *  on this, the way SEARCH_ENGINE_PLATFORMS gates search rows. */
export const DECISION_PLATFORMS = new Set(Object.keys(ADAPTERS));

// ── Routing ─────────────────────────────────────────────────────────────────

function logDecision(row: DecisionModelRow, keyId: number | null, status: 'ok' | 'error', latencyMs: number, questionCount: number, inputTokens: number, clientAgent: string | null, error: string | null): void {
  try {
    getDb()
      .prepare(`INSERT INTO decision_requests (model_id, platform, key_id, status, latency_ms, question_count, input_tokens, client_agent, error)
                VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`)
      .run(row.model_id, row.platform, keyId, status, latencyMs, questionCount, inputTokens, clientAgent, error);
  } catch (e) {
    console.error('Failed to log decision request:', e);
  }
  // The per-key monthly caps (#1158) and GET /v1/usage read the
  // key_monthly_usage ledger. Chat, embeddings, media, rerank, moderation and
  // search settle into it through their `requests` rows; decisions log to
  // decision_requests, so a served decision settles its spend directly here.
  // Not a `requests` row: decisions fire every routing turn, and one there
  // would double every chat turn in the dashboard's request analytics.
  if (status === 'ok' && keyId != null) {
    try {
      recordMonthlyUsage(keyId, inputTokens);
      noteBudgetAlerts(keyId);
    } catch (e) {
      console.error('Failed to settle decision spend:', e);
    }
  }
}

function coolDown(row: DecisionModelRow, keyId: number, e: DecisionError): void {
  if (e.status === 401 || e.status === 403) {
    setCooldown(row.platform, row.model_id, keyId, BAD_KEY_COOLDOWN_MS);
  } else if (e.status === 402) {
    setCooldown(row.platform, row.model_id, keyId, NO_CREDIT_COOLDOWN_MS, 'credit');
  } else if (e.status === 429) {
    setCooldown(row.platform, row.model_id, keyId, e.retryAfterMs ?? RATE_LIMIT_COOLDOWN_MS);
  }
}

function hasUsableKey(platform: string): boolean {
  return !!getDb()
    .prepare("SELECT 1 FROM api_keys WHERE platform = ? AND enabled = 1 AND status IN ('healthy', 'unknown') LIMIT 1")
    .get(platform);
}

/** True when at least one enabled decision model has a usable key. */
export function decisionsAvailable(): boolean {
  return allDecisionRows().some(r => r.enabled === 1 && ADAPTERS[r.platform] && hasUsableKey(r.platform));
}

function decisionChain(model: string | undefined, skipPlatforms?: ReadonlySet<string>): DecisionModelRow[] {
  const enabled = allDecisionRows().filter(r => r.enabled === 1 && ADAPTERS[r.platform]);
  if (!model || model === 'auto') return skipPlatforms ? enabled.filter(r => !skipPlatforms.has(r.platform)) : enabled;
  const pinned = enabled.filter(r => r.model_id === model);
  if (pinned.length === 0) {
    const known = allDecisionRows().some(r => r.model_id === model);
    throw new DecisionError(
      known
        ? `Decision model '${model}' is turned off. Turn it on from Models → Decision, or use 'auto'.`
        : `Unknown decision model '${model}'. Use 'auto' or a model id from GET /v1/decisions/models.`,
      400,
    );
  }
  return pinned;
}

export async function runDecision(body: DecisionRequest, opts: DecisionOptions = {}): Promise<DecisionResponse> {
  const chain = decisionChain(opts.model, opts.skipPlatforms);
  const clientAgent = opts.clientAgent !== undefined ? opts.clientAgent : getClientContext().agent;
  const questionCount = Object.keys(body.questions).length;

  let lastError: DecisionError | null = null;
  let attempted = 0;
  const hints = new RetryHintTracker();
  for (const row of chain) {
    const tried = new Set<number>();
    for (;;) {
      if (opts.signal?.aborted) throw new DecisionError('decision cancelled by the client', 499);
      const { credential, budgetBlocked } = reserveProviderCredential(
        { platform: row.platform, key_id: null },
        ESTIMATED_TOKENS,
        keyId => tried.has(keyId) || isOnCooldown(row.platform, row.model_id, keyId),
      );
      if (!credential) {
        if (budgetBlocked) {
          lastError = new DecisionError('Monthly key budget exhausted', 429, 'quota_exceeded');
          hints.record(429, secondsUntilNextMonth() * 1000);
        }
        break;
      }
      tried.add(credential.id);
      attempted++;
      const started = Date.now();
      try {
        const out = await ADAPTERS[row.platform]!(credential, row.model_id, body, opts.signal);
        logDecision(row, credential.id, 'ok', Date.now() - started, questionCount, out.usage.input_tokens, clientAgent, null);
        // A served decision is evidence the key works, the same as a served
        // chat request (#1493) or search (#1612). Decision routing fires every
        // turn, so a decision key that serves fine but once got demoted to
        // 'error' by a blip had no way back into the pool.
        markKeyHealthyFromRequest(credential.id);
        return { model: row.model_id, provider: row.platform, answers: out.answers, usage: out.usage };
      } catch (err: any) {
        const e = err instanceof DecisionError ? err : new DecisionError(String(err?.message ?? err), 502);
        logDecision(row, credential.id, 'error', Date.now() - started, questionCount, 0, clientAgent, e.message.slice(0, 300));
        coolDown(row, credential.id, e);
        lastError = e;
        hints.record(e.status, e.retryAfterMs);
        if (opts.signal?.aborted) throw new DecisionError('decision cancelled by the client', 499);
        // A malformed request is the caller's problem, not this key's: move to
        // the next model rather than burning every key on it.
        if (e.status === 400 || e.status === 422) break;
      } finally {
        credential.release();
      }
    }
  }

  if (attempted === 0 && !lastError) {
    if (decisionsAvailable()) {
      throw new DecisionError(
        'Every decision key is cooling down (rate limited or rejected). Try again later.',
        429,
        'decision_keys_cooling_down',
      );
    }
    throw new DecisionError(
      'No decision model is available. Add a Cloudflare or OpenRouter key on the Keys page.',
      503,
      'no_decision_keys',
    );
  }
  // Only statuses the caller can act on pass through: an upstream 401 means
  // OUR provider key failed, not the caller's. A 400 from every model is most
  // likely a malformed request, so it is passed through as one.
  const status = lastError?.status === 429 ? 429 : lastError?.status === 400 || lastError?.status === 422 ? 400 : 502;
  throw new DecisionError(
    `All decision models failed${lastError ? ` (last: ${lastError.message.slice(0, 160)})` : ' (no usable keys)'}.`,
    status,
    lastError?.code,
    status === 429 ? hints.retryAfterMs() : undefined,
  );
}

export function decisionRetryAfterSec(err: DecisionError): number | undefined {
  if (err.retryAfterMs !== undefined) return Math.ceil(err.retryAfterMs / 1000);
  if (err.code === 'quota_exceeded') return secondsUntilNextMonth();
  return undefined;
}

// ── Dashboard ───────────────────────────────────────────────────────────────

export interface DecisionModelView {
  id: number;
  platform: string;
  modelId: string;
  displayName: string;
  priority: number;
  enabled: boolean;
  quotaLabel: string;
  contextWindow: number | null;
  keyCount: number;
  healthyKeyCount: number;
  decisionsThisMonth: number;
}

export function listDecisionModels(): DecisionModelView[] {
  const db = getDb();
  const platforms = [...DECISION_PLATFORMS];
  const keyRows = db.prepare(`
    SELECT platform, enabled, status FROM api_keys
     WHERE platform IN (${platforms.map(() => '?').join(', ')})
  `).all(...platforms) as { platform: string; enabled: number; status: string }[];
  const healthy = (k: { enabled: number; status: string }) => k.enabled === 1 && (k.status === 'healthy' || k.status === 'unknown');
  const monthCounts = new Map(
    (db.prepare(`
      SELECT platform, model_id, COUNT(*) AS n FROM decision_requests
       WHERE status = 'ok' AND created_at >= datetime('now', 'start of month')
       GROUP BY platform, model_id
    `).all() as { platform: string; model_id: string; n: number }[]).map(r => [`${r.platform}:${r.model_id}`, r.n]),
  );
  return allDecisionRows().map(row => {
    const meta = parseMeta(row.meta_json);
    const keys = keyRows.filter(k => k.platform === row.platform);
    return {
      id: row.id,
      platform: row.platform,
      modelId: row.model_id,
      displayName: row.display_name,
      priority: row.priority,
      enabled: row.enabled === 1,
      quotaLabel: row.quota_label,
      contextWindow: typeof meta.contextWindow === 'number' ? meta.contextWindow : null,
      keyCount: keys.length,
      healthyKeyCount: keys.filter(healthy).length,
      decisionsThisMonth: monthCounts.get(`${row.platform}:${row.model_id}`) ?? 0,
    };
  });
}

/** Turn a decision model on or off. False when the id is not a decision model. */
export function setDecisionModelEnabled(id: number, enabled: boolean): boolean {
  const info = getDb()
    .prepare('UPDATE media_models SET enabled = ? WHERE id = ? AND modality = ?')
    .run(enabled ? 1 : 0, id, DECISION_MODALITY);
  return info.changes > 0;
}