7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.
TypeScript
feat(router): analytics-driven bandit routing with weighted axes
Tashfeen413b5e4df507
diff --git a/client/src/pages/FallbackPage.tsx b/client/src/pages/FallbackPage.tsxindex d7512e01..170976d6 100644--- a/client/src/pages/FallbackPage.tsx+++ b/client/src/pages/FallbackPage.tsx@@ -136,6 +136,149 @@ function TokenUsageBar({ data }: { data: TokenUsageData }) {)}+// ── Bandit routing strategy ─────────────────────────────────────────────────+type RoutingStrategy = 'priority' | 'balanced' | 'smartest' | 'fastest' | 'reliable'++interface RoutingScore {+ modelDbId: number+ platform: string+ modelId: string+ displayName: string+ enabled: boolean+ reliability: number+ speed: number+ intelligence: number+ headroom: number+ rateLimit: number+ score: number+ totalRequests: number+}++interface RoutingData {+ strategy: RoutingStrategy+ weights: { reliability: number; speed: number; intelligence: number } | null+ scores: RoutingScore[]+}++const STRATEGIES: { key: RoutingStrategy; label: string; blurb: string }[] = [+ { key: 'priority', label: 'Manual', blurb: 'Use the hand-ordered chain below (no scoring).' },+ { key: 'balanced', label: 'Balanced', blurb: 'Reliability leads; speed and intelligence split the rest.' },+ { key: 'smartest', label: 'Smartest', blurb: 'Prefer the most capable model that still works.' },+ { key: 'fastest', label: 'Fastest', blurb: 'Prefer the fastest model that still works.' },+ { key: 'reliable', label: 'Most reliable', blurb: 'Maximize success rate above all.' },+]++// A 0..1 value as a thin horizontal bar with the number beside it.+function AxisBar({ value, color }: { value: number; color: string }) {+ return (+ <div className="flex items-center gap-1.5">+ <div className="h-1.5 w-12 rounded-full bg-muted overflow-hidden">+ <div className="h-full rounded-full" style={{ width: `${Math.round(value * 100)}%`, backgroundColor: color }} />+ </div>+ <span className="font-mono text-[11px] text-muted-foreground tabular-nums w-7 text-right">+ {Math.round(value * 100)}+ </span>+ </div>+ )+}++function RoutingPanel({+ data,+ onSelect,+ pending,+}: {+ data: RoutingData+ onSelect: (s: RoutingStrategy) => void+ pending: boolean+}) {+ const active = STRATEGIES.find(s => s.key === data.strategy) ?? STRATEGIES[0]+ const banditOn = data.strategy !== 'priority'+ // Only show models that have been observed or are enabled, ranked by score.+ const rows = data.scores.filter(s => s.enabled || s.totalRequests > 0)++ return (+ <section className="rounded-lg border bg-card p-5">+ <div className="flex items-baseline justify-between mb-1">+ <h2 className="text-sm font-medium">Routing strategy</h2>+ <span className="text-xs text-muted-foreground">{active.blurb}</span>+ </div>++ <div className="mt-3 inline-flex flex-wrap gap-1 rounded-lg border p-1">+ {STRATEGIES.map(s => (+ <button+ key={s.key}+ disabled={pending}+ onClick={() => onSelect(s.key)}+ className={`px-3 py-1.5 text-xs rounded-md transition-colors ${+ s.key === data.strategy+ ? 'bg-foreground text-background font-medium'+ : 'text-muted-foreground hover:text-foreground hover:bg-muted'+ }`}+ >+ {s.label}+ </button>+ ))}+ </div>++ {data.weights && (+ <p className="mt-2 text-xs text-muted-foreground tabular-nums">+ Weights — reliability {Math.round(data.weights.reliability * 100)}% ·+ {' '}speed {Math.round(data.weights.speed * 100)}% ·+ {' '}intelligence {Math.round(data.weights.intelligence * 100)}%+ </p>+ )}++ {rows.length > 0 && (+ <div className="mt-4 overflow-x-auto">+ <table className="w-full text-xs">+ <thead>+ <tr className="text-left text-muted-foreground border-b">+ <th className="py-1.5 pr-3 font-medium">{banditOn ? '#' : ''}</th>+ <th className="py-1.5 pr-3 font-medium">Model</th>+ <th className="py-1.5 pr-3 font-medium">Reliability</th>+ <th className="py-1.5 pr-3 font-medium">Speed</th>+ <th className="py-1.5 pr-3 font-medium">Intelligence</th>+ <th className="py-1.5 pr-3 font-medium" title="Free-quota headroom × rate-limit guardrails">Guardrails</th>+ <th className="py-1.5 pr-3 font-medium text-right">Score</th>+ </tr>+ </thead>+ <tbody>+ {rows.map((r, i) => (+ <tr key={r.modelDbId} className={`border-b last:border-0 ${r.enabled ? '' : 'opacity-40'}`}>+ <td className="py-1.5 pr-3 font-mono text-muted-foreground tabular-nums">{banditOn ? i + 1 : '·'}</td>+ <td className="py-1.5 pr-3">+ <span className="font-medium">{r.displayName}</span>+ <span className="ml-1.5 text-muted-foreground">{r.platform}</span>+ {r.totalRequests > 0 && (+ <span className="ml-1.5 text-muted-foreground/70">({r.totalRequests} obs)</span>+ )}+ </td>+ <td className="py-1.5 pr-3"><AxisBar value={r.reliability} color="#22c55e" /></td>+ <td className="py-1.5 pr-3"><AxisBar value={r.speed} color="#3b82f6" /></td>+ <td className="py-1.5 pr-3"><AxisBar value={r.intelligence} color="#a855f7" /></td>+ <td className="py-1.5 pr-3 font-mono text-muted-foreground tabular-nums">+ {(r.headroom * r.rateLimit) < 0.999+ ? `×${(r.headroom * r.rateLimit).toFixed(2)}`+ : '—'}+ </td>+ <td className="py-1.5 pr-3 text-right font-mono font-medium tabular-nums">+ {r.score.toFixed(3)}+ </td>+ </tr>+ ))}+ </tbody>+ </table>+ {!banditOn && (+ <p className="mt-2 text-xs text-muted-foreground">+ Preview only — manual order (below) is active. Pick a strategy above to route by these scores.+ </p>+ )}+ </div>+ )}+ </section>+ )+}+function SortableModelRow({entry,index,@@ -239,6 +382,22 @@ export default function FallbackPage() {},})+ // Bandit routing: live per-model scores refresh on a short interval so the+ // table reflects recent traffic without a manual reload.+ const { data: routing } = useQuery<RoutingData>({+ queryKey: ['fallback', 'routing'],+ queryFn: () => apiFetch('/api/fallback/routing'),+ refetchInterval: 15_000,+ })++ const strategyMutation = useMutation({+ mutationFn: (strategy: RoutingStrategy) =>+ apiFetch('/api/fallback/routing', { method: 'PUT', body: JSON.stringify({ strategy }) }),+ onSuccess: () => {+ queryClient.invalidateQueries({ queryKey: ['fallback', 'routing'] })+ },+ })+const allEntries = localEntries ?? entriesconst displayEntries = allEntries.filter(e => e.keyCount > 0)const unconfiguredPlatforms = [...new Set(allEntries.filter(e => e.keyCount === 0).map(e => e.platform))]@@ -303,6 +462,14 @@ export default function FallbackPage() {/><div className="space-y-6">+ {routing && (+ <RoutingPanel+ data={routing}+ onSelect={(s) => strategyMutation.mutate(s)}+ pending={strategyMutation.isPending}+ />+ )}+{tokenUsage && tokenUsage.totalBudget > 0 && (<TokenUsageBar data={tokenUsage} />)}diff --git a/server/package.json b/server/package.jsonindex fc580996..c4266577 100644--- a/server/package.json+++ b/server/package.json@@ -1,6 +1,6 @@{"name": "@freellmapi/server",- "version": "0.1.0",+ "version": "0.2.0","private": true,"type": "module","scripts": {diff --git a/server/src/__tests__/services/router-bandit.test.ts b/server/src/__tests__/services/router-bandit.test.tsnew file mode 100644index 00000000..c0bd6408--- /dev/null+++ b/server/src/__tests__/services/router-bandit.test.ts@@ -0,0 +1,162 @@+import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest';+import {+ routeRequest, refreshStatsCache, getRoutingStrategy, setRoutingStrategy, getRoutingScores,+} from '../../services/router.js';+import * as ratelimit from '../../services/ratelimit.js';+import { getDb, initDb } from '../../db/index.js';++vi.mock('../../services/ratelimit.js', async () => {+ const actual = await vi.importActual('../../services/ratelimit.js');+ return {+ ...actual,+ canMakeRequest: vi.fn(() => true),+ canUseTokens: vi.fn(() => true),+ isOnCooldown: vi.fn(() => false),+ };+});++vi.mock('../../lib/crypto.js', async () => {+ const actual = await vi.importActual('../../lib/crypto.js');+ return { ...actual, decrypt: vi.fn(() => 'mocked-api-key') };+});++const ORIGINAL_DEV_MODE = process.env.DEV_MODE;+const ORIGINAL_NODE_ENV = process.env.NODE_ENV;++// Insert a model + its fallback entry; returns the model id.+function addModel(opts: {+ platform: string; modelId: string; name: string;+ intelligenceRank: number; sizeLabel: string; budget: string; priority: number;+}): number {+ const db = getDb();+ db.prepare(`+ INSERT INTO models (platform, model_id, display_name, intelligence_rank, speed_rank, size_label, monthly_token_budget, enabled)+ VALUES (?, ?, ?, ?, ?, ?, ?, 1)+ `).run(opts.platform, opts.modelId, opts.name, opts.intelligenceRank, 1, opts.sizeLabel, opts.budget);+ const id = (db.prepare('SELECT id FROM models WHERE platform = ? AND model_id = ?')+ .get(opts.platform, opts.modelId) as { id: number }).id;+ db.prepare('INSERT INTO fallback_config (model_db_id, priority, enabled) VALUES (?, ?, 1)').run(id, opts.priority);+ // every platform needs at least one healthy key to be routable+ db.prepare(`+ INSERT INTO api_keys (platform, label, encrypted_key, iv, auth_tag, status, enabled)+ VALUES (?, 'k', 'enc', 'iv', 'tag', 'healthy', 1)+ `).run(opts.platform);+ return id;+}++// Insert N request rows (now → age 0, decay weight 1) for stats.+function addHistory(platform: string, modelId: string, opts: {+ successes: number; failures: number; outTokens?: number; latencyMs?: number; ttfbMs?: number | null;+}) {+ const db = getDb();+ const ins = db.prepare(`+ INSERT INTO requests (platform, model_id, key_id, status, input_tokens, output_tokens, latency_ms, error, ttfb_ms)+ VALUES (?, ?, 1, ?, 0, ?, ?, ?, ?)+ `);+ for (let i = 0; i < opts.successes; i++) {+ ins.run(platform, modelId, 'success', opts.outTokens ?? 100, opts.latencyMs ?? 1000, null, opts.ttfbMs ?? null);+ }+ for (let i = 0; i < opts.failures; i++) {+ ins.run(platform, modelId, 'error', 0, opts.latencyMs ?? 1000, 'boom', opts.ttfbMs ?? null);+ }+}++function pickCounts(runs: number): Record<string, number> {+ const counts: Record<string, number> = {};+ for (let i = 0; i < runs; i++) {+ const r = routeRequest(100);+ counts[r.modelId] = (counts[r.modelId] ?? 0) + 1;+ }+ return counts;+}++describe('bandit router', () => {+ beforeEach(() => {+ process.env.DEV_MODE = 'true';+ process.env.NODE_ENV = 'test';+ initDb(':memory:');+ // initDb seeds the real catalog; wipe it so each test controls its own+ // models/keys/history (and seeded models don't share a platform with ours).+ getDb().exec('DELETE FROM fallback_config; DELETE FROM api_keys; DELETE FROM models; DELETE FROM requests;');+ vi.clearAllMocks();+ (ratelimit.canMakeRequest as any).mockReturnValue(true);+ (ratelimit.canUseTokens as any).mockReturnValue(true);+ (ratelimit.isOnCooldown as any).mockReturnValue(false);+ });++ afterEach(() => {+ if (ORIGINAL_DEV_MODE === undefined) delete process.env.DEV_MODE; else process.env.DEV_MODE = ORIGINAL_DEV_MODE;+ if (ORIGINAL_NODE_ENV === undefined) delete process.env.NODE_ENV; else process.env.NODE_ENV = ORIGINAL_NODE_ENV;+ });++ it('strategy persists to and from settings; defaults to balanced', () => {+ expect(getRoutingStrategy()).toBe('balanced');+ setRoutingStrategy('smartest');+ expect(getRoutingStrategy()).toBe('smartest');+ setRoutingStrategy('priority');+ expect(getRoutingStrategy()).toBe('priority');+ });++ it('priority strategy follows the manual chain order deterministically', () => {+ addModel({ platform: 'google', modelId: 'a', name: 'A', intelligenceRank: 9, sizeLabel: 'Small', budget: '~10M', priority: 1 });+ addModel({ platform: 'groq', modelId: 'b', name: 'B', intelligenceRank: 1, sizeLabel: 'Frontier', budget: '~10M', priority: 2 });+ setRoutingStrategy('priority');+ refreshStatsCache(getDb(), true);+ const counts = pickCounts(50);+ expect(counts['a']).toBe(50); // priority 1 always wins regardless of intelligence+ });++ it('balanced strategy favors the more reliable model', () => {+ addModel({ platform: 'google', modelId: 'good', name: 'Good', intelligenceRank: 3, sizeLabel: 'Large', budget: '~50M', priority: 1 });+ addModel({ platform: 'groq', modelId: 'flaky', name: 'Flaky', intelligenceRank: 3, sizeLabel: 'Large', budget: '~50M', priority: 2 });+ addHistory('google', 'good', { successes: 60, failures: 1 });+ addHistory('groq', 'flaky', { successes: 5, failures: 40 });+ setRoutingStrategy('balanced');+ refreshStatsCache(getDb(), true);+ const counts = pickCounts(300);+ expect(counts['good'] ?? 0).toBeGreaterThan((counts['flaky'] ?? 0) * 3);+ });++ it('explores unseen models — both get picked at least once', () => {+ addModel({ platform: 'google', modelId: 'x', name: 'X', intelligenceRank: 3, sizeLabel: 'Large', budget: '~50M', priority: 1 });+ addModel({ platform: 'groq', modelId: 'y', name: 'Y', intelligenceRank: 3, sizeLabel: 'Large', budget: '~50M', priority: 2 });+ setRoutingStrategy('balanced');+ refreshStatsCache(getDb(), true);+ const counts = pickCounts(200);+ expect(counts['x'] ?? 0).toBeGreaterThan(0);+ expect(counts['y'] ?? 0).toBeGreaterThan(0);+ });++ it('smartest vs fastest flips which model wins, at equal reliability', () => {+ // Smart: frontier tier, slow. Fast: small tier, high throughput. Equal success.+ addModel({ platform: 'google', modelId: 'smart', name: 'Smart', intelligenceRank: 1, sizeLabel: 'Frontier', budget: '~50M', priority: 1 });+ addModel({ platform: 'groq', modelId: 'fast', name: 'Fast', intelligenceRank: 9, sizeLabel: 'Small', budget: '~50M', priority: 2 });+ addHistory('google', 'smart', { successes: 40, failures: 1, outTokens: 100, latencyMs: 3000, ttfbMs: 2500 });+ addHistory('groq', 'fast', { successes: 40, failures: 1, outTokens: 1000, latencyMs: 1000, ttfbMs: 150 });++ setRoutingStrategy('smartest');+ refreshStatsCache(getDb(), true);+ const smartRun = pickCounts(300);+ expect((smartRun['smart'] ?? 0)).toBeGreaterThan(smartRun['fast'] ?? 0);++ setRoutingStrategy('fastest');+ refreshStatsCache(getDb(), true);+ const fastRun = pickCounts(300);+ expect((fastRun['fast'] ?? 0)).toBeGreaterThan(fastRun['smart'] ?? 0);+ });++ it('getRoutingScores returns a per-axis breakdown ranked by score', () => {+ addModel({ platform: 'google', modelId: 'm1', name: 'M1', intelligenceRank: 1, sizeLabel: 'Frontier', budget: '~50M', priority: 1 });+ addHistory('google', 'm1', { successes: 30, failures: 0, outTokens: 500, latencyMs: 1000, ttfbMs: 200 });+ setRoutingStrategy('balanced');+ refreshStatsCache(getDb(), true);+ const { strategy, weights, scores } = getRoutingScores();+ expect(strategy).toBe('balanced');+ expect(weights).toEqual({ reliability: 0.5, speed: 0.25, intelligence: 0.25 });+ expect(scores).toHaveLength(1);+ expect(scores[0]).toMatchObject({ modelId: 'm1', enabled: true });+ expect(scores[0].reliability).toBeGreaterThan(0.9);+ expect(scores[0].score).toBeGreaterThan(0);+ expect(scores[0].score).toBeLessThanOrEqual(1);+ });+});diff --git a/server/src/__tests__/services/router.test.ts b/server/src/__tests__/services/router.test.tsindex e0e9e489..bf80611f 100644--- a/server/src/__tests__/services/router.test.ts+++ b/server/src/__tests__/services/router.test.ts@@ -1,7 +1,7 @@import { describe, it, expect, beforeAll, beforeEach } from 'vitest';import { initDb, getDb } from '../../db/index.js';import { encrypt } from '../../lib/crypto.js';-import { routeRequest } from '../../services/router.js';+import { routeRequest, setRoutingStrategy } from '../../services/router.js';describe('Router', () => {beforeAll(() => {@@ -11,6 +11,9 @@ describe('Router', () => {beforeEach(() => {const db = getDb();+ // These cases assert the manual priority order specifically; pin it so the+ // bandit (now the default strategy) doesn't reorder by score.+ setRoutingStrategy('priority');db.prepare('DELETE FROM api_keys').run();// Reset fallback order to intelligence rankingconst models = db.prepare('SELECT id, intelligence_rank FROM models ORDER BY intelligence_rank ASC').all() as any[];diff --git a/server/src/__tests__/services/routing-exhaustion.test.ts b/server/src/__tests__/services/routing-exhaustion.test.tsindex 45d1304b..c2df83f9 100644--- a/server/src/__tests__/services/routing-exhaustion.test.ts+++ b/server/src/__tests__/services/routing-exhaustion.test.ts@@ -1,5 +1,5 @@import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest';-import { routeRequest } from '../../services/router.js';+import { routeRequest, setRoutingStrategy } from '../../services/router.js';import * as ratelimit from '../../services/ratelimit.js';import { getDb, initDb } from '../../db/index.js';import * as crypto from '../../lib/crypto.js';@@ -45,8 +45,13 @@ describe('Routing Key Exhaustion', () => {process.env.DEV_MODE = 'true';process.env.NODE_ENV = 'test';initDb(':memory:');+ // This suite asserts deterministic key/model fallback mechanics, which are+ // strategy-independent — pin the legacy priority order so the bandit's+ // score-based reordering (now the default) doesn't pick seeded catalog+ // models that share the 'google' platform.+ setRoutingStrategy('priority');const db = getDb();-+// Setup: 2 models (Pro and Flash)// Pro is higher priority (priority 1), Flash is lower (priority 2)db.prepare("INSERT INTO models (platform, model_id, display_name, intelligence_rank, speed_rank, enabled) VALUES ('google', 'gemini-1.5-pro', 'Pro', 1, 1, 1)").run();diff --git a/server/src/__tests__/services/scoring.test.ts b/server/src/__tests__/services/scoring.test.tsnew file mode 100644index 00000000..b62d7140--- /dev/null+++ b/server/src/__tests__/services/scoring.test.ts@@ -0,0 +1,145 @@+import { describe, it, expect } from 'vitest';+import {+ BANDIT_PRESETS, combineScore, speedScore, intelligenceScore,+ headroomFactor, rateLimitFactor, sampleBeta, reliabilityPosterior,+ expectedReliability, SPEED_PRIOR, HEADROOM_FLOOR,+} from '../../services/scoring.js';++describe('scoring: reliability posterior', () => {+ it('uniform prior makes an unseen model genuinely uncertain (mean 0.5)', () => {+ expect(expectedReliability(0, 0)).toBeCloseTo(0.5, 5);+ });++ it('successes pull the expected rate up, failures down', () => {+ expect(expectedReliability(9, 1)).toBeGreaterThan(0.7);+ expect(expectedReliability(1, 9)).toBeLessThan(0.3);+ });++ it('posterior adds the priors to the observed counts', () => {+ expect(reliabilityPosterior(5, 3)).toEqual({ alpha: 6, beta: 4 });+ });+});++describe('scoring: speed axis', () => {+ it('returns the exploration prior when there is no data at all', () => {+ expect(speedScore(0, null)).toBe(SPEED_PRIOR);+ });++ it('is bounded in [0,1] and monotonic in throughput', () => {+ const a = speedScore(10, null);+ const b = speedScore(50, null);+ const c = speedScore(200, null);+ expect(a).toBeGreaterThanOrEqual(0);+ expect(c).toBeLessThanOrEqual(1);+ expect(a).toBeLessThan(b);+ expect(b).toBeLessThan(c);+ });++ it('a fast TTFB raises the score versus a slow one at equal throughput', () => {+ const fast = speedScore(80, 200);+ const slow = speedScore(80, 6000);+ expect(fast).toBeGreaterThan(slow);+ });++ it('falls back to throughput-only when TTFB is unknown', () => {+ expect(speedScore(80, null)).toBeGreaterThan(0);+ });+});++describe('scoring: intelligence axis', () => {+ it('maps min→0, max→1', () => {+ expect(intelligenceScore(1000, 1000, 4000)).toBeCloseTo(0, 5);+ expect(intelligenceScore(4000, 1000, 4000)).toBeCloseTo(1, 5);+ expect(intelligenceScore(2500, 1000, 4000)).toBeCloseTo(0.5, 5);+ });++ it('returns neutral-high when all models are equal', () => {+ expect(intelligenceScore(5, 5, 5)).toBe(1);+ });+});++describe('scoring: guardrails', () => {+ it('headroom is 1 with plenty left and ramps to the floor when exhausted', () => {+ expect(headroomFactor(0, 1_000_000)).toBe(1);+ expect(headroomFactor(500_000, 1_000_000)).toBe(1); // 50% left → no opinion+ expect(headroomFactor(1_000_000, 1_000_000)).toBeCloseTo(HEADROOM_FLOOR, 5); // fully used+ expect(headroomFactor(900_000, 1_000_000)).toBeLessThan(1); // 10% left → protecting+ });++ it('unknown budget yields no opinion (factor 1)', () => {+ expect(headroomFactor(123, 0)).toBe(1);+ });++ it('rate-limit factor is 1 at no penalty and damped but non-zero at max', () => {+ expect(rateLimitFactor(0)).toBe(1);+ expect(rateLimitFactor(10)).toBeCloseTo(0.4, 5);+ expect(rateLimitFactor(100)).toBeCloseTo(0.4, 5); // clamped+ });+});++describe('scoring: combineScore', () => {+ const perfect = { reliability: 1, speed: 1, intelligence: 1, headroom: 1, rateLimit: 1 };++ it('stays within [0,1] for in-range inputs', () => {+ expect(combineScore(perfect, BANDIT_PRESETS.balanced)).toBeLessThanOrEqual(1);+ expect(combineScore({ reliability: 0, speed: 0, intelligence: 0, headroom: 1, rateLimit: 1 }, BANDIT_PRESETS.balanced)).toBe(0);+ });++ it('a 100%-reliable slow model beats a 0%-reliable fast one under balanced — no hand-cap needed', () => {+ const reliable = combineScore({ reliability: 1, speed: 0.1, intelligence: 0.5, headroom: 1, rateLimit: 1 }, BANDIT_PRESETS.balanced);+ const flaky = combineScore({ reliability: 0, speed: 1, intelligence: 0.5, headroom: 1, rateLimit: 1 }, BANDIT_PRESETS.balanced);+ expect(reliable).toBeGreaterThan(flaky);+ });++ it('the smartest preset ranks a high-intelligence model above a fast one', () => {+ const smart = combineScore({ reliability: 0.8, speed: 0.2, intelligence: 1, headroom: 1, rateLimit: 1 }, BANDIT_PRESETS.smartest);+ const fast = combineScore({ reliability: 0.8, speed: 1, intelligence: 0.2, headroom: 1, rateLimit: 1 }, BANDIT_PRESETS.smartest);+ expect(smart).toBeGreaterThan(fast);+ });++ it('the fastest preset flips that ordering', () => {+ const smart = combineScore({ reliability: 0.8, speed: 0.2, intelligence: 1, headroom: 1, rateLimit: 1 }, BANDIT_PRESETS.fastest);+ const fast = combineScore({ reliability: 0.8, speed: 1, intelligence: 0.2, headroom: 1, rateLimit: 1 }, BANDIT_PRESETS.fastest);+ expect(fast).toBeGreaterThan(smart);+ });++ it('guardrails multiply the base down', () => {+ const base = combineScore(perfect, BANDIT_PRESETS.balanced);+ const throttled = combineScore({ ...perfect, rateLimit: 0.4 }, BANDIT_PRESETS.balanced);+ expect(throttled).toBeCloseTo(base * 0.4, 5);+ });++ it('every preset weight vector sums to 1', () => {+ for (const w of Object.values(BANDIT_PRESETS)) {+ expect(w.reliability + w.speed + w.intelligence).toBeCloseTo(1, 5);+ }+ });+});++describe('scoring: Beta sampler (Thompson exploration)', () => {+ it('draws stay within (0,1)', () => {+ for (let i = 0; i < 1000; i++) {+ const x = sampleBeta(3, 5);+ expect(x).toBeGreaterThanOrEqual(0);+ expect(x).toBeLessThanOrEqual(1);+ }+ });++ it('the sample mean approximates alpha/(alpha+beta)', () => {+ let sum = 0;+ const n = 20000;+ for (let i = 0; i < n; i++) sum += sampleBeta(8, 2);+ expect(sum / n).toBeCloseTo(0.8, 1); // E[Beta(8,2)] = 0.8+ });++ it('explores: a strong model does NOT win every single draw vs a decent one', () => {+ // Beta(20,2) ≈ 0.91 vs Beta(12,4) ≈ 0.75 — overlapping tails mean the+ // weaker model should still sometimes sample higher. That overlap is what+ // keeps the router from freezing onto a single model.+ let weakerWonAtLeastOnce = false;+ for (let i = 0; i < 2000 && !weakerWonAtLeastOnce; i++) {+ if (sampleBeta(12, 4) > sampleBeta(20, 2)) weakerWonAtLeastOnce = true;+ }+ expect(weakerWonAtLeastOnce).toBe(true);+ });+});diff --git a/server/src/db/index.ts b/server/src/db/index.tsindex a7469bea..3129873f 100644--- a/server/src/db/index.ts+++ b/server/src/db/index.ts@@ -162,6 +162,17 @@ function createTables(db: Database.Database) {ensureRequestKeyIdColumn(db);ensureApiKeysBaseUrlColumn(db);+ ensureRequestTtfbColumn(db);+}++// `ttfb_ms` is the time-to-first-byte for streaming responses (ms from dispatch+// to the first chunk). NULL for non-streaming or pre-existing rows. Feeds the+// bandit router's latency axis (server/src/services/scoring.ts).+function ensureRequestTtfbColumn(db: Database.Database) {+ const columns = db.prepare('PRAGMA table_info(requests)').all() as { name: string }[];+ if (!columns.some(col => col.name === 'ttfb_ms')) {+ db.prepare('ALTER TABLE requests ADD COLUMN ttfb_ms INTEGER').run();+ }}function ensureRequestKeyIdColumn(db: Database.Database) {@@ -1374,3 +1385,18 @@ export function regenerateUnifiedKey(): string {db.prepare("UPDATE settings SET value = ? WHERE key = 'unified_api_key'").run(key);return key;}++// Generic key/value settings accessors (used by routing strategy, etc.).+export function getSetting(key: string): string | undefined {+ const db = getDb();+ const row = db.prepare('SELECT value FROM settings WHERE key = ?').get(key) as { value: string } | undefined;+ return row?.value;+}++export function setSetting(key: string, value: string): void {+ const db = getDb();+ db.prepare(`+ INSERT INTO settings (key, value) VALUES (?, ?)+ ON CONFLICT(key) DO UPDATE SET value = excluded.value+ `).run(key, value);+}diff --git a/server/src/lib/budget.ts b/server/src/lib/budget.tsnew file mode 100644index 00000000..fb7a86a7--- /dev/null+++ b/server/src/lib/budget.ts@@ -0,0 +1,13 @@+// Monthly free-tier budgets are stored as human labels like '~120M', '~50-100M',+// '~12M', or '~500K'. Parse the upper bound to an absolute token count for+// quota math (headroom guardrail, token-usage bar). Returns 0 for unknown/empty+// labels, which callers treat as "no budget info".+export function parseBudget(s: string): number {+ if (!s) return 0;+ const m = s.match(/~?([\d.]+)(?:-([\d.]+))?([MK])?/);+ if (!m) return 0;+ const high = parseFloat(m[2] ?? m[1]);+ if (Number.isNaN(high)) return 0;+ const unit = m[3] === 'M' ? 1_000_000 : m[3] === 'K' ? 1_000 : 1;+ return high * unit;+}diff --git a/server/src/routes/fallback.ts b/server/src/routes/fallback.tsindex 7cbd3b23..678d5c9b 100644--- a/server/src/routes/fallback.ts+++ b/server/src/routes/fallback.ts@@ -2,10 +2,35 @@ import { Router } from 'express';import type { Request, Response } from 'express';import { z } from 'zod';import { getDb } from '../db/index.js';-import { getAllPenalties } from '../services/router.js';+import { getAllPenalties, getRoutingScores, getRoutingStrategy, setRoutingStrategy } from '../services/router.js';+import { BANDIT_PRESETS, type RoutingStrategy } from '../services/scoring.js';+import { parseBudget } from '../lib/budget.js';export const fallbackRouter = Router();+// ── Bandit routing strategy ─────────────────────────────────────────────────+// GET /routing → active strategy, preset weights, and the per-model score+// breakdown (reliability / speed / intelligence + guardrails).+fallbackRouter.get('/routing', (_req: Request, res: Response) => {+ res.json(getRoutingScores());+});++const routingSchema = z.object({+ strategy: z.enum(['priority', 'balanced', 'smartest', 'fastest', 'reliable']),+});++// PUT /routing → switch strategy. Presets are just weight vectors over the three+// axes; 'priority' falls back to the legacy manual chain order.+fallbackRouter.put('/routing', (req: Request, res: Response) => {+ const parsed = routingSchema.safeParse(req.body);+ if (!parsed.success) {+ res.status(400).json({ error: { message: parsed.error.errors.map(e => e.message).join(', ') } });+ return;+ }+ setRoutingStrategy(parsed.data.strategy as RoutingStrategy);+ res.json({ strategy: getRoutingStrategy(), presets: BANDIT_PRESETS });+});+// Get fallback chain (with dynamic penalties)fallbackRouter.get('/', (_req: Request, res: Response) => {const db = getDb();@@ -144,14 +169,6 @@ fallbackRouter.get('/token-usage', (_req: Request, res: Response) => {ORDER BY fc.priority ASC`).all() as { platform: string; model_id: string; display_name: string; monthly_token_budget: string; priority: number }[];- function parseBudget(s: string): number {- const m = s.match(/~?([\d.]+)(?:-([\d.]+))?([MK])?/);- if (!m) return 0;- const high = parseFloat(m[2] ?? m[1]);- const unit = m[3] === 'M' ? 1_000_000 : m[3] === 'K' ? 1_000 : 1;- return high * unit;- }-// Build per-model breakdown (only platforms with keys)const modelBudgets = models.filter(m => platformSet.has(m.platform))diff --git a/server/src/routes/proxy.ts b/server/src/routes/proxy.tsindex 23391384..961bad3c 100644--- a/server/src/routes/proxy.ts+++ b/server/src/routes/proxy.ts@@ -407,6 +407,7 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {// instead of a silently truncated stream.let totalOutputTokens = 0;let streamStarted = false;+ let ttfbMs: number | null = null;try {const gen = route.provider.streamChatCompletion(route.apiKey, messages, route.modelId,@@ -415,6 +416,9 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {for await (const chunk of gen) {if (!streamStarted) {+ // Time-to-first-byte: dispatch → first chunk. Feeds the router's+ // latency axis (server/src/services/scoring.ts).+ ttfbMs = Date.now() - start;res.setHeader('Content-Type', 'text/event-stream');res.setHeader('Cache-Control', 'no-cache');res.setHeader('Connection', 'keep-alive');@@ -438,7 +442,7 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {recordTokens(route.platform, route.modelId, route.keyId, estimatedInputTokens + totalOutputTokens);recordSuccess(route.modelDbId);setStickyModel(messages, route.modelDbId);- logRequest(route.platform, route.modelId, route.keyId, 'success', estimatedInputTokens, totalOutputTokens, Date.now() - start, null);+ logRequest(route.platform, route.modelId, route.keyId, 'success', estimatedInputTokens, totalOutputTokens, Date.now() - start, null, ttfbMs);return;} catch (streamErr: any) {if (streamStarted) {@@ -531,13 +535,14 @@ export function logRequest(outputTokens: number,latencyMs: number,error: string | null,+ ttfbMs: number | null = null,) {try {const db = getDb();db.prepare(`- INSERT INTO requests (platform, model_id, key_id, status, input_tokens, output_tokens, latency_ms, error)- VALUES (?, ?, ?, ?, ?, ?, ?, ?)- `).run(platform, modelId, keyId, status, inputTokens, outputTokens, latencyMs, error);+ INSERT INTO requests (platform, model_id, key_id, status, input_tokens, output_tokens, latency_ms, error, ttfb_ms)+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)+ `).run(platform, modelId, keyId, status, inputTokens, outputTokens, latencyMs, error, ttfbMs);} catch (e) {console.error('Failed to log request:', e);}diff --git a/server/src/scripts/routing-sim.ts b/server/src/scripts/routing-sim.tsnew file mode 100644index 00000000..3687314b--- /dev/null+++ b/server/src/scripts/routing-sim.ts@@ -0,0 +1,149 @@+/**+ * Routing simulation — exercises the real routeRequest() against an in-memory+ * DB seeded with synthetic traffic, so you can SEE how each strategy distributes+ * requests and how the bandit adapts when a model starts failing.+ *+ * cd server && npm run build && node dist/scripts/routing-sim.js+ */+process.env.ENCRYPTION_KEY = '0'.repeat(64);+process.env.DEV_MODE = 'true';++import { initDb, getDb } from '../db/index.js';+import { encrypt } from '../lib/crypto.js';+import {+ routeRequest, refreshStatsCache, setRoutingStrategy, getRoutingScores,+} from '../services/router.js';+import type { RoutingStrategy } from '../services/scoring.js';++interface Profile {+ platform: string; modelId: string; name: string;+ intelligenceRank: number; sizeLabel: string; budget: string;+ // synthetic history+ successes: number; failures: number; outTokens: number; latencyMs: number; ttfbMs: number;+}++// Deliberately no model that wins on every axis — so each strategy's weight+// vector picks a DIFFERENT winner (that's the whole point of the redesign).+const PROFILES: Profile[] = [+ // Frontier intelligence, good-not-great reliability, genuinely slow → wins SMARTEST.+ { platform: 'google', modelId: 'gemini-x', name: 'Frontier-Smart', intelligenceRank: 1, sizeLabel: 'Frontier', budget: '~50M', successes: 22, failures: 4, outTokens: 200, latencyMs: 5000, ttfbMs: 2500 },+ // Small/dumb but blazing fast and reliable → wins FASTEST.+ { platform: 'groq', modelId: 'llama-fast', name: 'Speed-King', intelligenceRank: 9, sizeLabel: 'Small', budget: '~50M', successes: 27, failures: 3, outTokens: 1100, latencyMs: 700, ttfbMs: 110 },+ // Middling intel/speed but rock-solid reliability → wins RELIABLE.+ { platform: 'openrouter', modelId: 'rock', name: 'Rock-Solid', intelligenceRank: 5, sizeLabel: 'Medium', budget: '~50M', successes: 40, failures: 1, outTokens: 250, latencyMs: 2000, ttfbMs: 1500 },+ // Large/smart but flaky — high intelligence dragged down by failures.+ { platform: 'cerebras', modelId: 'smart-flaky', name: 'Smart-Flaky', intelligenceRank: 3, sizeLabel: 'Large', budget: '~50M', successes: 18, failures: 14, outTokens: 500, latencyMs: 1500, ttfbMs: 600 },+];++function seed() {+ const db = getDb();+ db.exec('DELETE FROM fallback_config; DELETE FROM api_keys; DELETE FROM models; DELETE FROM requests;');+ const insModel = db.prepare(`+ INSERT INTO models (platform, model_id, display_name, intelligence_rank, speed_rank, size_label, rpm_limit, rpd_limit, tpm_limit, tpd_limit, monthly_token_budget, enabled)+ VALUES (?, ?, ?, ?, 1, ?, 100000, 1000000, 100000000, 1000000000, ?, 1)+ `);+ const insFb = db.prepare('INSERT INTO fallback_config (model_db_id, priority, enabled) VALUES (?, ?, 1)');+ const insHist = db.prepare(`+ INSERT INTO requests (platform, model_id, key_id, status, input_tokens, output_tokens, latency_ms, error, ttfb_ms)+ VALUES (?, ?, 1, ?, 0, ?, ?, ?, ?)+ `);++ PROFILES.forEach((p, i) => {+ insModel.run(p.platform, p.modelId, p.name, p.intelligenceRank, p.sizeLabel, p.budget);+ const id = (db.prepare('SELECT id FROM models WHERE platform=? AND model_id=?').get(p.platform, p.modelId) as { id: number }).id;+ insFb.run(id, i + 1);+ const { encrypted, iv, authTag } = encrypt(`key-${p.platform}`);+ db.prepare(`INSERT INTO api_keys (platform, label, encrypted_key, iv, auth_tag, status, enabled) VALUES (?, 'sim', ?, ?, ?, 'healthy', 1)`)+ .run(p.platform, encrypted, iv, authTag);+ for (let s = 0; s < p.successes; s++) insHist.run(p.platform, p.modelId, 'success', p.outTokens, p.latencyMs, null, p.ttfbMs);+ for (let f = 0; f < p.failures; f++) insHist.run(p.platform, p.modelId, 'error', 0, p.latencyMs, 'sim-fail', p.ttfbMs);+ });+}++function distribution(runs: number): Map<string, number> {+ const counts = new Map<string, number>();+ for (let i = 0; i < runs; i++) {+ const r = routeRequest(100);+ counts.set(r.displayName, (counts.get(r.displayName) ?? 0) + 1);+ }+ return counts;+}++function pct(n: number, total: number): string {+ return `${((n / total) * 100).toFixed(1)}%`.padStart(6);+}++function printDistribution(title: string, counts: Map<string, number>, runs: number) {+ console.log(`\n ${title}`);+ const sorted = [...counts.entries()].sort((a, b) => b[1] - a[1]);+ for (const [name, n] of sorted) {+ const bar = '█'.repeat(Math.round((n / runs) * 30));+ console.log(` ${name.padEnd(22)} ${pct(n, runs)} ${bar}`);+ }+}++function printScores() {+ const { scores } = getRoutingScores();+ console.log(' model rel spd int guard score');+ for (const s of scores) {+ const guard = s.headroom * s.rateLimit;+ console.log(+ ` ${s.displayName.padEnd(22)} ` ++ `${Math.round(s.reliability * 100).toString().padStart(3)} ` ++ `${Math.round(s.speed * 100).toString().padStart(3)} ` ++ `${Math.round(s.intelligence * 100).toString().padStart(3)} ` ++ `${guard.toFixed(2).padStart(5)} ` ++ `${s.score.toFixed(3)}`,+ );+ }+}++function main() {+ initDb(':memory:');+ seed();+ const RUNS = 2000;++ console.log('\n══════════════════════════════════════════════════════════════');+ console.log(' ROUTING SIMULATION — 4 models, 2000 requests per strategy');+ console.log('══════════════════════════════════════════════════════════════');+ console.log('\n Synthetic profiles:');+ for (const p of PROFILES) {+ const rate = (p.successes / (p.successes + p.failures) * 100).toFixed(0);+ const tps = (p.outTokens * 1000 / p.latencyMs).toFixed(0);+ console.log(` ${p.name.padEnd(22)} ${p.sizeLabel.padEnd(9)} success ${rate}% ${tps} tok/s ${p.ttfbMs}ms ttfb`);+ }++ const strategies: RoutingStrategy[] = ['priority', 'balanced', 'smartest', 'fastest', 'reliable'];+ for (const strat of strategies) {+ setRoutingStrategy(strat);+ refreshStatsCache(getDb(), true);+ if (strat === 'balanced') { console.log('\n ── balanced score breakdown ──'); printScores(); }+ printDistribution(`Strategy: ${strat.toUpperCase()}`, distribution(RUNS), RUNS);+ }++ // ── Adaptation: the favored model under 'balanced' suddenly starts failing ──+ console.log('\n══════════════════════════════════════════════════════════════');+ console.log(' ADAPTATION — inject a failure burst into the top model');+ console.log('══════════════════════════════════════════════════════════════');+ setRoutingStrategy('balanced');+ refreshStatsCache(getDb(), true);+ const before = distribution(RUNS);+ printDistribution('Before (balanced, steady state)', before, RUNS);++ // Find the current favorite and slam it with fresh failures.+ const fav = [...before.entries()].sort((a, b) => b[1] - a[1])[0][0];+ const favProfile = PROFILES.find(p => p.name === fav)!;+ const db = getDb();+ const insHist = db.prepare(`+ INSERT INTO requests (platform, model_id, key_id, status, input_tokens, output_tokens, latency_ms, error, ttfb_ms)+ VALUES (?, ?, 1, 'error', 0, 0, 1000, 'outage', NULL)+ `);+ for (let i = 0; i < 300; i++) insHist.run(favProfile.platform, favProfile.modelId);+ console.log(`\n → Injected 300 fresh failures into "${fav}" (simulated outage)…`);+ refreshStatsCache(getDb(), true);+ printScores();+ printDistribution('After (balanced, post-outage)', distribution(RUNS), RUNS);+ console.log('');+}++main();diff --git a/server/src/services/router.ts b/server/src/services/router.tsindex 8aa7bef8..6349f6dd 100644--- a/server/src/services/router.ts+++ b/server/src/services/router.ts@@ -1,20 +1,15 @@-import { getDb } from '../db/index.js';+import { getDb, getSetting, setSetting } from '../db/index.js';import { getProvider, resolveProvider } from '../providers/index.js';import { decrypt } from '../lib/crypto.js';import { canMakeRequest, canUseTokens, isOnCooldown } from './ratelimit.js';+import {+ BANDIT_PRESETS, DEFAULT_STRATEGY, type RoutingStrategy, type RoutingWeights,+ reliabilityPosterior, expectedReliability, sampleBeta,+ speedScore, intelligenceScore, headroomFactor, rateLimitFactor, combineScore,+} from './scoring.js';+import { parseBudget } from '../lib/budget.js';import type { BaseProvider } from '../providers/base.js';--interface ModelRow {- id: number;- platform: string;- model_id: string;- display_name: string;- rpm_limit: number | null;- rpd_limit: number | null;- tpm_limit: number | null;- tpd_limit: number | null;- supports_vision: number;-}+import type { Database } from 'better-sqlite3';interface KeyRow {id: number;@@ -27,10 +22,22 @@ interface KeyRow {base_url: string | null;}-interface FallbackRow {+// Chain row joined with the model fields the bandit needs to score it.+interface ChainRow {model_db_id: number;priority: number;enabled: number;+ platform: string;+ model_id: string;+ display_name: string;+ intelligence_rank: number;+ size_label: string;+ monthly_token_budget: string;+ rpm_limit: number | null;+ rpd_limit: number | null;+ tpm_limit: number | null;+ tpd_limit: number | null;+ supports_vision: number;}export interface RouteResult {@@ -125,10 +132,205 @@ export function getAllPenalties(): Array<{ modelDbId: number; count: number; penreturn result.sort((a, b) => b.penalty - a.penalty);}+// ── Routing strategy (persisted) ────────────────────────────────────────────+const STRATEGY_KEY = 'routing_strategy';+const VALID_STRATEGIES: RoutingStrategy[] = ['priority', 'balanced', 'smartest', 'fastest', 'reliable'];++export function getRoutingStrategy(): RoutingStrategy {+ const raw = getSetting(STRATEGY_KEY);+ return (raw && VALID_STRATEGIES.includes(raw as RoutingStrategy))+ ? (raw as RoutingStrategy)+ : DEFAULT_STRATEGY;+}++export function setRoutingStrategy(strategy: RoutingStrategy): void {+ if (!VALID_STRATEGIES.includes(strategy)) {+ throw new Error(`Unknown routing strategy: ${strategy}`);+ }+ setSetting(STRATEGY_KEY, strategy);+}++function weightsFor(strategy: RoutingStrategy): RoutingWeights | null {+ return strategy === 'priority' ? null : BANDIT_PRESETS[strategy];+}++// ── Analytics stats cache (decay-weighted) ──────────────────────────────────+// Instead of the fork's flat 7-day window (where a model that degrades today+// keeps a stale week-long average), each request is weighted by an exponential+// decay so recent behavior dominates while older data still stabilizes the+// estimate. We aggregate by (model, integer day age) in SQL — at most ~7 rows+// per model — then apply the per-bucket decay weight in JS.+const WINDOW_MS = 7 * 24 * 60 * 60 * 1000;+const HALF_LIFE_DAYS = 2; // a 2-day-old request counts half as much as a fresh one+const CACHE_TTL_MS = 60 * 1000;++interface ModelStats {+ successes: number; // decay-weighted pseudo-count+ failures: number; // decay-weighted pseudo-count+ tokPerSec: number; // from successful requests only (0 = no data)+ avgTtfbMs: number | null; // null = no first-byte timing yet+ monthlyUsedTokens: number; // calendar-month usage, for the headroom guardrail+}++let statsCache: Map<string, ModelStats> | null = null;+let statsCacheTime = 0;++function decayWeight(ageDays: number): number {+ return Math.pow(0.5, Math.max(0, ageDays) / HALF_LIFE_DAYS);+}++export function refreshStatsCache(db: Database, force = false): void {+ if (!force && statsCache && Date.now() - statsCacheTime < CACHE_TTL_MS) return;++ const since = new Date(Date.now() - WINDOW_MS).toISOString();+ const buckets = db.prepare(`+ SELECT platform, model_id,+ CAST((julianday('now') - julianday(created_at)) AS INTEGER) AS age_days,+ COUNT(*) AS total,+ SUM(CASE WHEN status = 'success' THEN 1 ELSE 0 END) AS successes,+ SUM(CASE WHEN status = 'success' THEN output_tokens ELSE 0 END) AS succ_out,+ SUM(CASE WHEN status = 'success' THEN latency_ms ELSE 0 END) AS succ_lat,+ SUM(CASE WHEN status = 'success' AND ttfb_ms IS NOT NULL THEN ttfb_ms ELSE 0 END) AS succ_ttfb_sum,+ SUM(CASE WHEN status = 'success' AND ttfb_ms IS NOT NULL THEN 1 ELSE 0 END) AS succ_ttfb_cnt+ FROM requests+ WHERE created_at >= ?+ GROUP BY platform, model_id, age_days+ `).all(since) as Array<{+ platform: string; model_id: string; age_days: number; total: number; successes: number;+ succ_out: number; succ_lat: number; succ_ttfb_sum: number; succ_ttfb_cnt: number;+ }>;++ // Accumulate decay-weighted sums per model.+ const acc = new Map<string, {+ wSucc: number; wFail: number; wOut: number; wLat: number; wTtfbSum: number; wTtfbCnt: number;+ }>();+ for (const b of buckets) {+ const key = `${b.platform}:${b.model_id}`;+ const w = decayWeight(b.age_days);+ const a = acc.get(key) ?? { wSucc: 0, wFail: 0, wOut: 0, wLat: 0, wTtfbSum: 0, wTtfbCnt: 0 };+ a.wSucc += w * b.successes;+ a.wFail += w * (b.total - b.successes);+ a.wOut += w * b.succ_out;+ a.wLat += w * b.succ_lat;+ a.wTtfbSum += w * b.succ_ttfb_sum;+ a.wTtfbCnt += w * b.succ_ttfb_cnt;+ acc.set(key, a);+ }++ // Calendar-month token usage per model, for the headroom guardrail.+ const usageRows = db.prepare(`+ SELECT platform, model_id, COALESCE(SUM(input_tokens + output_tokens), 0) AS used+ FROM requests+ WHERE created_at >= datetime('now', 'start of month')+ GROUP BY platform, model_id+ `).all() as Array<{ platform: string; model_id: string; used: number }>;+ const usageMap = new Map(usageRows.map(r => [`${r.platform}:${r.model_id}`, r.used]));++ const next = new Map<string, ModelStats>();+ for (const [key, a] of acc) {+ next.set(key, {+ successes: a.wSucc,+ failures: a.wFail,+ tokPerSec: a.wLat > 0 ? (a.wOut * 1000) / a.wLat : 0,+ avgTtfbMs: a.wTtfbCnt > 0 ? a.wTtfbSum / a.wTtfbCnt : null,+ monthlyUsedTokens: usageMap.get(key) ?? 0,+ });+ }+ // Models with month usage but no recent window data still need a headroom number.+ for (const [key, used] of usageMap) {+ if (!next.has(key)) {+ next.set(key, { successes: 0, failures: 0, tokPerSec: 0, avgTtfbMs: null, monthlyUsedTokens: used });+ }+ }++ statsCache = next;+ statsCacheTime = Date.now();+}++// Composite intelligence: size_label is the cross-provider capability tier+// (issue #135 — intelligence_rank is only meaningful within one provider), so+// tier dominates and intelligence_rank breaks ties inside a tier.+const TIER_VALUE: Record<string, number> = { Frontier: 4, Large: 3, Medium: 2, Small: 1 };+function intelligenceComposite(sizeLabel: string, intelligenceRank: number): number {+ const tier = TIER_VALUE[sizeLabel] ?? 0;+ // tier*1000 keeps tiers strictly separated; -rank prefers lower rank in-tier.+ return tier * 1000 - intelligenceRank;+}++// Per-model axis values + the final score. `sampled` chooses Thompson sampling+// (for routing) vs. the expected value (for a stable dashboard display).+interface ScoredEntry {+ axes: { reliability: number; speed: number; intelligence: number };+ headroom: number;+ rateLimit: number;+ score: number;+}++function scoreChainEntry(+ entry: ChainRow,+ weights: RoutingWeights,+ intelMin: number,+ intelMax: number,+ sampled: boolean,+): ScoredEntry {+ const stats = statsCache?.get(`${entry.platform}:${entry.model_id}`);+ const successes = stats?.successes ?? 0;+ const failures = stats?.failures ?? 0;++ let reliability: number;+ if (sampled) {+ const { alpha, beta } = reliabilityPosterior(successes, failures);+ reliability = sampleBeta(alpha, beta);+ } else {+ reliability = expectedReliability(successes, failures);+ }++ const speed = speedScore(stats?.tokPerSec ?? 0, stats?.avgTtfbMs ?? null);+ const intelligence = intelligenceScore(+ intelligenceComposite(entry.size_label, entry.intelligence_rank), intelMin, intelMax,+ );++ const budget = parseBudget(entry.monthly_token_budget);+ const headroom = headroomFactor(stats?.monthlyUsedTokens ?? 0, budget);+ const rl = rateLimitFactor(getPenalty(entry.model_db_id));++ const score = combineScore({ reliability, speed, intelligence, headroom, rateLimit: rl }, weights);+ return { axes: { reliability, speed, intelligence }, headroom, rateLimit: rl, score };+}++/**+ * Order the enabled fallback chain for routing.+ * - 'priority' strategy → legacy manual order + 429 penalty (unchanged).+ * - bandit strategy → Thompson-sampled convex score, manual priority as+ * the deterministic tiebreaker for (near-)equal scores.+ */+function orderChain(chain: ChainRow[], strategy: RoutingStrategy): ChainRow[] {+ const weights = weightsFor(strategy);+ if (!weights) {+ // Legacy priority mode: base priority + 429 penalty, ascending.+ return chain+ .map(e => ({ e, eff: e.priority + getPenalty(e.model_db_id) }))+ .sort((a, b) => a.eff - b.eff || a.e.priority - b.e.priority)+ .map(x => x.e);+ }++ const composites = chain.map(e => intelligenceComposite(e.size_label, e.intelligence_rank));+ const intelMin = composites.length ? Math.min(...composites) : 0;+ const intelMax = composites.length ? Math.max(...composites) : 0;++ return chain+ .map(e => ({ e, s: scoreChainEntry(e, weights, intelMin, intelMax, true).score }))+ // Higher score first; manual priority breaks ties so the chain still matters.+ .sort((a, b) => b.s - a.s || a.e.priority - b.e.priority)+ .map(x => x.e);+}+/*** Route a request to the best available model.- * Models are sorted by (base_priority + rate_limit_penalty) so frequently- * rate-limited models automatically sink below working ones.+ *+ * Ordering depends on the configured strategy (see orderChain). Everything+ * downstream — key round-robin, cooldowns, token pre-checks, custom base_url+ * resolution, vision filtering, sticky sessions — is strategy-independent.** If preferredModelDbId is set, that model gets tried FIRST (sticky sessions).* This prevents hallucination from model switching mid-conversation.@@ -141,18 +343,21 @@ export function getAllPenalties(): Array<{ modelDbId: number; count: number; penexport function routeRequest(estimatedTokens = 1000, skipKeys?: Set<string>, preferredModelDbId?: number, requireVision = false): RouteResult {const db = getDb();- // Get fallback chain ordered by priority- const fallbackChain = db.prepare(`- SELECT fc.model_db_id, fc.priority, fc.enabled+ const strategy = getRoutingStrategy();+ if (strategy !== 'priority') refreshStatsCache(db);++ // Get the enabled fallback chain joined with the fields the scorer needs.+ const chain = db.prepare(`+ SELECT fc.model_db_id, fc.priority, fc.enabled,+ m.platform, m.model_id, m.display_name, m.intelligence_rank,+ m.size_label, m.monthly_token_budget,+ m.rpm_limit, m.rpd_limit, m.tpm_limit, m.tpd_limit, m.supports_visionFROM fallback_config fc- ORDER BY fc.priority ASC- `).all() as FallbackRow[];+ JOIN models m ON m.id = fc.model_db_id AND m.enabled = 1+ WHERE fc.enabled = 1+ `).all() as ChainRow[];- // Apply dynamic penalties: sort by (base priority + penalty)- const sortedChain = fallbackChain.map(entry => ({- ...entry,- effectivePriority: entry.priority + getPenalty(entry.model_db_id),- })).sort((a, b) => a.effectivePriority - b.effectivePriority);+ const sortedChain = orderChain(chain, strategy);// Sticky session: move preferred model to front of chainif (preferredModelDbId) {@@ -164,51 +369,45 @@ export function routeRequest(estimatedTokens = 1000, skipKeys?: Set<string>, pre}for (const entry of sortedChain) {- if (!entry.enabled) continue;-- // Get model details- const model = db.prepare('SELECT * FROM models WHERE id = ? AND enabled = 1').get(entry.model_db_id) as ModelRow | undefined;- if (!model) continue;-// Vision requests skip text-only models — including a sticky/preferred one,// which is correct: don't pin an image turn to a model that can't see it.- if (requireVision && !model.supports_vision) continue;+ if (requireVision && !entry.supports_vision) continue;// Check if we have a provider for this platform- const provider = getProvider(model.platform as any);+ const provider = getProvider(entry.platform as any);if (!provider) continue;// Get enabled keys that have not already failed validation or decryption.const keys = db.prepare("SELECT * FROM api_keys WHERE platform = ? AND enabled = 1 AND status IN ('healthy', 'unknown')"- ).all(model.platform) as KeyRow[];+ ).all(entry.platform) as KeyRow[];if (keys.length === 0) continue;// Get limits once for this modelconst limits = {- rpm: model.rpm_limit,- rpd: model.rpd_limit,- tpm: model.tpm_limit,- tpd: model.tpd_limit,+ rpm: entry.rpm_limit,+ rpd: entry.rpd_limit,+ tpm: entry.tpm_limit,+ tpd: entry.tpd_limit,};// Try all keys for this model before giving up on it- const rrKey = `${model.platform}:${model.model_id}`;+ const rrKey = `${entry.platform}:${entry.model_id}`;let idx = roundRobinIndex.get(rrKey) ?? 0;for (let attempt = 0; attempt < keys.length; attempt++) {const key = keys[idx % keys.length];idx++;- const skipId = `${model.platform}:${model.model_id}:${key.id}`;+ const skipId = `${entry.platform}:${entry.model_id}:${key.id}`;if (skipKeys?.has(skipId)) continue;// Check cooldown (from previous 429s)- if (isOnCooldown(model.platform, model.model_id, key.id)) continue;+ if (isOnCooldown(entry.platform, entry.model_id, key.id)) continue;- if (!canMakeRequest(model.platform, model.model_id, key.id, limits)) continue;- if (!canUseTokens(model.platform, model.model_id, key.id, estimatedTokens, limits)) continue;+ if (!canMakeRequest(entry.platform, entry.model_id, key.id, limits)) continue;+ if (!canUseTokens(entry.platform, entry.model_id, key.id, estimatedTokens, limits)) continue;let decryptedKey: string;try {@@ -222,7 +421,7 @@ export function routeRequest(estimatedTokens = 1000, skipKeys?: Set<string>, pre// For the 'custom' platform the real provider is built from this key's// base_url (the registered instance is just a placeholder). A custom key// with no base_url can't be routed — skip it.- const resolvedProvider = model.platform === 'custom'+ const resolvedProvider = entry.platform === 'custom'? resolveProvider('custom', key.base_url): provider;if (!resolvedProvider) continue;@@ -231,12 +430,12 @@ export function routeRequest(estimatedTokens = 1000, skipKeys?: Set<string>, preroundRobinIndex.set(rrKey, idx);return {provider: resolvedProvider,- modelId: model.model_id,- modelDbId: model.id,+ modelId: entry.model_id,+ modelDbId: entry.model_db_id,apiKey: decryptedKey,keyId: key.id,- platform: model.platform,- displayName: model.display_name,+ platform: entry.platform,+ displayName: entry.display_name,rpdLimit: limits.rpd,tpdLimit: limits.tpd,};@@ -245,10 +444,10 @@ export function routeRequest(estimatedTokens = 1000, skipKeys?: Set<string>, pre// If we reach here, this specific model has NO available keys.// Update round-robin index even if we failed so we don't get stuck.roundRobinIndex.set(rrKey, idx);-- // We don't explicitly penalize the model here because the fact that we- // couldn't find a key means we will naturally move to the next model- // in the `sortedChain` for THIS specific request.++ // We don't explicitly penalize the model here because the fact that we+ // couldn't find a key means we will naturally move to the next model+ // in the sortedChain for THIS specific request.}const err = new Error('All models exhausted. Add more API keys or wait for rate limits to reset.') as any;@@ -256,6 +455,70 @@ export function routeRequest(estimatedTokens = 1000, skipKeys?: Set<string>, prethrow err;}+/**+ * Per-model routing scores for the dashboard. Deterministic (expected+ * reliability, not sampled) so the table is stable between polls. Returns the+ * axis breakdown plus the final score under the active strategy's weights.+ */+export interface RoutingScore {+ modelDbId: number;+ platform: string;+ modelId: string;+ displayName: string;+ enabled: boolean;+ reliability: number;+ speed: number;+ intelligence: number;+ headroom: number;+ rateLimit: number;+ score: number;+ totalRequests: number; // decay-weighted observations+}++export function getRoutingScores(): { strategy: RoutingStrategy; weights: RoutingWeights | null; scores: RoutingScore[] } {+ const db = getDb();+ const strategy = getRoutingStrategy();+ refreshStatsCache(db);++ const chain = db.prepare(`+ SELECT fc.model_db_id, fc.priority, fc.enabled,+ m.platform, m.model_id, m.display_name, m.intelligence_rank,+ m.size_label, m.monthly_token_budget,+ m.rpm_limit, m.rpd_limit, m.tpm_limit, m.tpd_limit, m.supports_vision+ FROM fallback_config fc+ JOIN models m ON m.id = fc.model_db_id+ WHERE m.enabled = 1+ `).all() as ChainRow[];++ // For display we score under 'balanced' weights when in priority mode, so the+ // table still shows a meaningful ranking even with the bandit turned off.+ const weights = weightsFor(strategy) ?? BANDIT_PRESETS.balanced;+ const composites = chain.map(e => intelligenceComposite(e.size_label, e.intelligence_rank));+ const intelMin = composites.length ? Math.min(...composites) : 0;+ const intelMax = composites.length ? Math.max(...composites) : 0;++ const scores: RoutingScore[] = chain.map(entry => {+ const scored = scoreChainEntry(entry, weights, intelMin, intelMax, false);+ const stats = statsCache?.get(`${entry.platform}:${entry.model_id}`);+ return {+ modelDbId: entry.model_db_id,+ platform: entry.platform,+ modelId: entry.model_id,+ displayName: entry.display_name,+ enabled: entry.enabled === 1,+ reliability: scored.axes.reliability,+ speed: scored.axes.speed,+ intelligence: scored.axes.intelligence,+ headroom: scored.headroom,+ rateLimit: scored.rateLimit,+ score: scored.score,+ totalRequests: Math.round((stats?.successes ?? 0) + (stats?.failures ?? 0)),+ };+ }).sort((a, b) => b.score - a.score);++ return { strategy, weights: weightsFor(strategy), scores };+}+// Whether at least one vision-capable model is enabled in the fallback chain.// Used to give image requests a clear "enable a vision model" error instead of// the generic exhaustion message when none is configured (#118, #125).diff --git a/server/src/services/scoring.ts b/server/src/services/scoring.tsnew file mode 100644index 00000000..98951f6b--- /dev/null+++ b/server/src/services/scoring.ts@@ -0,0 +1,191 @@+// ── Bandit routing score ────────────────────────────────────────────────────+//+// A redesign of the analytics-driven router. Instead of summing a pile of+// hand-tuned, dimensionally-incompatible bonuses (a probability + a raw latency+// term + an intelligence term, each hand-capped to keep orderings sane), every+// signal here is normalized to [0, 1] and combined as a CONVEX COMBINATION:+//+// base = w_rel·reliability + w_speed·speed + w_intel·intelligence+// (weights are a preset that sums to 1, so base ∈ [0, 1])+//+// Two always-on GUARDRAILS then multiply the base — they never reorder good+// models against each other, they only pull a model down as it gets dangerous:+//+// effective = base × headroomFactor × rateLimitFactor+//+// headroomFactor → protects a model that is nearly out of its free quota+// rateLimitFactor → demotes a model that is currently throwing 429s+//+// Reliability is drawn from a Beta posterior (Thompson sampling) so exploration+// is automatic and proportional to uncertainty — a model is never permanently+// frozen out after a couple of failures. Speed and intelligence are+// deterministic. The result stays in a bounded, interpretable range and no term+// needs a manual cap to "still beat a 0%-success model".++export interface RoutingWeights {+ reliability: number;+ speed: number;+ intelligence: number;+}++// Strategy is either the legacy manual chain ('priority') or one of the bandit+// presets. Each preset is just a weight vector — the engine is identical.+export type RoutingStrategy = 'priority' | 'balanced' | 'smartest' | 'fastest' | 'reliable';++export const BANDIT_PRESETS: Record<Exclude<RoutingStrategy, 'priority'>, RoutingWeights> = {+ // Reliability leads; speed and intelligence split the rest evenly.+ balanced: { reliability: 0.5, speed: 0.25, intelligence: 0.25 },+ // Intelligence leads, but reliability still carries real weight so a smart+ // model that keeps failing doesn't win.+ smartest: { reliability: 0.35, speed: 0.1, intelligence: 0.55 },+ // Speed leads; reliability keeps a fast-but-broken model from winning.+ fastest: { reliability: 0.35, speed: 0.55, intelligence: 0.1 },+ // Reliability dominates — for clients that just want it to work.+ reliable: { reliability: 0.7, speed: 0.15, intelligence: 0.15 },+};++// Analytics-driven routing is on by default ('balanced'). Operators who want the+// old hand-ordered chain can switch the strategy to 'priority' from the+// dashboard or PUT /api/fallback/routing.+export const DEFAULT_STRATEGY: RoutingStrategy = 'balanced';++// ── Reliability ───────────────────────────────────────────────────────────+// Beta(1,1) prior = uniform: an unseen model is genuinely uncertain, not assumed+// good or bad. With decay-weighted pseudo-counts the alpha/beta are continuous.+export const PRIOR_SUCCESS = 1;+export const PRIOR_FAILURE = 1;++export function reliabilityPosterior(successes: number, failures: number): { alpha: number; beta: number } {+ return {+ alpha: Math.max(0, successes) + PRIOR_SUCCESS,+ beta: Math.max(0, failures) + PRIOR_FAILURE,+ };+}++// Deterministic expected reliability — used for the dashboard display score.+export function expectedReliability(successes: number, failures: number): number {+ const { alpha, beta } = reliabilityPosterior(successes, failures);+ return alpha / (alpha + beta);+}++// ── Speed (throughput + TTFB blended into one [0,1] axis) ───────────────────+// Throughput uses a saturating curve so one very fast tiny model can't make a+// perfectly-fine larger model look "slow" (the global-max-normalization bug in+// the fork). TTFB is a simple linear ramp from "instant" to "painfully slow".+export const SPEED_SCALE_TOK_S = 60; // tok/s at which throughput ≈ 0.63+export const TTFB_BEST_MS = 300; // ≤ this → full latency credit+export const TTFB_WORST_MS = 5000; // ≥ this → zero latency credit+const THROUGHPUT_WEIGHT = 0.6; // within the speed axis+const TTFB_WEIGHT = 0.4;+// Optimistic prior so unmeasured models still get explored on the speed axis.+export const SPEED_PRIOR = 0.6;++function throughputScore(tokPerSec: number): number {+ if (tokPerSec <= 0) return 0;+ return 1 - Math.exp(-tokPerSec / SPEED_SCALE_TOK_S);+}++function ttfbScore(ttfbMs: number): number {+ if (ttfbMs <= TTFB_BEST_MS) return 1;+ if (ttfbMs >= TTFB_WORST_MS) return 0;+ return 1 - (ttfbMs - TTFB_BEST_MS) / (TTFB_WORST_MS - TTFB_BEST_MS);+}++/**+ * Blend throughput and TTFB into a single [0,1] speed score.+ * `tokPerSec <= 0` means no successful samples → return the exploration prior.+ * `ttfbMs === null` means we have throughput but no first-byte timing → fall+ * back to throughput alone rather than guessing latency.+ */+export function speedScore(tokPerSec: number, ttfbMs: number | null): number {+ if (tokPerSec <= 0 && ttfbMs === null) return SPEED_PRIOR;+ const tp = throughputScore(tokPerSec);+ if (ttfbMs === null) return tp;+ if (tokPerSec <= 0) return ttfbScore(ttfbMs);+ return THROUGHPUT_WEIGHT * tp + TTFB_WEIGHT * ttfbScore(ttfbMs);+}++// ── Intelligence ────────────────────────────────────────────────────────────+// Caller supplies a composite (tier-first, rank-as-tiebreaker — see router) and+// the min/max across the enabled chain. We min-max normalize to [0,1], 1 = best.+export function intelligenceScore(composite: number, min: number, max: number): number {+ if (max <= min) return 1; // single model or all equal → neutral-high+ return (composite - min) / (max - min);+}++// ── Guardrail: free-quota headroom ──────────────────────────────────────────+// Multiplier that stays at 1 while a model has comfortable monthly headroom and+// ramps down to a floor as it approaches its free-tier cap, so we stop steering+// traffic at a model we're about to burn out. Unknown budget (0) → no opinion.+export const HEADROOM_FLOOR = 0.1;+export const HEADROOM_RAMP_START = 0.2; // start protecting at 20% remaining++export function headroomFactor(usedTokens: number, budgetTokens: number): number {+ if (!budgetTokens || budgetTokens <= 0) return 1; // unknown budget → no opinion+ const remaining = Math.max(0, 1 - usedTokens / budgetTokens);+ if (remaining >= HEADROOM_RAMP_START) return 1;+ // Linear from (0 remaining → floor) to (RAMP_START remaining → 1).+ return HEADROOM_FLOOR + (1 - HEADROOM_FLOOR) * (remaining / HEADROOM_RAMP_START);+}++// ── Guardrail: live rate-limit penalty ──────────────────────────────────────+// Maps the existing 0..MAX_PENALTY 429 penalty to a multiplier. At max penalty a+// model keeps 40% of its score — demoted hard but never fully excluded, so it+// can recover once the penalty decays.+export const MAX_PENALTY = 10;+export const RATE_LIMIT_MAX_DAMP = 0.6;++export function rateLimitFactor(penalty: number): number {+ const p = Math.min(Math.max(0, penalty), MAX_PENALTY);+ return 1 - (p / MAX_PENALTY) * RATE_LIMIT_MAX_DAMP;+}++// ── Beta sampler (Marsaglia & Tsang via two Gamma draws) ────────────────────+function randomNormal(): number {+ const u1 = Math.random() || Number.EPSILON;+ return Math.sqrt(-2 * Math.log(u1)) * Math.cos(2 * Math.PI * Math.random());+}++function sampleGamma(shape: number): number {+ if (shape < 1) return sampleGamma(shape + 1) * Math.pow(Math.random() || Number.EPSILON, 1 / shape);+ const d = shape - 1 / 3;+ const c = 1 / Math.sqrt(9 * d);+ for (;;) {+ let x: number, v: number;+ do { x = randomNormal(); v = 1 + c * x; } while (v <= 0);+ v = v ** 3;+ const u = Math.random();+ if (u < 1 - 0.0331 * x ** 4) return d * v;+ if (Math.log(u) < 0.5 * x * x + d * (1 - v + Math.log(v))) return d * v;+ }+}++export function sampleBeta(alpha: number, beta: number): number {+ const x = sampleGamma(alpha);+ const y = sampleGamma(beta);+ const sum = x + y;+ return sum > 0 ? x / sum : 0.5;+}++// ── The combined score ──────────────────────────────────────────────────────+export interface ScoreInputs {+ reliability: number; // [0,1] — sampled (routing) or expected (display)+ speed: number; // [0,1]+ intelligence: number; // [0,1]+ headroom: number; // [floor,1] multiplier+ rateLimit: number; // [floor,1] multiplier+}++/**+ * Convex base (∈[0,1]) × the two guardrail multipliers. The weights are assumed+ * to sum to 1; if a caller passes a non-normalized vector we renormalize so the+ * base never escapes [0,1].+ */+export function combineScore(inputs: ScoreInputs, weights: RoutingWeights): number {+ const wSum = weights.reliability + weights.speed + weights.intelligence || 1;+ const base =+ (weights.reliability * inputs.reliability ++ weights.speed * inputs.speed ++ weights.intelligence * inputs.intelligence) / wSum;+ return base * inputs.headroom * inputs.rateLimit;+}