7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.
TypeScript
feat(routing): fold community reliability priors into the Beta posterior (#744)
suantea1e675cca5ce6
diff --git a/server/src/__tests__/services/router-bandit.test.ts b/server/src/__tests__/services/router-bandit.test.tsindex 566c0c59..b7cdeb1e 100644--- a/server/src/__tests__/services/router-bandit.test.ts+++ b/server/src/__tests__/services/router-bandit.test.ts@@ -1,7 +1,8 @@import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest';import {routeRequest, refreshStatsCache, getRoutingStrategy, setRoutingStrategy, getRoutingScores,- getCustomWeights, setCustomWeights, getExploreEnabled, setExploreEnabled, EXPLORE_MIN_SAMPLES,+ getCustomWeights, setCustomWeights, getExploreEnabled, setExploreEnabled,+ getCommunityPrior, setCommunityPriors, getCommunityPriorEnabled, setCommunityPriorEnabled,} from '../../services/router.js';import * as ratelimit from '../../services/ratelimit.js';import { getDb, initDb } from '../../db/index.js';@@ -223,4 +224,83 @@ describe('bandit router', () => {const withExplore = pickCounts(500);expect(withExplore['new'] ?? 0).toBeGreaterThan(0);});++ it('community priors persist, drop invalid entries, and cap effective sample size (#685)', () => {+ expect(getCommunityPrior('groq', 'llama', undefined)).toBeUndefined();++ // Invalid entries (negative, all-zero, missing ':') are dropped.+ const kept = setCommunityPriors({+ 'groq:llama': { successes: 980, failures: 20 },+ 'groq:small': { successes: 30, failures: 10 },+ 'bad:key': { successes: -5, failures: 1 },+ 'allzero': { successes: 0, failures: 0 },+ 'no-sep': { successes: 1, failures: 1 },+ });+ expect(kept).toBe(2);++ // Oversized priors are rescaled to at most COMMUNITY_PRIOR_MAX_SAMPLES+ // pseudo-observations, preserving the success/failure ratio; the capped+ // form is what persists (fresh read from settings).+ expect(getCommunityPrior('groq', 'llama', undefined))+ .toEqual({ successes: 49, failures: 1 });+ // Priors already under the cap are stored untouched.+ expect(getCommunityPrior('groq', 'small', undefined))+ .toEqual({ successes: 30, failures: 10 });+ });++ it('community priors are ignored until the opt-in flag is on (#685)', () => {+ // A model with no local samples reads as 0.5 (uniform prior). A stored+ // community record must NOT move that while the flag is off (default).+ addModel({ platform: 'google', modelId: 'g1', name: 'G1', intelligenceRank: 1, sizeLabel: 'Frontier', budget: '~50M', priority: 1 });+ setRoutingStrategy('balanced');+ refreshStatsCache(getDb(), true);++ const plain = getRoutingScores();+ expect(plain.scores[0]!.reliability).toBeCloseTo(0.5, 2);++ setCommunityPriors({ 'google:g1': { successes: 980, failures: 20 } });+ expect(getCommunityPriorEnabled()).toBe(false);+ const gated = getRoutingScores();+ expect(gated.scores[0]!.reliability).toBeCloseTo(0.5, 2);++ // Flag on → the (capped) prior seeds the axis near the community rate.+ setCommunityPriorEnabled(true);+ expect(getCommunityPriorEnabled()).toBe(true);+ const seeded = getRoutingScores();+ expect(seeded.scores[0]!.reliability).toBeGreaterThan(0.9);+ });++ it('local failures override a capped community prior (#685)', () => {+ // 5 local successes vs 95 failures: even with a glowing (capped) community+ // record the displayed reliability must fall well below the prior's rate —+ // a huge upstream count can no longer pin a locally-broken model at ~0.98.+ addModel({ platform: 'google', modelId: 'g1', name: 'G1', intelligenceRank: 1, sizeLabel: 'Frontier', budget: '~50M', priority: 1 });+ addHistory('google', 'g1', { successes: 5, failures: 95 });+ setRoutingStrategy('balanced');+ setCommunityPriors({ 'google:g1': { successes: 980, failures: 20 } });+ setCommunityPriorEnabled(true);+ refreshStatsCache(getDb(), true);++ const { scores } = getRoutingScores();+ expect(scores[0]!.reliability).toBeLessThan(0.5);+ });++ it('tiny community priors vanish under real local traffic (#685)', () => {+ // A small pessimistic prior barely moves a model with hundreds of local+ // successes: local evidence dominates.+ addModel({ platform: 'google', modelId: 'g1', name: 'G1', intelligenceRank: 1, sizeLabel: 'Frontier', budget: '~50M', priority: 1 });+ addHistory('google', 'g1', { successes: 480, failures: 20 });+ setRoutingStrategy('balanced');+ setCommunityPriorEnabled(true);+ refreshStatsCache(getDb(), true);++ const localOnly = getRoutingScores().scores[0]!.reliability;++ setCommunityPriors({ 'google:g1': { successes: 1, failures: 9 } });+ refreshStatsCache(getDb(), true);+ const withPrior = getRoutingScores().scores[0]!.reliability;++ expect(withPrior).toBeGreaterThan(0.9);+ expect(Math.abs(localOnly - withPrior)).toBeLessThan(0.03);+ });});diff --git a/server/src/__tests__/services/scoring.test.ts b/server/src/__tests__/services/scoring.test.tsindex b62d7140..d0afffe8 100644--- a/server/src/__tests__/services/scoring.test.ts+++ b/server/src/__tests__/services/scoring.test.ts@@ -18,6 +18,25 @@ describe('scoring: reliability posterior', () => {it('posterior adds the priors to the observed counts', () => {expect(reliabilityPosterior(5, 3)).toEqual({ alpha: 6, beta: 4 });});++ it('community prior folds in as the starting balance (#685)', () => {+ // Unseen model + a strong community record → starts near the community rate.+ const withCommunity = expectedReliability(0, 0, { successes: 98, failures: 2 });+ expect(withCommunity).toBeGreaterThan(0.9);+ expect(withCommunity).toBeLessThan(1);+ // The posterior carries both the community counts and the uniform priors.+ expect(reliabilityPosterior(1, 0, { successes: 98, failures: 2 }))+ .toEqual({ alpha: 100, beta: 3 });+ });++ it('local samples dilute the community prior automatically (#685)', () => {+ // A tiny community prior barely moves the posterior once the install has+ // hundreds of its own samples: 1001/1012 ≈ 0.989 vs 0.99 local-only.+ const mostlyLocal = expectedReliability(990, 10, { successes: 10, failures: 0 });+ expect(mostlyLocal).toBeGreaterThan(0.98);+ // No community prior → pure uniform prior on an unseen model.+ expect(expectedReliability(0, 0)).toBeCloseTo(0.5, 5);+ });});describe('scoring: speed axis', () => {diff --git a/server/src/services/router.ts b/server/src/services/router.tsindex d59991de..7ff1c911 100644--- a/server/src/services/router.ts+++ b/server/src/services/router.ts@@ -293,6 +293,8 @@ export function getAllPenalties(): Array<{ modelDbId: number; count: number; penconst STRATEGY_KEY = 'routing_strategy';const CUSTOM_WEIGHTS_KEY = 'routing_custom_weights';const EXPLORE_KEY = 'routing_explore_enabled';+const COMMUNITY_PRIOR_KEY = 'routing_community_prior';+const COMMUNITY_PRIOR_ENABLED_KEY = 'routing_community_prior_enabled';/** Chance per request that an unmeasured model gets tried first when the* exploration toggle is on. The bandit's Thompson sampling already explores@@ -367,6 +369,112 @@ export function setCustomWeights(weights: RoutingWeights): void {}));}+// ── Community reliability prior (persisted) ────────────────────────────────+// Aggregated, de-poisoned counts from other self-hosted instances, folded into+// the Beta posterior as a starting balance so a brand-new model isn't blind+// (#685 follow-up). Keyed "platform:model_id" (endpoint-scoped keys use the+// same modelStatsKey form). Local samples dilute it automatically.+//+// Opt-in: priors only reach the posterior when routing_community_prior_enabled+// is on (default off). Server-side only for now — there is deliberately no+// ingestion path yet, so the flag pins the opt-in semantics before one lands.+type CommunityPriorMap = Record<string, { successes: number; failures: number }>;++/** Ceiling on a single prior's effective sample size. Local counts are+ * decay-weighted (2-day half-life — a busy install still only carries on the+ * order of a hundred effective samples), so an unbounded, undecayed community+ * count would drown local evidence forever and collapse the Thompson-sampling+ * variance to zero. Capping at ~50 pseudo-observations keeps a prior worth+ * roughly half the local evidence at most: enough to seed a brand-new model,+ * cheap for real local traffic to override. */+export const COMMUNITY_PRIOR_MAX_SAMPLES = 50;++/** Validate a raw prior map and cap each entry's effective sample size.+ * Shared by the read path and the write path so a value is bounded no matter+ * how it entered (fresh set, legacy stored blob, hand-edited settings row).+ * Invalid entries (negative, all-zero, no ':') are dropped; oversized ones+ * are rescaled preserving the success/failure ratio (980/20 → 49/1). */+function sanitizeCommunityPriors(priors: unknown): CommunityPriorMap {+ const clean: CommunityPriorMap = {};+ if (!priors || typeof priors !== 'object') return clean;+ for (const [key, v] of Object.entries(priors as Record<string, { successes: number; failures: number }>)) {+ if (+ key.includes(':') &&+ v && typeof v === 'object' &&+ Number.isFinite(v.successes) && v.successes >= 0 &&+ Number.isFinite(v.failures) && v.failures >= 0 &&+ v.successes + v.failures > 0+ ) {+ const total = v.successes + v.failures;+ const scale = total > COMMUNITY_PRIOR_MAX_SAMPLES ? COMMUNITY_PRIOR_MAX_SAMPLES / total : 1;+ const entry = { successes: Math.round(v.successes * scale), failures: Math.round(v.failures * scale) };+ if (entry.successes + entry.failures > 0) clean[key] = entry;+ }+ }+ return clean;+}++// Parsed-prior cache, same 60s shape as the stats cache: routing reads the map+// once per chain entry (and once per key in orderKeysByScore), so hitting+// sqlite + JSON.parse on every lookup is pure waste. Invalidated by the two+// setters and by refreshStatsCache, so tests and future ingestion see writes+// immediately.+let communityPriorCache: { map: CommunityPriorMap; enabled: boolean } | null = null;+let communityPriorCacheTime = 0;++function communityPriorState(): { map: CommunityPriorMap; enabled: boolean } {+ const now = Date.now();+ if (communityPriorCache && now - communityPriorCacheTime < CACHE_TTL_MS) return communityPriorCache;+ let map: CommunityPriorMap = {};+ const raw = getSetting(COMMUNITY_PRIOR_KEY);+ if (raw) {+ try {+ map = sanitizeCommunityPriors(JSON.parse(raw));+ } catch { /* corrupt setting → no priors */ }+ }+ communityPriorCache = { map, enabled: getSetting(COMMUNITY_PRIOR_ENABLED_KEY) === '1' };+ communityPriorCacheTime = now;+ return communityPriorCache;+}++function invalidateCommunityPriorCache(): void {+ communityPriorCache = null;+}++/** Whether stored community priors are folded into the posterior. Default off. */+export function getCommunityPriorEnabled(): boolean {+ return communityPriorState().enabled;+}++export function setCommunityPriorEnabled(enabled: boolean): void {+ setSetting(COMMUNITY_PRIOR_ENABLED_KEY, enabled ? '1' : '0');+ invalidateCommunityPriorCache();+}++/** Community prior for one model, or undefined when none is stored.+ * Raw read — ignores the enabled flag; routing goes through+ * activeCommunityPrior, which honors it. */+export function getCommunityPrior(platform: string, modelId: string, endpointScope?: string):+ { successes: number; failures: number } | undefined {+ return communityPriorState().map[modelStatsKey(platform, modelId, endpointScope)];+}++/** Gated read for routing: undefined unless the opt-in flag is on. */+function activeCommunityPrior(platform: string, modelId: string, endpointScope?: string):+ { successes: number; failures: number } | undefined {+ const state = communityPriorState();+ return state.enabled ? state.map[modelStatsKey(platform, modelId, endpointScope)] : undefined;+}++/** Replace the whole community-prior map (e.g. after an aggregation fetch).+ * Invalid entries are dropped and oversized ones capped, never stored raw. */+export function setCommunityPriors(priors: CommunityPriorMap): number {+ const clean = sanitizeCommunityPriors(priors);+ setSetting(COMMUNITY_PRIOR_KEY, JSON.stringify(clean));+ invalidateCommunityPriorCache();+ return Object.keys(clean).length;+}+function weightsFor(strategy: RoutingStrategy): RoutingWeights | null {if (strategy === 'priority') return null;if (strategy === 'custom') return getCustomWeights();@@ -441,6 +549,10 @@ function customEndpointScopes(db: Db): Map<number, string> {export function refreshStatsCache(db: Db, force = false): void {if (!force && statsCache && Date.now() - statsCacheTime < CACHE_TTL_MS) return;+ // Re-read the community priors alongside the stats they season, so a forced+ // refresh (tests, admin actions) never routes on a stale prior snapshot.+ invalidateCommunityPriorCache();+const since = new Date(Date.now() - WINDOW_MS).toISOString();// Grouped by (model, key, day age): still a handful of rows per model — key// count × ≤7 day buckets — so the finer grain keeps the same one-query,@@ -671,12 +783,13 @@ function scoreChainEntry(const successes = stats?.successes ?? 0;const failures = stats?.failures ?? 0;+ const community = activeCommunityPrior(entry.platform, entry.model_id, entry.endpoint_scope);let reliability: number;if (sampled) {- const { alpha, beta } = reliabilityPosterior(successes, failures);+ const { alpha, beta } = reliabilityPosterior(successes, failures, community);reliability = sampleBeta(alpha, beta);} else {- reliability = expectedReliability(successes, failures);+ reliability = expectedReliability(successes, failures, community);}const speed = speedScore(stats?.tokPerSec ?? 0, stats?.avgTtfbMs ?? null);@@ -908,10 +1021,12 @@ function orderKeysByScore(entry: ChainRow, keys: KeyRow[]): KeyRow[] | null {const prefix = `${modelStatsKey(entry.platform, entry.model_id, entry.endpoint_scope)}:`;if (!keys.some(k => keyStatsCache!.has(prefix + k.id))) return null;+ // The prior is per-model, not per-key: look it up once outside the loop.+ const community = activeCommunityPrior(entry.platform, entry.model_id, entry.endpoint_scope);return keys.map(k => {const stats = keyStatsCache!.get(prefix + k.id);- const { alpha, beta } = reliabilityPosterior(stats?.successes ?? 0, stats?.failures ?? 0);+ const { alpha, beta } = reliabilityPosterior(stats?.successes ?? 0, stats?.failures ?? 0, community);const rel = sampleBeta(alpha, beta);const spd = speedScore(stats?.tokPerSec ?? 0, stats?.avgTtfbMs ?? null);return { k, s: KEY_SCORE_WEIGHTS.reliability * rel + KEY_SCORE_WEIGHTS.speed * spd };diff --git a/server/src/services/scoring.ts b/server/src/services/scoring.tsindex 7124171b..2f6b0c40 100644--- a/server/src/services/scoring.ts+++ b/server/src/services/scoring.ts@@ -56,16 +56,35 @@ export const DEFAULT_STRATEGY: RoutingStrategy = 'balanced';export const PRIOR_SUCCESS = 1;export const PRIOR_FAILURE = 1;-export function reliabilityPosterior(successes: number, failures: number): { alpha: number; beta: number } {+/** Community-sourced prior counts, folded into the Beta posterior as the+ * starting balance (#685 follow-up). `successes`/`failures` are the+ * decay-weighted LOCAL sample counts; the community numbers are the+ * aggregated, de-poisoned counts from other instances. Local samples dilute+ * the community prior automatically: the more this install has observed, the+ * less the shared starting point matters. */+export interface CommunityReliabilityPrior {+ successes: number;+ failures: number;+}++export function reliabilityPosterior(+ successes: number,+ failures: number,+ community?: CommunityReliabilityPrior,+): { alpha: number; beta: number } {return {- alpha: Math.max(0, successes) + PRIOR_SUCCESS,- beta: Math.max(0, failures) + PRIOR_FAILURE,+ alpha: Math.max(0, successes) + (community?.successes ?? 0) + PRIOR_SUCCESS,+ beta: Math.max(0, failures) + (community?.failures ?? 0) + PRIOR_FAILURE,};}// Deterministic expected reliability — used for the dashboard display score.-export function expectedReliability(successes: number, failures: number): number {- const { alpha, beta } = reliabilityPosterior(successes, failures);+export function expectedReliability(+ successes: number,+ failures: number,+ community?: CommunityReliabilityPrior,+): number {+ const { alpha, beta } = reliabilityPosterior(successes, failures, community);return alpha / (alpha + beta);}