Skip to content

7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.

TypeScript

feat(routing): fold community reliability priors into the Beta posterior (#744)

suantea1e675cca5ce6
diff --git a/server/src/__tests__/services/router-bandit.test.ts b/server/src/__tests__/services/router-bandit.test.ts
index 566c0c59..b7cdeb1e 100644
--- a/server/src/__tests__/services/router-bandit.test.ts
+++ b/server/src/__tests__/services/router-bandit.test.ts
@@ -1,7 +1,8 @@
import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest';
import {
routeRequest, refreshStatsCache, getRoutingStrategy, setRoutingStrategy, getRoutingScores,
- getCustomWeights, setCustomWeights, getExploreEnabled, setExploreEnabled, EXPLORE_MIN_SAMPLES,
+ getCustomWeights, setCustomWeights, getExploreEnabled, setExploreEnabled,
+ getCommunityPrior, setCommunityPriors, getCommunityPriorEnabled, setCommunityPriorEnabled,
} from '../../services/router.js';
import * as ratelimit from '../../services/ratelimit.js';
import { getDb, initDb } from '../../db/index.js';
@@ -223,4 +224,83 @@ describe('bandit router', () => {
const withExplore = pickCounts(500);
expect(withExplore['new'] ?? 0).toBeGreaterThan(0);
});
+
+ it('community priors persist, drop invalid entries, and cap effective sample size (#685)', () => {
+ expect(getCommunityPrior('groq', 'llama', undefined)).toBeUndefined();
+
+ // Invalid entries (negative, all-zero, missing ':') are dropped.
+ const kept = setCommunityPriors({
+ 'groq:llama': { successes: 980, failures: 20 },
+ 'groq:small': { successes: 30, failures: 10 },
+ 'bad:key': { successes: -5, failures: 1 },
+ 'allzero': { successes: 0, failures: 0 },
+ 'no-sep': { successes: 1, failures: 1 },
+ });
+ expect(kept).toBe(2);
+
+ // Oversized priors are rescaled to at most COMMUNITY_PRIOR_MAX_SAMPLES
+ // pseudo-observations, preserving the success/failure ratio; the capped
+ // form is what persists (fresh read from settings).
+ expect(getCommunityPrior('groq', 'llama', undefined))
+ .toEqual({ successes: 49, failures: 1 });
+ // Priors already under the cap are stored untouched.
+ expect(getCommunityPrior('groq', 'small', undefined))
+ .toEqual({ successes: 30, failures: 10 });
+ });
+
+ it('community priors are ignored until the opt-in flag is on (#685)', () => {
+ // A model with no local samples reads as 0.5 (uniform prior). A stored
+ // community record must NOT move that while the flag is off (default).
+ addModel({ platform: 'google', modelId: 'g1', name: 'G1', intelligenceRank: 1, sizeLabel: 'Frontier', budget: '~50M', priority: 1 });
+ setRoutingStrategy('balanced');
+ refreshStatsCache(getDb(), true);
+
+ const plain = getRoutingScores();
+ expect(plain.scores[0]!.reliability).toBeCloseTo(0.5, 2);
+
+ setCommunityPriors({ 'google:g1': { successes: 980, failures: 20 } });
+ expect(getCommunityPriorEnabled()).toBe(false);
+ const gated = getRoutingScores();
+ expect(gated.scores[0]!.reliability).toBeCloseTo(0.5, 2);
+
+ // Flag on → the (capped) prior seeds the axis near the community rate.
+ setCommunityPriorEnabled(true);
+ expect(getCommunityPriorEnabled()).toBe(true);
+ const seeded = getRoutingScores();
+ expect(seeded.scores[0]!.reliability).toBeGreaterThan(0.9);
+ });
+
+ it('local failures override a capped community prior (#685)', () => {
+ // 5 local successes vs 95 failures: even with a glowing (capped) community
+ // record the displayed reliability must fall well below the prior's rate —
+ // a huge upstream count can no longer pin a locally-broken model at ~0.98.
+ addModel({ platform: 'google', modelId: 'g1', name: 'G1', intelligenceRank: 1, sizeLabel: 'Frontier', budget: '~50M', priority: 1 });
+ addHistory('google', 'g1', { successes: 5, failures: 95 });
+ setRoutingStrategy('balanced');
+ setCommunityPriors({ 'google:g1': { successes: 980, failures: 20 } });
+ setCommunityPriorEnabled(true);
+ refreshStatsCache(getDb(), true);
+
+ const { scores } = getRoutingScores();
+ expect(scores[0]!.reliability).toBeLessThan(0.5);
+ });
+
+ it('tiny community priors vanish under real local traffic (#685)', () => {
+ // A small pessimistic prior barely moves a model with hundreds of local
+ // successes: local evidence dominates.
+ addModel({ platform: 'google', modelId: 'g1', name: 'G1', intelligenceRank: 1, sizeLabel: 'Frontier', budget: '~50M', priority: 1 });
+ addHistory('google', 'g1', { successes: 480, failures: 20 });
+ setRoutingStrategy('balanced');
+ setCommunityPriorEnabled(true);
+ refreshStatsCache(getDb(), true);
+
+ const localOnly = getRoutingScores().scores[0]!.reliability;
+
+ setCommunityPriors({ 'google:g1': { successes: 1, failures: 9 } });
+ refreshStatsCache(getDb(), true);
+ const withPrior = getRoutingScores().scores[0]!.reliability;
+
+ expect(withPrior).toBeGreaterThan(0.9);
+ expect(Math.abs(localOnly - withPrior)).toBeLessThan(0.03);
+ });
});
diff --git a/server/src/__tests__/services/scoring.test.ts b/server/src/__tests__/services/scoring.test.ts
index b62d7140..d0afffe8 100644
--- a/server/src/__tests__/services/scoring.test.ts
+++ b/server/src/__tests__/services/scoring.test.ts
@@ -18,6 +18,25 @@ describe('scoring: reliability posterior', () => {
it('posterior adds the priors to the observed counts', () => {
expect(reliabilityPosterior(5, 3)).toEqual({ alpha: 6, beta: 4 });
});
+
+ it('community prior folds in as the starting balance (#685)', () => {
+ // Unseen model + a strong community record → starts near the community rate.
+ const withCommunity = expectedReliability(0, 0, { successes: 98, failures: 2 });
+ expect(withCommunity).toBeGreaterThan(0.9);
+ expect(withCommunity).toBeLessThan(1);
+ // The posterior carries both the community counts and the uniform priors.
+ expect(reliabilityPosterior(1, 0, { successes: 98, failures: 2 }))
+ .toEqual({ alpha: 100, beta: 3 });
+ });
+
+ it('local samples dilute the community prior automatically (#685)', () => {
+ // A tiny community prior barely moves the posterior once the install has
+ // hundreds of its own samples: 1001/1012 ≈ 0.989 vs 0.99 local-only.
+ const mostlyLocal = expectedReliability(990, 10, { successes: 10, failures: 0 });
+ expect(mostlyLocal).toBeGreaterThan(0.98);
+ // No community prior → pure uniform prior on an unseen model.
+ expect(expectedReliability(0, 0)).toBeCloseTo(0.5, 5);
+ });
});
describe('scoring: speed axis', () => {
diff --git a/server/src/services/router.ts b/server/src/services/router.ts
index d59991de..7ff1c911 100644
--- a/server/src/services/router.ts
+++ b/server/src/services/router.ts
@@ -293,6 +293,8 @@ export function getAllPenalties(): Array<{ modelDbId: number; count: number; pen
const STRATEGY_KEY = 'routing_strategy';
const CUSTOM_WEIGHTS_KEY = 'routing_custom_weights';
const EXPLORE_KEY = 'routing_explore_enabled';
+const COMMUNITY_PRIOR_KEY = 'routing_community_prior';
+const COMMUNITY_PRIOR_ENABLED_KEY = 'routing_community_prior_enabled';
/** Chance per request that an unmeasured model gets tried first when the
* exploration toggle is on. The bandit's Thompson sampling already explores
@@ -367,6 +369,112 @@ export function setCustomWeights(weights: RoutingWeights): void {
}));
}
+// ── Community reliability prior (persisted) ────────────────────────────────
+// Aggregated, de-poisoned counts from other self-hosted instances, folded into
+// the Beta posterior as a starting balance so a brand-new model isn't blind
+// (#685 follow-up). Keyed "platform:model_id" (endpoint-scoped keys use the
+// same modelStatsKey form). Local samples dilute it automatically.
+//
+// Opt-in: priors only reach the posterior when routing_community_prior_enabled
+// is on (default off). Server-side only for now — there is deliberately no
+// ingestion path yet, so the flag pins the opt-in semantics before one lands.
+type CommunityPriorMap = Record<string, { successes: number; failures: number }>;
+
+/** Ceiling on a single prior's effective sample size. Local counts are
+ * decay-weighted (2-day half-life — a busy install still only carries on the
+ * order of a hundred effective samples), so an unbounded, undecayed community
+ * count would drown local evidence forever and collapse the Thompson-sampling
+ * variance to zero. Capping at ~50 pseudo-observations keeps a prior worth
+ * roughly half the local evidence at most: enough to seed a brand-new model,
+ * cheap for real local traffic to override. */
+export const COMMUNITY_PRIOR_MAX_SAMPLES = 50;
+
+/** Validate a raw prior map and cap each entry's effective sample size.
+ * Shared by the read path and the write path so a value is bounded no matter
+ * how it entered (fresh set, legacy stored blob, hand-edited settings row).
+ * Invalid entries (negative, all-zero, no ':') are dropped; oversized ones
+ * are rescaled preserving the success/failure ratio (980/20 → 49/1). */
+function sanitizeCommunityPriors(priors: unknown): CommunityPriorMap {
+ const clean: CommunityPriorMap = {};
+ if (!priors || typeof priors !== 'object') return clean;
+ for (const [key, v] of Object.entries(priors as Record<string, { successes: number; failures: number }>)) {
+ if (
+ key.includes(':') &&
+ v && typeof v === 'object' &&
+ Number.isFinite(v.successes) && v.successes >= 0 &&
+ Number.isFinite(v.failures) && v.failures >= 0 &&
+ v.successes + v.failures > 0
+ ) {
+ const total = v.successes + v.failures;
+ const scale = total > COMMUNITY_PRIOR_MAX_SAMPLES ? COMMUNITY_PRIOR_MAX_SAMPLES / total : 1;
+ const entry = { successes: Math.round(v.successes * scale), failures: Math.round(v.failures * scale) };
+ if (entry.successes + entry.failures > 0) clean[key] = entry;
+ }
+ }
+ return clean;
+}
+
+// Parsed-prior cache, same 60s shape as the stats cache: routing reads the map
+// once per chain entry (and once per key in orderKeysByScore), so hitting
+// sqlite + JSON.parse on every lookup is pure waste. Invalidated by the two
+// setters and by refreshStatsCache, so tests and future ingestion see writes
+// immediately.
+let communityPriorCache: { map: CommunityPriorMap; enabled: boolean } | null = null;
+let communityPriorCacheTime = 0;
+
+function communityPriorState(): { map: CommunityPriorMap; enabled: boolean } {
+ const now = Date.now();
+ if (communityPriorCache && now - communityPriorCacheTime < CACHE_TTL_MS) return communityPriorCache;
+ let map: CommunityPriorMap = {};
+ const raw = getSetting(COMMUNITY_PRIOR_KEY);
+ if (raw) {
+ try {
+ map = sanitizeCommunityPriors(JSON.parse(raw));
+ } catch { /* corrupt setting → no priors */ }
+ }
+ communityPriorCache = { map, enabled: getSetting(COMMUNITY_PRIOR_ENABLED_KEY) === '1' };
+ communityPriorCacheTime = now;
+ return communityPriorCache;
+}
+
+function invalidateCommunityPriorCache(): void {
+ communityPriorCache = null;
+}
+
+/** Whether stored community priors are folded into the posterior. Default off. */
+export function getCommunityPriorEnabled(): boolean {
+ return communityPriorState().enabled;
+}
+
+export function setCommunityPriorEnabled(enabled: boolean): void {
+ setSetting(COMMUNITY_PRIOR_ENABLED_KEY, enabled ? '1' : '0');
+ invalidateCommunityPriorCache();
+}
+
+/** Community prior for one model, or undefined when none is stored.
+ * Raw read — ignores the enabled flag; routing goes through
+ * activeCommunityPrior, which honors it. */
+export function getCommunityPrior(platform: string, modelId: string, endpointScope?: string):
+ { successes: number; failures: number } | undefined {
+ return communityPriorState().map[modelStatsKey(platform, modelId, endpointScope)];
+}
+
+/** Gated read for routing: undefined unless the opt-in flag is on. */
+function activeCommunityPrior(platform: string, modelId: string, endpointScope?: string):
+ { successes: number; failures: number } | undefined {
+ const state = communityPriorState();
+ return state.enabled ? state.map[modelStatsKey(platform, modelId, endpointScope)] : undefined;
+}
+
+/** Replace the whole community-prior map (e.g. after an aggregation fetch).
+ * Invalid entries are dropped and oversized ones capped, never stored raw. */
+export function setCommunityPriors(priors: CommunityPriorMap): number {
+ const clean = sanitizeCommunityPriors(priors);
+ setSetting(COMMUNITY_PRIOR_KEY, JSON.stringify(clean));
+ invalidateCommunityPriorCache();
+ return Object.keys(clean).length;
+}
+
function weightsFor(strategy: RoutingStrategy): RoutingWeights | null {
if (strategy === 'priority') return null;
if (strategy === 'custom') return getCustomWeights();
@@ -441,6 +549,10 @@ function customEndpointScopes(db: Db): Map<number, string> {
export function refreshStatsCache(db: Db, force = false): void {
if (!force && statsCache && Date.now() - statsCacheTime < CACHE_TTL_MS) return;
+ // Re-read the community priors alongside the stats they season, so a forced
+ // refresh (tests, admin actions) never routes on a stale prior snapshot.
+ invalidateCommunityPriorCache();
+
const since = new Date(Date.now() - WINDOW_MS).toISOString();
// Grouped by (model, key, day age): still a handful of rows per model — key
// count × ≤7 day buckets — so the finer grain keeps the same one-query,
@@ -671,12 +783,13 @@ function scoreChainEntry(
const successes = stats?.successes ?? 0;
const failures = stats?.failures ?? 0;
+ const community = activeCommunityPrior(entry.platform, entry.model_id, entry.endpoint_scope);
let reliability: number;
if (sampled) {
- const { alpha, beta } = reliabilityPosterior(successes, failures);
+ const { alpha, beta } = reliabilityPosterior(successes, failures, community);
reliability = sampleBeta(alpha, beta);
} else {
- reliability = expectedReliability(successes, failures);
+ reliability = expectedReliability(successes, failures, community);
}
const speed = speedScore(stats?.tokPerSec ?? 0, stats?.avgTtfbMs ?? null);
@@ -908,10 +1021,12 @@ function orderKeysByScore(entry: ChainRow, keys: KeyRow[]): KeyRow[] | null {
const prefix = `${modelStatsKey(entry.platform, entry.model_id, entry.endpoint_scope)}:`;
if (!keys.some(k => keyStatsCache!.has(prefix + k.id))) return null;
+ // The prior is per-model, not per-key: look it up once outside the loop.
+ const community = activeCommunityPrior(entry.platform, entry.model_id, entry.endpoint_scope);
return keys
.map(k => {
const stats = keyStatsCache!.get(prefix + k.id);
- const { alpha, beta } = reliabilityPosterior(stats?.successes ?? 0, stats?.failures ?? 0);
+ const { alpha, beta } = reliabilityPosterior(stats?.successes ?? 0, stats?.failures ?? 0, community);
const rel = sampleBeta(alpha, beta);
const spd = speedScore(stats?.tokPerSec ?? 0, stats?.avgTtfbMs ?? null);
return { k, s: KEY_SCORE_WEIGHTS.reliability * rel + KEY_SCORE_WEIGHTS.speed * spd };
diff --git a/server/src/services/scoring.ts b/server/src/services/scoring.ts
index 7124171b..2f6b0c40 100644
--- a/server/src/services/scoring.ts
+++ b/server/src/services/scoring.ts
@@ -56,16 +56,35 @@ export const DEFAULT_STRATEGY: RoutingStrategy = 'balanced';
export const PRIOR_SUCCESS = 1;
export const PRIOR_FAILURE = 1;
-export function reliabilityPosterior(successes: number, failures: number): { alpha: number; beta: number } {
+/** Community-sourced prior counts, folded into the Beta posterior as the
+ * starting balance (#685 follow-up). `successes`/`failures` are the
+ * decay-weighted LOCAL sample counts; the community numbers are the
+ * aggregated, de-poisoned counts from other instances. Local samples dilute
+ * the community prior automatically: the more this install has observed, the
+ * less the shared starting point matters. */
+export interface CommunityReliabilityPrior {
+ successes: number;
+ failures: number;
+}
+
+export function reliabilityPosterior(
+ successes: number,
+ failures: number,
+ community?: CommunityReliabilityPrior,
+): { alpha: number; beta: number } {
return {
- alpha: Math.max(0, successes) + PRIOR_SUCCESS,
- beta: Math.max(0, failures) + PRIOR_FAILURE,
+ alpha: Math.max(0, successes) + (community?.successes ?? 0) + PRIOR_SUCCESS,
+ beta: Math.max(0, failures) + (community?.failures ?? 0) + PRIOR_FAILURE,
};
}
// Deterministic expected reliability — used for the dashboard display score.
-export function expectedReliability(successes: number, failures: number): number {
- const { alpha, beta } = reliabilityPosterior(successes, failures);
+export function expectedReliability(
+ successes: number,
+ failures: number,
+ community?: CommunityReliabilityPrior,
+): number {
+ const { alpha, beta } = reliabilityPosterior(successes, failures, community);
return alpha / (alpha + beta);
}