7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.
TypeScript
fix(scoring): make the intelligence rank override visible on the axis (#727)
suanteafd9ee8e39b4c
diff --git a/server/src/__tests__/services/router-bandit.test.ts b/server/src/__tests__/services/router-bandit.test.tsindex b7cdeb1e..2c0a508f 100644--- a/server/src/__tests__/services/router-bandit.test.ts+++ b/server/src/__tests__/services/router-bandit.test.ts@@ -303,4 +303,37 @@ describe('bandit router', () => {expect(withPrior).toBeGreaterThan(0.9);expect(Math.abs(localOnly - withPrior)).toBeLessThan(0.03);});++ it('a saved intelligence_rank override visibly moves the axis (#673)', () => {+ // Regression for #673. A realistic chain: a Frontier flagship on top, a+ // Medium model at the bottom, and the Large model in between — the one the+ // user re-ranks from 6 to 1 ("this is actually the best model I have").+ //+ // Under the old LINEAR rank term that edit shifted the Large model's+ // normalized axis by ~0.25 points out of 100, i.e. the dashboard rendered+ // the SAME integer before and after and the edit looked like a no-op. The+ // sqrt-compressed term moves it ~2 points, which is visible.+ addModel({ platform: 'google', modelId: 'flagship', name: 'Flagship', intelligenceRank: 1, sizeLabel: 'Frontier', budget: '~50M', priority: 1 });+ addModel({ platform: 'groq', modelId: 'workhorse', name: 'Workhorse', intelligenceRank: 6, sizeLabel: 'Large', budget: '~50M', priority: 2 });+ addModel({ platform: 'meta', modelId: 'compact', name: 'Compact', intelligenceRank: 50, sizeLabel: 'Medium', budget: '~50M', priority: 3 });+ setRoutingStrategy('balanced');+ refreshStatsCache(getDb(), true);++ const before = getRoutingScores().scores.find(s => s.modelId === 'workhorse')!;++ // PATCH /api/models/:id { intelligenceRank: 1 } bottoms out in exactly this+ // UPDATE (routes/models.ts maps intelligenceRank → the intelligence_rank+ // column); this suite drives the router directly and has no HTTP harness,+ // so write the column and refresh the cache the same way the route does.+ const row = getDb().prepare('SELECT id FROM models WHERE platform = ? AND model_id = ?').get('groq', 'workhorse') as { id: number };+ getDb().prepare('UPDATE models SET intelligence_rank = 1 WHERE id = ?').run(row.id);+ refreshStatsCache(getDb(), true);++ const after = getRoutingScores().scores.find(s => s.modelId === 'workhorse')!;++ // The dashboard renders Math.round(value * 100) (client AxisBar), so assert+ // on the number the user actually sees: it must move by at least 2 points.+ const shown = (v: number) => Math.round(v * 100);+ expect(shown(after.intelligence)).toBeGreaterThanOrEqual(shown(before.intelligence) + 2);+ });});diff --git a/server/src/__tests__/services/scoring.test.ts b/server/src/__tests__/services/scoring.test.tsindex d0afffe8..e02ee5c5 100644--- a/server/src/__tests__/services/scoring.test.ts+++ b/server/src/__tests__/services/scoring.test.ts@@ -1,6 +1,6 @@import { describe, it, expect } from 'vitest';import {- BANDIT_PRESETS, combineScore, speedScore, intelligenceScore,+ BANDIT_PRESETS, combineScore, speedScore, intelligenceScore, intelligenceComposite,headroomFactor, rateLimitFactor, sampleBeta, reliabilityPosterior,expectedReliability, SPEED_PRIOR, HEADROOM_FLOOR,} from '../../services/scoring.js';@@ -77,6 +77,45 @@ describe('scoring: intelligence axis', () => {});});+describe('scoring: intelligence_rank is visible on the axis (#673)', () => {+ // A realistic three-tier chain. The Frontier model pins the top of the+ // min-max normalization and the Medium model pins the bottom; the Large+ // model in between is the one the user re-ranks, so the span is unchanged by+ // the edit and the whole move comes from the rank term.+ const top = intelligenceComposite('Frontier', 1);+ const bottom = intelligenceComposite('Medium', 50);+ const axis = (rank: number) => intelligenceScore(intelligenceComposite('Large', rank), bottom, top);+ // The dashboard renders Math.round(value * 100) (client AxisBar).+ const shown = (rank: number) => Math.round(axis(rank) * 100);++ it('a rank 6 → 1 edit moves the displayed axis by at least 2 points', () => {+ // Under the old linear `tier*1000 - rank` term this move was ~0.25 points,+ // i.e. the same rendered integer — the edit looked like it did nothing.+ expect(shown(1) - shown(6)).toBeGreaterThanOrEqual(2);+ });++ it('even a small rank improvement is worth more than a rounding artifact', () => {+ // 10 → 3 is a typical "promote this model" edit.+ expect(shown(3) - shown(10)).toBeGreaterThanOrEqual(2);+ });++ it('stays monotonic: a better rank never scores lower', () => {+ for (let rank = 2; rank <= 200; rank++) {+ expect(intelligenceComposite('Large', rank)).toBeLessThan(intelligenceComposite('Large', rank - 1));+ }+ });++ it('keeps tier dominance: the worst rank in a tier still beats the best rank below it', () => {+ for (const [better, worse] of [['Frontier', 'Large'], ['Large', 'Medium'], ['Medium', 'Small']] as const) {+ expect(intelligenceComposite(better, 1000)).toBeGreaterThan(intelligenceComposite(worse, 1));+ }+ });++ it('an unrecognized size label still scores below every real tier', () => {+ expect(intelligenceComposite('', 1)).toBeLessThan(intelligenceComposite('Small', 1000));+ });+});+describe('scoring: guardrails', () => {it('headroom is 1 with plenty left and ramps to the floor when exhausted', () => {expect(headroomFactor(0, 1_000_000)).toBe(1);diff --git a/server/src/services/scoring.ts b/server/src/services/scoring.tsindex 2f6b0c40..d17fe520 100644--- a/server/src/services/scoring.ts+++ b/server/src/services/scoring.ts@@ -161,23 +161,38 @@ export function observedSpeedRank(speed: number): number {}// ── Intelligence ────────────────────────────────────────────────────────────-// `size_label` is the CROSS-PROVIDER capability tier (issue #135 —-// intelligence_rank is only meaningful within one provider's own catalog), so-// tier dominates and intelligence_rank breaks ties inside a tier. A label we-// don't recognize scores below every real tier, which is also why a model-// seeded with a placeholder label can never win an auto-route (#488).+// `size_label` is the CROSS-PROVIDER capability tier (issue #135 — a seeded+// intelligence_rank is only calibrated within one provider's own catalog), so+// tier still dominates and no rank can promote a model past the tier above it.+// Inside a tier, though, rank is no longer a near-invisible tiebreak: it now+// has a real, visible effect on the composite ACROSS providers by design, so+// that a user's rank edit actually moves the axis and the routing order+// (#673). A label we don't recognize scores below every real tier, which is+// also why a model seeded with a placeholder label can never win an+// auto-route (#488).export const TIER_VALUE: Record<string, number> = { Frontier: 4, Large: 3, Medium: 2, Small: 1 };export function tierValue(sizeLabel: string): number {return TIER_VALUE[sizeLabel] ?? 0;}+// Rank is 1..1000 with 1 = best. A LINEAR rank term made the axis effectively+// blind to user edits (#673): tier*1000 dwarfed the rank, and the chain-level+// min-max normalization diluted a few-rank-point change to well under a+// displayed percentage point, so "set the intelligence rank to a better+// number" looked like it did nothing. Compress the rank with a square root so+// edits near the top of the range (1 vs 3 vs 10) are visible on the axis and+// in routing, while the tier keeps strict dominance: the worst rank in a tier+// (sqrt(1000)*31 ≈ 980 < 1000) still beats the best rank of the tier below.+const RANK_SCALE = 31;+export function intelligenceComposite(sizeLabel: string, intelligenceRank: number): number {- // tier*1000 keeps tiers strictly separated; -rank prefers lower rank in-tier.- return tierValue(sizeLabel) * 1000 - intelligenceRank;+ // tier*1000 keeps tiers strictly separated; -sqrt(rank)*RANK_SCALE prefers+ // lower rank in-tier but lets rank edits move the axis (see #673).+ return tierValue(sizeLabel) * 1000 - Math.sqrt(Math.max(1, intelligenceRank)) * RANK_SCALE;}-// Caller supplies a composite (tier-first, rank-as-tiebreaker — see above) and+// Caller supplies a composite (tier-first, sqrt-compressed rank — see above) and// the min/max across the enabled chain. We min-max normalize to [0,1], 1 = best.export function intelligenceScore(composite: number, min: number, max: number): number {if (max <= min) return 1; // single model or all equal → neutral-high