Skip to content

7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.

TypeScript

fix(scoring): make the intelligence rank override visible on the axis (#727)

suanteafd9ee8e39b4c
diff --git a/server/src/__tests__/services/router-bandit.test.ts b/server/src/__tests__/services/router-bandit.test.ts
index b7cdeb1e..2c0a508f 100644
--- a/server/src/__tests__/services/router-bandit.test.ts
+++ b/server/src/__tests__/services/router-bandit.test.ts
@@ -303,4 +303,37 @@ describe('bandit router', () => {
expect(withPrior).toBeGreaterThan(0.9);
expect(Math.abs(localOnly - withPrior)).toBeLessThan(0.03);
});
+
+ it('a saved intelligence_rank override visibly moves the axis (#673)', () => {
+ // Regression for #673. A realistic chain: a Frontier flagship on top, a
+ // Medium model at the bottom, and the Large model in between — the one the
+ // user re-ranks from 6 to 1 ("this is actually the best model I have").
+ //
+ // Under the old LINEAR rank term that edit shifted the Large model's
+ // normalized axis by ~0.25 points out of 100, i.e. the dashboard rendered
+ // the SAME integer before and after and the edit looked like a no-op. The
+ // sqrt-compressed term moves it ~2 points, which is visible.
+ addModel({ platform: 'google', modelId: 'flagship', name: 'Flagship', intelligenceRank: 1, sizeLabel: 'Frontier', budget: '~50M', priority: 1 });
+ addModel({ platform: 'groq', modelId: 'workhorse', name: 'Workhorse', intelligenceRank: 6, sizeLabel: 'Large', budget: '~50M', priority: 2 });
+ addModel({ platform: 'meta', modelId: 'compact', name: 'Compact', intelligenceRank: 50, sizeLabel: 'Medium', budget: '~50M', priority: 3 });
+ setRoutingStrategy('balanced');
+ refreshStatsCache(getDb(), true);
+
+ const before = getRoutingScores().scores.find(s => s.modelId === 'workhorse')!;
+
+ // PATCH /api/models/:id { intelligenceRank: 1 } bottoms out in exactly this
+ // UPDATE (routes/models.ts maps intelligenceRank → the intelligence_rank
+ // column); this suite drives the router directly and has no HTTP harness,
+ // so write the column and refresh the cache the same way the route does.
+ const row = getDb().prepare('SELECT id FROM models WHERE platform = ? AND model_id = ?').get('groq', 'workhorse') as { id: number };
+ getDb().prepare('UPDATE models SET intelligence_rank = 1 WHERE id = ?').run(row.id);
+ refreshStatsCache(getDb(), true);
+
+ const after = getRoutingScores().scores.find(s => s.modelId === 'workhorse')!;
+
+ // The dashboard renders Math.round(value * 100) (client AxisBar), so assert
+ // on the number the user actually sees: it must move by at least 2 points.
+ const shown = (v: number) => Math.round(v * 100);
+ expect(shown(after.intelligence)).toBeGreaterThanOrEqual(shown(before.intelligence) + 2);
+ });
});
diff --git a/server/src/__tests__/services/scoring.test.ts b/server/src/__tests__/services/scoring.test.ts
index d0afffe8..e02ee5c5 100644
--- a/server/src/__tests__/services/scoring.test.ts
+++ b/server/src/__tests__/services/scoring.test.ts
@@ -1,6 +1,6 @@
import { describe, it, expect } from 'vitest';
import {
- BANDIT_PRESETS, combineScore, speedScore, intelligenceScore,
+ BANDIT_PRESETS, combineScore, speedScore, intelligenceScore, intelligenceComposite,
headroomFactor, rateLimitFactor, sampleBeta, reliabilityPosterior,
expectedReliability, SPEED_PRIOR, HEADROOM_FLOOR,
} from '../../services/scoring.js';
@@ -77,6 +77,45 @@ describe('scoring: intelligence axis', () => {
});
});
+describe('scoring: intelligence_rank is visible on the axis (#673)', () => {
+ // A realistic three-tier chain. The Frontier model pins the top of the
+ // min-max normalization and the Medium model pins the bottom; the Large
+ // model in between is the one the user re-ranks, so the span is unchanged by
+ // the edit and the whole move comes from the rank term.
+ const top = intelligenceComposite('Frontier', 1);
+ const bottom = intelligenceComposite('Medium', 50);
+ const axis = (rank: number) => intelligenceScore(intelligenceComposite('Large', rank), bottom, top);
+ // The dashboard renders Math.round(value * 100) (client AxisBar).
+ const shown = (rank: number) => Math.round(axis(rank) * 100);
+
+ it('a rank 6 → 1 edit moves the displayed axis by at least 2 points', () => {
+ // Under the old linear `tier*1000 - rank` term this move was ~0.25 points,
+ // i.e. the same rendered integer — the edit looked like it did nothing.
+ expect(shown(1) - shown(6)).toBeGreaterThanOrEqual(2);
+ });
+
+ it('even a small rank improvement is worth more than a rounding artifact', () => {
+ // 10 → 3 is a typical "promote this model" edit.
+ expect(shown(3) - shown(10)).toBeGreaterThanOrEqual(2);
+ });
+
+ it('stays monotonic: a better rank never scores lower', () => {
+ for (let rank = 2; rank <= 200; rank++) {
+ expect(intelligenceComposite('Large', rank)).toBeLessThan(intelligenceComposite('Large', rank - 1));
+ }
+ });
+
+ it('keeps tier dominance: the worst rank in a tier still beats the best rank below it', () => {
+ for (const [better, worse] of [['Frontier', 'Large'], ['Large', 'Medium'], ['Medium', 'Small']] as const) {
+ expect(intelligenceComposite(better, 1000)).toBeGreaterThan(intelligenceComposite(worse, 1));
+ }
+ });
+
+ it('an unrecognized size label still scores below every real tier', () => {
+ expect(intelligenceComposite('', 1)).toBeLessThan(intelligenceComposite('Small', 1000));
+ });
+});
+
describe('scoring: guardrails', () => {
it('headroom is 1 with plenty left and ramps to the floor when exhausted', () => {
expect(headroomFactor(0, 1_000_000)).toBe(1);
diff --git a/server/src/services/scoring.ts b/server/src/services/scoring.ts
index 2f6b0c40..d17fe520 100644
--- a/server/src/services/scoring.ts
+++ b/server/src/services/scoring.ts
@@ -161,23 +161,38 @@ export function observedSpeedRank(speed: number): number {
}
// ── Intelligence ────────────────────────────────────────────────────────────
-// `size_label` is the CROSS-PROVIDER capability tier (issue #135 —
-// intelligence_rank is only meaningful within one provider's own catalog), so
-// tier dominates and intelligence_rank breaks ties inside a tier. A label we
-// don't recognize scores below every real tier, which is also why a model
-// seeded with a placeholder label can never win an auto-route (#488).
+// `size_label` is the CROSS-PROVIDER capability tier (issue #135 — a seeded
+// intelligence_rank is only calibrated within one provider's own catalog), so
+// tier still dominates and no rank can promote a model past the tier above it.
+// Inside a tier, though, rank is no longer a near-invisible tiebreak: it now
+// has a real, visible effect on the composite ACROSS providers by design, so
+// that a user's rank edit actually moves the axis and the routing order
+// (#673). A label we don't recognize scores below every real tier, which is
+// also why a model seeded with a placeholder label can never win an
+// auto-route (#488).
export const TIER_VALUE: Record<string, number> = { Frontier: 4, Large: 3, Medium: 2, Small: 1 };
export function tierValue(sizeLabel: string): number {
return TIER_VALUE[sizeLabel] ?? 0;
}
+// Rank is 1..1000 with 1 = best. A LINEAR rank term made the axis effectively
+// blind to user edits (#673): tier*1000 dwarfed the rank, and the chain-level
+// min-max normalization diluted a few-rank-point change to well under a
+// displayed percentage point, so "set the intelligence rank to a better
+// number" looked like it did nothing. Compress the rank with a square root so
+// edits near the top of the range (1 vs 3 vs 10) are visible on the axis and
+// in routing, while the tier keeps strict dominance: the worst rank in a tier
+// (sqrt(1000)*31 ≈ 980 < 1000) still beats the best rank of the tier below.
+const RANK_SCALE = 31;
+
export function intelligenceComposite(sizeLabel: string, intelligenceRank: number): number {
- // tier*1000 keeps tiers strictly separated; -rank prefers lower rank in-tier.
- return tierValue(sizeLabel) * 1000 - intelligenceRank;
+ // tier*1000 keeps tiers strictly separated; -sqrt(rank)*RANK_SCALE prefers
+ // lower rank in-tier but lets rank edits move the axis (see #673).
+ return tierValue(sizeLabel) * 1000 - Math.sqrt(Math.max(1, intelligenceRank)) * RANK_SCALE;
}
-// Caller supplies a composite (tier-first, rank-as-tiebreaker — see above) and
+// Caller supplies a composite (tier-first, sqrt-compressed rank — see above) and
// the min/max across the enabled chain. We min-max normalize to [0,1], 1 = best.
export function intelligenceScore(composite: number, min: number, max: number): number {
if (max <= min) return 1; // single model or all equal → neutral-high