7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.
TypeScript
fix(router): let timeouts cost speed, and write an observed speed_rank (#619) (#645)
Tashfeenfc4e47dc5c29
diff --git a/server/src/__tests__/services/router-speed-scoring.test.ts b/server/src/__tests__/services/router-speed-scoring.test.tsnew file mode 100644index 00000000..37a835b7--- /dev/null+++ b/server/src/__tests__/services/router-speed-scoring.test.ts@@ -0,0 +1,244 @@+import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest';+import {+ refreshStatsCache, getRoutingScores, setRoutingStrategy,+ writeObservedSpeedRanks, resetSpeedRankWriteback,+} from '../../services/router.js';+import { observedSpeedRank, TIMEOUT_LATENCY_CAP_MS } from '../../services/scoring.js';+import { upsertModelOverrides } from '../../services/model-state.js';+import { getDb, initDb } from '../../db/index.js';++vi.mock('../../lib/crypto.js', async () => {+ const actual = await vi.importActual('../../lib/crypto.js');+ return { ...actual, decrypt: vi.fn(() => 'mocked-api-key') };+});++const ORIGINAL_DEV_MODE = process.env.DEV_MODE;+const ORIGINAL_NODE_ENV = process.env.NODE_ENV;++function addModel(opts: {+ platform: string; modelId: string; name: string; speedRank?: number; priority?: number;+}): number {+ const db = getDb();+ db.prepare(`+ INSERT INTO models (platform, model_id, display_name, intelligence_rank, speed_rank, size_label, monthly_token_budget, enabled)+ VALUES (?, ?, ?, 3, ?, 'Large', '~50M', 1)+ `).run(opts.platform, opts.modelId, opts.name, opts.speedRank ?? 5);+ const id = (db.prepare('SELECT id FROM models WHERE platform = ? AND model_id = ?')+ .get(opts.platform, opts.modelId) as { id: number }).id;+ db.prepare('INSERT INTO fallback_config (model_db_id, priority, enabled) VALUES (?, ?, 1)')+ .run(id, opts.priority ?? 1);+ db.prepare(`+ INSERT INTO api_keys (platform, label, encrypted_key, iv, auth_tag, status, enabled)+ VALUES (?, 'k', 'enc', 'iv', 'tag', 'healthy', 1)+ `).run(opts.platform);+ return id;+}++// Fresh rows (age 0 → decay weight 1) so the arithmetic in the assertions is exact.+function addRequests(platform: string, modelId: string, opts: {+ count: number; status: string; outTokens?: number; latencyMs?: number;+ ttfbMs?: number | null; error?: string | null;+}) {+ const ins = getDb().prepare(`+ INSERT INTO requests (platform, model_id, key_id, status, input_tokens, output_tokens, latency_ms, error, ttfb_ms)+ VALUES (?, ?, 1, ?, 0, ?, ?, ?, ?)+ `);+ for (let i = 0; i < opts.count; i++) {+ ins.run(platform, modelId, opts.status, opts.outTokens ?? 0,+ opts.latencyMs ?? 0, opts.error ?? null, opts.ttfbMs ?? null);+ }+}++function speedOf(modelId: string): number {+ const { scores } = getRoutingScores();+ const row = scores.find(s => s.modelId === modelId);+ if (!row) throw new Error(`no score row for ${modelId}`);+ return row.speed;+}++function speedRankOf(platform: string, modelId: string): number {+ return (getDb().prepare('SELECT speed_rank FROM models WHERE platform = ? AND model_id = ?')+ .get(platform, modelId) as { speed_rank: number }).speed_rank;+}++describe('speed scoring: timeouts count against speed (#619)', () => {+ beforeEach(() => {+ process.env.DEV_MODE = 'true';+ process.env.NODE_ENV = 'test';+ initDb(':memory:');+ getDb().exec('DELETE FROM fallback_config; DELETE FROM api_keys; DELETE FROM models; DELETE FROM requests; DELETE FROM model_overrides;');+ setRoutingStrategy('balanced');+ resetSpeedRankWriteback();+ });++ afterEach(() => {+ if (ORIGINAL_DEV_MODE === undefined) delete process.env.DEV_MODE; else process.env.DEV_MODE = ORIGINAL_DEV_MODE;+ if (ORIGINAL_NODE_ENV === undefined) delete process.env.NODE_ENV; else process.env.NODE_ENV = ORIGINAL_NODE_ENV;+ });++ it('the same successes score measurably slower once timeouts are in the window', () => {+ addModel({ platform: 'google', modelId: 'clean', name: 'Clean', priority: 1 });+ addModel({ platform: 'groq', modelId: 'timeouty', name: 'Timeouty', priority: 2 });++ // Identical successful traffic on both.+ for (const [platform, modelId] of [['google', 'clean'], ['groq', 'timeouty']] as const) {+ addRequests(platform, modelId, { count: 40, status: 'success', outTokens: 100, latencyMs: 1000, ttfbMs: 200 });+ }+ // …but one of them also times out constantly. A timeout is logged as an+ // 'error' row whose text carries the abort/timeout marker.+ addRequests('groq', 'timeouty', {+ count: 10, status: 'error', outTokens: 0, latencyMs: 120_000, ttfbMs: null,+ error: 'The operation was aborted (groq, chat, 120s)',+ });++ refreshStatsCache(getDb(), true);+ const clean = speedOf('clean');+ const timeouty = speedOf('timeouty');+ expect(timeouty).toBeLessThan(clean - 0.2);+ });++ it('a model that only ever times out lands at the bottom of the speed axis', () => {+ addModel({ platform: 'google', modelId: 'dead', name: 'Dead', priority: 1 });+ addRequests('google', 'dead', {+ count: 25, status: 'error', outTokens: 0, latencyMs: 90_000, ttfbMs: null,+ error: 'google stream stalled: no data for 90000ms (timeout)',+ });++ refreshStatsCache(getDb(), true);+ // Not the optimistic no-data prior (0.6) — it timed out every single time.+ expect(speedOf('dead')).toBeLessThan(0.05);+ });++ it('one pathological hang is capped, not allowed to dominate the window', () => {+ addModel({ platform: 'google', modelId: 'capped', name: 'Capped', priority: 1 });+ addModel({ platform: 'groq', modelId: 'uncapped', name: 'Uncapped', priority: 2 });+ for (const [platform, modelId] of [['google', 'capped'], ['groq', 'uncapped']] as const) {+ addRequests(platform, modelId, { count: 40, status: 'success', outTokens: 100, latencyMs: 1000, ttfbMs: 200 });+ addRequests(platform, modelId, {+ count: 1, status: 'error', outTokens: 0,+ latencyMs: modelId === 'capped' ? TIMEOUT_LATENCY_CAP_MS : TIMEOUT_LATENCY_CAP_MS * 50,+ error: 'The operation was aborted (x, chat, 120s)',+ });+ }+ refreshStatsCache(getDb(), true);+ expect(speedOf('uncapped')).toBeCloseTo(speedOf('capped'), 10);+ });++ it('a non-timeout failure leaves the speed axis alone (it only costs reliability)', () => {+ addModel({ platform: 'google', modelId: 'fast', name: 'Fast', priority: 1 });+ addModel({ platform: 'groq', modelId: 'refusing', name: 'Refusing', priority: 2 });+ for (const [platform, modelId] of [['google', 'fast'], ['groq', 'refusing']] as const) {+ addRequests(platform, modelId, { count: 40, status: 'success', outTokens: 100, latencyMs: 1000, ttfbMs: 200 });+ }+ addRequests('groq', 'refusing', {+ count: 20, status: 'error', outTokens: 0, latencyMs: 200,+ error: 'API error 401: invalid api key',+ });++ refreshStatsCache(getDb(), true);+ expect(speedOf('refusing')).toBeCloseTo(speedOf('fast'), 10);+ const { scores } = getRoutingScores();+ const refusing = scores.find(s => s.modelId === 'refusing')!;+ const fast = scores.find(s => s.modelId === 'fast')!;+ expect(refusing.reliability).toBeLessThan(fast.reliability);+ });+});++describe('observed speed_rank writeback (#619)', () => {+ beforeEach(() => {+ process.env.DEV_MODE = 'true';+ process.env.NODE_ENV = 'test';+ initDb(':memory:');+ getDb().exec('DELETE FROM fallback_config; DELETE FROM api_keys; DELETE FROM models; DELETE FROM requests; DELETE FROM model_overrides;');+ setRoutingStrategy('balanced');+ // Drain the periodic writeback against the now-empty DB so its interval gate+ // is closed: each test below drives writeObservedSpeedRanks itself and wants+ // an exact count. The one test that checks the automatic tail re-opens it.+ resetSpeedRankWriteback();+ refreshStatsCache(getDb(), true);+ });++ afterEach(() => {+ if (ORIGINAL_DEV_MODE === undefined) delete process.env.DEV_MODE; else process.env.DEV_MODE = ORIGINAL_DEV_MODE;+ if (ORIGINAL_NODE_ENV === undefined) delete process.env.NODE_ENV; else process.env.NODE_ENV = ORIGINAL_NODE_ENV;+ });++ it('maps the speed axis onto the catalog rank scale (1 = fastest, low is fast)', () => {+ expect(observedSpeedRank(1)).toBe(1);+ expect(observedSpeedRank(0)).toBe(10);+ expect(observedSpeedRank(0.5)).toBeGreaterThan(observedSpeedRank(0.9));+ // Out-of-range inputs are clamped, never NaN.+ expect(observedSpeedRank(5)).toBe(1);+ expect(observedSpeedRank(-5)).toBe(10);+ });++ it('writes an observed rank for a well-sampled model', () => {+ addModel({ platform: 'google', modelId: 'quick', name: 'Quick', speedRank: 9 });+ addRequests('google', 'quick', { count: 40, status: 'success', outTokens: 4000, latencyMs: 1000, ttfbMs: 100 });++ refreshStatsCache(getDb(), true);+ expect(writeObservedSpeedRanks(getDb())).toBe(1);+ // Genuinely fast traffic (4000 tok/s, 100ms TTFB) → the top of the scale.+ expect(speedRankOf('google', 'quick')).toBe(1);+ });++ it('demotes a model whose observed speed is dragged down by timeouts', () => {+ addModel({ platform: 'google', modelId: 'slowish', name: 'Slowish', speedRank: 1 });+ addRequests('google', 'slowish', { count: 30, status: 'success', outTokens: 100, latencyMs: 1000, ttfbMs: 200 });+ addRequests('google', 'slowish', {+ count: 30, status: 'error', outTokens: 0, latencyMs: 120_000,+ error: 'The operation was aborted (google, chat, 120s)',+ });++ refreshStatsCache(getDb(), true);+ writeObservedSpeedRanks(getDb());+ expect(speedRankOf('google', 'slowish')).toBeGreaterThan(5);+ });++ it('leaves a low-sample model on its catalog rank', () => {+ addModel({ platform: 'google', modelId: 'shy', name: 'Shy', speedRank: 7 });+ addRequests('google', 'shy', { count: 3, status: 'success', outTokens: 4000, latencyMs: 1000, ttfbMs: 100 });++ refreshStatsCache(getDb(), true);+ expect(writeObservedSpeedRanks(getDb())).toBe(0);+ expect(speedRankOf('google', 'shy')).toBe(7);+ });++ it('never clobbers a user-set speed_rank override', () => {+ addModel({ platform: 'google', modelId: 'pinned', name: 'Pinned', speedRank: 2 });+ upsertModelOverrides(getDb(), 'google', 'pinned', { speedRank: 2 });+ addRequests('google', 'pinned', { count: 40, status: 'success', outTokens: 4000, latencyMs: 1000, ttfbMs: 100 });++ refreshStatsCache(getDb(), true);+ expect(writeObservedSpeedRanks(getDb())).toBe(0);+ expect(speedRankOf('google', 'pinned')).toBe(2);+ });++ it('an override on a DIFFERENT field does not block the observed rank', () => {+ addModel({ platform: 'google', modelId: 'renamed', name: 'Renamed', speedRank: 9 });+ upsertModelOverrides(getDb(), 'google', 'renamed', { displayName: 'My Name' });+ addRequests('google', 'renamed', { count: 40, status: 'success', outTokens: 4000, latencyMs: 1000, ttfbMs: 100 });++ refreshStatsCache(getDb(), true);+ expect(writeObservedSpeedRanks(getDb())).toBe(1);+ expect(speedRankOf('google', 'renamed')).toBe(1);+ });++ it('is idempotent — a second pass writes nothing', () => {+ addModel({ platform: 'google', modelId: 'steady', name: 'Steady', speedRank: 9 });+ addRequests('google', 'steady', { count: 40, status: 'success', outTokens: 4000, latencyMs: 1000, ttfbMs: 100 });++ refreshStatsCache(getDb(), true);+ expect(writeObservedSpeedRanks(getDb())).toBe(1);+ expect(writeObservedSpeedRanks(getDb())).toBe(0);+ });++ it('refreshStatsCache drives the writeback on its own', () => {+ addModel({ platform: 'google', modelId: 'auto', name: 'Auto', speedRank: 9 });+ addRequests('google', 'auto', { count: 40, status: 'success', outTokens: 4000, latencyMs: 1000, ttfbMs: 100 });++ resetSpeedRankWriteback();+ refreshStatsCache(getDb(), true);+ expect(speedRankOf('google', 'auto')).toBe(1);+ });+});diff --git a/server/src/lib/error-classify.ts b/server/src/lib/error-classify.tsindex 76b82874..3893e5e0 100644--- a/server/src/lib/error-classify.ts+++ b/server/src/lib/error-classify.ts@@ -119,6 +119,30 @@ export function isRateLimitSignal(err: any): boolean {|| msg.includes('quota') || msg.includes('resource_exhausted');}+// ── Timeout wording ─────────────────────────────────────────────────────────+// The message markers that mean "this attempt ran out of time", in every+// wording the stack produces: the per-attempt HTTP deadline ("The operation was+// aborted (groq, chat, 120s)" — enrichAbort in lib/proxy.ts), the mid-stream+// inactivity watchdog ("… stream stalled: no data for 90000ms (timeout)"), the+// first-byte grace budget, and a raw socket ETIMEDOUT.+//+// A client hang-up is deliberately NOT in here: newClientAbortError's message+// ("client disconnected — upstream request canceled") carries none of these+// markers precisely so a vanished client is never read as provider slowness.+//+// Single source of truth for two consumers: the attempt-trail classifier+// (classifyAttemptError) and the analytics query that folds timeouts into the+// speed score (refreshStatsCache, #619) — so the trail and the score always+// agree about what a timeout is.+export const TIMEOUT_ERROR_MARKERS = ['timeout', 'stalled', 'etimedout', 'aborted'] as const;++/** True when an error message reads as a timeout. Expects raw text; caller need+ * not lowercase it. */+export function isTimeoutErrorText(message: unknown): boolean {+ const msg = (typeof message === 'string' ? message : String(message ?? '')).toLowerCase();+ return TIMEOUT_ERROR_MARKERS.some(marker => msg.includes(marker));+}+// ── Client-caused aborts ─────────────────────────────────────────────────────// When the gateway's OWN client hangs up, the proxy surfaces abort the composed// fetch signal with this marked error, and undici rejects the in-flight fetch /diff --git a/server/src/lib/fallback-loop.ts b/server/src/lib/fallback-loop.tsindex 07c4dbf7..e9790cb8 100644--- a/server/src/lib/fallback-loop.ts+++ b/server/src/lib/fallback-loop.ts@@ -41,6 +41,7 @@ import {isProviderBadRequestError,isProviderDegradedError,isContextTooLargeError,+ isTimeoutErrorText,} from './error-classify.js';import { sanitizeProviderErrorMessage, summarizeAttemptError } from './error-redaction.js';import { checkKeyHealth, markKeyHealthyFromRequest } from '../services/health.js';@@ -295,7 +296,7 @@ export function classifyAttemptError(err: any): AttemptErrorClass {const msg = (err?.message ?? '').toLowerCase();if (msg.includes('empty completion')) return 'empty_completion';if (msg.includes('ignored response_format') || msg.includes('truncated json')) return 'format_ignored';- if (msg.includes('timeout') || msg.includes('stalled') || msg.includes('etimedout') || msg.includes('aborted')) return 'timeout';+ if (isTimeoutErrorText(msg)) return 'timeout';if (msg.includes('429') || msg.includes('rate limit') || msg.includes('too many requests') || msg.includes('quota')) return 'rate_limited';const status = typeof err?.status === 'number' ? err.status : 0;if (status >= 500 || msg.includes('500') || msg.includes('502') || msg.includes('503') || msg.includes('unavailable') || msg.includes('internal server error')) return 'upstream_error';diff --git a/server/src/services/model-state.ts b/server/src/services/model-state.tsindex cef802d9..360af803 100644--- a/server/src/services/model-state.ts+++ b/server/src/services/model-state.ts@@ -138,6 +138,30 @@ export function getModelOverrides(return parseOverrides(row?.overrides_json);}+/**+ * Every model whose stored overrides pin ONE given field, as a set of+ * "platform:model_id" keys. One query over a table that only ever holds the+ * models a user has actually touched, so callers that need "is this field+ * user-owned?" for a whole catalog don't do it per model.+ *+ * Note it keys off the field, not the row: a user who renamed a model has an+ * override row but has said nothing about its speed_rank, so a derived value+ * may still fill that column (#619).+ */+export function modelsWithOverriddenField(+ db: Db,+ field: keyof ModelOverridePatch,+): Set<string> {+ const rows = db.prepare('SELECT platform, model_id, overrides_json FROM model_overrides')+ .all() as { platform: string; model_id: string; overrides_json: string }[];+ const pinned = new Set<string>();+ for (const row of rows) {+ const overrides = parseOverrides(row.overrides_json);+ if (overrides[field] !== undefined) pinned.add(`${row.platform}:${row.model_id}`);+ }+ return pinned;+}+export function applyModelOverrides(db: Db,platform: string,diff --git a/server/src/services/router.ts b/server/src/services/router.tsindex 4d95b587..b7b0ae7c 100644--- a/server/src/services/router.ts+++ b/server/src/services/router.ts@@ -17,7 +17,10 @@ import {BANDIT_PRESETS, DEFAULT_STRATEGY, type RoutingStrategy, type RoutingWeights,reliabilityPosterior, expectedReliability, sampleBeta,speedScore, intelligenceScore, headroomFactor, rateLimitFactor, combineScore,+ observedSpeedRank, TIMEOUT_LATENCY_CAP_MS,} from './scoring.js';+import { TIMEOUT_ERROR_MARKERS } from '../lib/error-classify.js';+import { modelsWithOverriddenField } from './model-state.js';import { parseBudget } from '../lib/budget.js';import { platformDropsResponseFormat } from '../lib/sampling-params.js';import { isUnifyEnabled, getModelGroups, resolveRequestedIdToMembers } from './model-groups.js';@@ -341,9 +344,16 @@ const CACHE_TTL_MS = 60 * 1000;interface ModelStats {successes: number; // decay-weighted pseudo-countfailures: number; // decay-weighted pseudo-count- tokPerSec: number; // from successful requests only (0 = no data)+ // Output tokens from successes over the time spent on successes AND timeouts+ // (#619 — see the accumulator below); 0 = no data.+ tokPerSec: number;avgTtfbMs: number | null; // null = no first-byte timing yetmonthlyUsedTokens: number; // calendar-month usage, for the headroom guardrail+ // Decay-weighted requests that actually SAY something about speed: successes+ // plus timeouts. A model can have hundreds of 401s and still no speed signal,+ // so this — not successes + failures — is what gates the observed speed_rank+ // writeback.+ speedSamples: number;}// Per-key slice of the same window (#580): reliability/speed observed through@@ -366,6 +376,16 @@ function decayWeight(ageDays: number): number {return Math.pow(0.5, Math.max(0, ageDays) / HALF_LIFE_DAYS);}+// SQL predicate for "this row is a timed-out request" (#619). `requests.status`+// only ever holds 'success' or 'error' — a timeout is an error row whose text+// carries one of the shared timeout markers (lib/error-classify.ts), which is+// also what the failover attempt trail classifies on. The markers are+// hard-coded lowercase identifiers from our own source, never user input, so+// interpolating them into the LIKE list is safe.+const IS_TIMEOUT_SQL = `(status != 'success' AND (${+ TIMEOUT_ERROR_MARKERS.map(m => `LOWER(COALESCE(error, '')) LIKE '%${m}%'`).join(' OR ')+}))`;+export function refreshStatsCache(db: Db, force = false): void {if (!force && statsCache && Date.now() - statsCacheTime < CACHE_TTL_MS) return;@@ -382,25 +402,37 @@ export function refreshStatsCache(db: Db, force = false): void {SUM(CASE WHEN status = 'success' THEN output_tokens ELSE 0 END) AS succ_out,SUM(CASE WHEN status = 'success' THEN latency_ms ELSE 0 END) AS succ_lat,SUM(CASE WHEN status = 'success' AND ttfb_ms IS NOT NULL THEN ttfb_ms ELSE 0 END) AS succ_ttfb_sum,- SUM(CASE WHEN status = 'success' AND ttfb_ms IS NOT NULL THEN 1 ELSE 0 END) AS succ_ttfb_cnt+ SUM(CASE WHEN status = 'success' AND ttfb_ms IS NOT NULL THEN 1 ELSE 0 END) AS succ_ttfb_cnt,+ SUM(CASE WHEN ${IS_TIMEOUT_SQL} THEN 1 ELSE 0 END) AS timeouts,+ SUM(CASE WHEN ${IS_TIMEOUT_SQL} THEN MIN(MAX(latency_ms, 0), ${TIMEOUT_LATENCY_CAP_MS}) ELSE 0 END) AS timeout_latFROM requestsWHERE created_at >= ?GROUP BY platform, model_id, key_id, age_days`).all(since) as Array<{platform: string; model_id: string; key_id: number | null; age_days: number; total: number; successes: number;succ_out: number; succ_lat: number; succ_ttfb_sum: number; succ_ttfb_cnt: number;+ timeouts: number; timeout_lat: number;}>;// Accumulate decay-weighted sums per model AND per key.- interface Acc { wSucc: number; wFail: number; wOut: number; wLat: number; wTtfbSum: number; wTtfbCnt: number }- const emptyAcc = (): Acc => ({ wSucc: 0, wFail: 0, wOut: 0, wLat: 0, wTtfbSum: 0, wTtfbCnt: 0 });+ //+ // Timeouts (#619) land in the SAME latency/TTFB accumulators as successes,+ // because that is what they are: time spent, nothing produced. Each one adds+ // its capped wall-clock latency to the throughput denominator with zero+ // output tokens, and that same figure as a first-byte sample — which is past+ // TTFB_WORST_MS, so it scores no latency credit. Net effect: a model that+ // times out constantly can no longer keep a stellar speed number just because+ // its handful of successes were quick.+ interface Acc { wSucc: number; wFail: number; wOut: number; wLat: number; wTtfbSum: number; wTtfbCnt: number; wTimeouts: number }+ const emptyAcc = (): Acc => ({ wSucc: 0, wFail: 0, wOut: 0, wLat: 0, wTtfbSum: 0, wTtfbCnt: 0, wTimeouts: 0 });const addBucket = (a: Acc, w: number, b: (typeof buckets)[number]): void => {a.wSucc += w * b.successes;a.wFail += w * (b.total - b.successes);a.wOut += w * b.succ_out;- a.wLat += w * b.succ_lat;- a.wTtfbSum += w * b.succ_ttfb_sum;- a.wTtfbCnt += w * b.succ_ttfb_cnt;+ a.wLat += w * (b.succ_lat + b.timeout_lat);+ a.wTtfbSum += w * (b.succ_ttfb_sum + b.timeout_lat);+ a.wTtfbCnt += w * (b.succ_ttfb_cnt + b.timeouts);+ a.wTimeouts += w * b.timeouts;};const acc = new Map<string, Acc>();const keyAcc = new Map<string, Acc>();@@ -436,12 +468,13 @@ export function refreshStatsCache(db: Db, force = false): void {tokPerSec: a.wLat > 0 ? (a.wOut * 1000) / a.wLat : 0,avgTtfbMs: a.wTtfbCnt > 0 ? a.wTtfbSum / a.wTtfbCnt : null,monthlyUsedTokens: usageMap.get(key) ?? 0,+ speedSamples: a.wSucc + a.wTimeouts,});}// Models with month usage but no recent window data still need a headroom number.for (const [key, used] of usageMap) {if (!next.has(key)) {- next.set(key, { successes: 0, failures: 0, tokPerSec: 0, avgTtfbMs: null, monthlyUsedTokens: used });+ next.set(key, { successes: 0, failures: 0, tokPerSec: 0, avgTtfbMs: null, monthlyUsedTokens: used, speedSamples: 0 });}}@@ -458,6 +491,78 @@ export function refreshStatsCache(db: Db, force = false): void {statsCache = next;keyStatsCache = nextKeys;statsCacheTime = Date.now();++ // Natural tail of a recompute: fold what we just measured back into+ // models.speed_rank (#619). Never allowed to break routing — the caches above+ // are already published, and a failed write just means the column keeps its+ // previous value until the next pass.+ if (Date.now() - speedRankWriteTime >= SPEED_RANK_WRITE_INTERVAL_MS) {+ speedRankWriteTime = Date.now();+ try {+ writeObservedSpeedRanks(db);+ } catch (e) {+ console.error('Failed to write observed speed ranks:', e);+ }+ }+}++// ── Observed speed_rank writeback (#619) ────────────────────────────────────+// models.speed_rank is the catalog's hand-assigned speed ordering and drives+// the dashboard's sort-by-speed preset. It was only ever WRITTEN by the seed+// migrations, catalog sync, and an explicit user override — never by anything+// that had actually watched the model run, so a relay model that hangs on half+// its calls kept whatever rank the catalog guessed for it forever.+//+// This folds the live speed axis back into the column, under three rules:+// - a model needs SPEED_RANK_MIN_SAMPLES decay-weighted speed-bearing+// requests (successes + timeouts) before we claim to know anything; below+// that it keeps its catalog value;+// - a user-set speed_rank override always wins — we skip those models+// entirely rather than fight applyModelOverrides for the column;+// - the UPDATE is guarded on the value actually changing, so a steady system+// writes nothing at all.+//+// A catalog sync re-stamps speed_rank from the catalog; the next pass simply+// re-derives the observed value, which is why this is a periodic write rather+// than a one-shot migration.+export const SPEED_RANK_MIN_SAMPLES = 20;+const SPEED_RANK_WRITE_INTERVAL_MS = 10 * 60 * 1000;+let speedRankWriteTime = 0;++/** Test hook: forget when the last writeback ran so the next refresh does one. */+export function resetSpeedRankWriteback(): void {+ speedRankWriteTime = 0;+}++/**+ * Write an observed speed rank for every model with enough recent samples and+ * no user-set speed_rank override. Returns how many rows actually changed.+ * Reads the stats cache as-is — callers refresh it first (refreshStatsCache+ * calls this from its own tail).+ */+export function writeObservedSpeedRanks(db: Db): number {+ if (!statsCache || statsCache.size === 0) return 0;++ const pinned = modelsWithOverriddenField(db, 'speedRank');+ const rows = db.prepare('SELECT id, platform, model_id, speed_rank FROM models')+ .all() as { id: number; platform: string; model_id: string; speed_rank: number }[];++ const update = db.prepare('UPDATE models SET speed_rank = ? WHERE id = ?');+ let written = 0;+ const tx = db.transaction(() => {+ for (const row of rows) {+ const key = `${row.platform}:${row.model_id}`;+ if (pinned.has(key)) continue;+ const stats = statsCache!.get(key);+ if (!stats || stats.speedSamples < SPEED_RANK_MIN_SAMPLES) continue;+ const rank = observedSpeedRank(speedScore(stats.tokPerSec, stats.avgTtfbMs));+ if (rank === row.speed_rank) continue;+ update.run(rank, row.id);+ written++;+ }+ });+ tx();+ return written;}// Composite intelligence: size_label is the cross-provider capability tierdiff --git a/server/src/services/scoring.ts b/server/src/services/scoring.tsindex d713176b..4520604c 100644--- a/server/src/services/scoring.ts+++ b/server/src/services/scoring.ts@@ -106,6 +106,41 @@ export function speedScore(tokPerSec: number, ttfbMs: number | null): number {return THROUGHPUT_WEIGHT * tp + TTFB_WEIGHT * ttfbScore(ttfbMs);}+// ── Speed: what a timeout costs (#619) ──────────────────────────────────────+// A request that times out IS the model being slow — the reporter's exact+// complaint: a relay model that hangs on half its calls loses reliability but+// used to keep a pristine speed number, because the analytics query fed the+// speed axis from `status = 'success'` rows ONLY. So a timeout now contributes+// to the speed window as what it actually was: its wall-clock latency with ZERO+// output tokens (dragging throughput down) and that same latency as its+// first-byte time (which lands past TTFB_WORST_MS, i.e. no latency credit).+//+// Capped, because `latency_ms` on a timed-out row is unbounded — a provider+// that holds a socket open for twenty minutes, or a per-platform+// PROVIDER_TIMEOUT override in the hundreds of seconds, would otherwise let a+// single hang dominate the whole 7-day window and flatten the axis for every+// model on that platform. Two minutes is above the built-in per-attempt HTTP+// deadline and the 90s stream-stall watchdog, so a normal timeout is counted in+// full and only genuine outliers are clipped.+export const TIMEOUT_LATENCY_CAP_MS = 120_000;++// ── Observed speed rank ─────────────────────────────────────────────────────+// `models.speed_rank` is the catalog's hand-assigned "how fast is this model"+// ordering (1 = fastest; the shipped catalog spans 1..11) and it drives the+// dashboard's sort-by-speed preset. Until now nothing ever wrote it at runtime,+// so it stayed frozen at whatever the seed said no matter how the model+// actually behaved (#619). observedSpeedRank projects the live [0,1] speed axis+// back onto that same low-is-fast scale so an observed rank is directly+// comparable with a catalog one.+export const OBSERVED_SPEED_RANK_BEST = 1;+export const OBSERVED_SPEED_RANK_WORST = 10;++export function observedSpeedRank(speed: number): number {+ const s = Math.min(1, Math.max(0, Number.isFinite(speed) ? speed : 0));+ const span = OBSERVED_SPEED_RANK_WORST - OBSERVED_SPEED_RANK_BEST;+ return OBSERVED_SPEED_RANK_BEST + Math.round((1 - s) * span);+}+// ── Intelligence ────────────────────────────────────────────────────────────// Caller supplies a composite (tier-first, rank-as-tiebreaker — see router) and// the min/max across the enabled chain. We min-max normalize to [0,1], 1 = best.