Skip to content

7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.

TypeScript

fix(router): let timeouts cost speed, and write an observed speed_rank (#619) (#645)

Tashfeenfc4e47dc5c29
diff --git a/server/src/__tests__/services/router-speed-scoring.test.ts b/server/src/__tests__/services/router-speed-scoring.test.ts
new file mode 100644
index 00000000..37a835b7
--- /dev/null
+++ b/server/src/__tests__/services/router-speed-scoring.test.ts
@@ -0,0 +1,244 @@
+import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest';
+import {
+ refreshStatsCache, getRoutingScores, setRoutingStrategy,
+ writeObservedSpeedRanks, resetSpeedRankWriteback,
+} from '../../services/router.js';
+import { observedSpeedRank, TIMEOUT_LATENCY_CAP_MS } from '../../services/scoring.js';
+import { upsertModelOverrides } from '../../services/model-state.js';
+import { getDb, initDb } from '../../db/index.js';
+
+vi.mock('../../lib/crypto.js', async () => {
+ const actual = await vi.importActual('../../lib/crypto.js');
+ return { ...actual, decrypt: vi.fn(() => 'mocked-api-key') };
+});
+
+const ORIGINAL_DEV_MODE = process.env.DEV_MODE;
+const ORIGINAL_NODE_ENV = process.env.NODE_ENV;
+
+function addModel(opts: {
+ platform: string; modelId: string; name: string; speedRank?: number; priority?: number;
+}): number {
+ const db = getDb();
+ db.prepare(`
+ INSERT INTO models (platform, model_id, display_name, intelligence_rank, speed_rank, size_label, monthly_token_budget, enabled)
+ VALUES (?, ?, ?, 3, ?, 'Large', '~50M', 1)
+ `).run(opts.platform, opts.modelId, opts.name, opts.speedRank ?? 5);
+ const id = (db.prepare('SELECT id FROM models WHERE platform = ? AND model_id = ?')
+ .get(opts.platform, opts.modelId) as { id: number }).id;
+ db.prepare('INSERT INTO fallback_config (model_db_id, priority, enabled) VALUES (?, ?, 1)')
+ .run(id, opts.priority ?? 1);
+ db.prepare(`
+ INSERT INTO api_keys (platform, label, encrypted_key, iv, auth_tag, status, enabled)
+ VALUES (?, 'k', 'enc', 'iv', 'tag', 'healthy', 1)
+ `).run(opts.platform);
+ return id;
+}
+
+// Fresh rows (age 0 → decay weight 1) so the arithmetic in the assertions is exact.
+function addRequests(platform: string, modelId: string, opts: {
+ count: number; status: string; outTokens?: number; latencyMs?: number;
+ ttfbMs?: number | null; error?: string | null;
+}) {
+ const ins = getDb().prepare(`
+ INSERT INTO requests (platform, model_id, key_id, status, input_tokens, output_tokens, latency_ms, error, ttfb_ms)
+ VALUES (?, ?, 1, ?, 0, ?, ?, ?, ?)
+ `);
+ for (let i = 0; i < opts.count; i++) {
+ ins.run(platform, modelId, opts.status, opts.outTokens ?? 0,
+ opts.latencyMs ?? 0, opts.error ?? null, opts.ttfbMs ?? null);
+ }
+}
+
+function speedOf(modelId: string): number {
+ const { scores } = getRoutingScores();
+ const row = scores.find(s => s.modelId === modelId);
+ if (!row) throw new Error(`no score row for ${modelId}`);
+ return row.speed;
+}
+
+function speedRankOf(platform: string, modelId: string): number {
+ return (getDb().prepare('SELECT speed_rank FROM models WHERE platform = ? AND model_id = ?')
+ .get(platform, modelId) as { speed_rank: number }).speed_rank;
+}
+
+describe('speed scoring: timeouts count against speed (#619)', () => {
+ beforeEach(() => {
+ process.env.DEV_MODE = 'true';
+ process.env.NODE_ENV = 'test';
+ initDb(':memory:');
+ getDb().exec('DELETE FROM fallback_config; DELETE FROM api_keys; DELETE FROM models; DELETE FROM requests; DELETE FROM model_overrides;');
+ setRoutingStrategy('balanced');
+ resetSpeedRankWriteback();
+ });
+
+ afterEach(() => {
+ if (ORIGINAL_DEV_MODE === undefined) delete process.env.DEV_MODE; else process.env.DEV_MODE = ORIGINAL_DEV_MODE;
+ if (ORIGINAL_NODE_ENV === undefined) delete process.env.NODE_ENV; else process.env.NODE_ENV = ORIGINAL_NODE_ENV;
+ });
+
+ it('the same successes score measurably slower once timeouts are in the window', () => {
+ addModel({ platform: 'google', modelId: 'clean', name: 'Clean', priority: 1 });
+ addModel({ platform: 'groq', modelId: 'timeouty', name: 'Timeouty', priority: 2 });
+
+ // Identical successful traffic on both.
+ for (const [platform, modelId] of [['google', 'clean'], ['groq', 'timeouty']] as const) {
+ addRequests(platform, modelId, { count: 40, status: 'success', outTokens: 100, latencyMs: 1000, ttfbMs: 200 });
+ }
+ // …but one of them also times out constantly. A timeout is logged as an
+ // 'error' row whose text carries the abort/timeout marker.
+ addRequests('groq', 'timeouty', {
+ count: 10, status: 'error', outTokens: 0, latencyMs: 120_000, ttfbMs: null,
+ error: 'The operation was aborted (groq, chat, 120s)',
+ });
+
+ refreshStatsCache(getDb(), true);
+ const clean = speedOf('clean');
+ const timeouty = speedOf('timeouty');
+ expect(timeouty).toBeLessThan(clean - 0.2);
+ });
+
+ it('a model that only ever times out lands at the bottom of the speed axis', () => {
+ addModel({ platform: 'google', modelId: 'dead', name: 'Dead', priority: 1 });
+ addRequests('google', 'dead', {
+ count: 25, status: 'error', outTokens: 0, latencyMs: 90_000, ttfbMs: null,
+ error: 'google stream stalled: no data for 90000ms (timeout)',
+ });
+
+ refreshStatsCache(getDb(), true);
+ // Not the optimistic no-data prior (0.6) — it timed out every single time.
+ expect(speedOf('dead')).toBeLessThan(0.05);
+ });
+
+ it('one pathological hang is capped, not allowed to dominate the window', () => {
+ addModel({ platform: 'google', modelId: 'capped', name: 'Capped', priority: 1 });
+ addModel({ platform: 'groq', modelId: 'uncapped', name: 'Uncapped', priority: 2 });
+ for (const [platform, modelId] of [['google', 'capped'], ['groq', 'uncapped']] as const) {
+ addRequests(platform, modelId, { count: 40, status: 'success', outTokens: 100, latencyMs: 1000, ttfbMs: 200 });
+ addRequests(platform, modelId, {
+ count: 1, status: 'error', outTokens: 0,
+ latencyMs: modelId === 'capped' ? TIMEOUT_LATENCY_CAP_MS : TIMEOUT_LATENCY_CAP_MS * 50,
+ error: 'The operation was aborted (x, chat, 120s)',
+ });
+ }
+ refreshStatsCache(getDb(), true);
+ expect(speedOf('uncapped')).toBeCloseTo(speedOf('capped'), 10);
+ });
+
+ it('a non-timeout failure leaves the speed axis alone (it only costs reliability)', () => {
+ addModel({ platform: 'google', modelId: 'fast', name: 'Fast', priority: 1 });
+ addModel({ platform: 'groq', modelId: 'refusing', name: 'Refusing', priority: 2 });
+ for (const [platform, modelId] of [['google', 'fast'], ['groq', 'refusing']] as const) {
+ addRequests(platform, modelId, { count: 40, status: 'success', outTokens: 100, latencyMs: 1000, ttfbMs: 200 });
+ }
+ addRequests('groq', 'refusing', {
+ count: 20, status: 'error', outTokens: 0, latencyMs: 200,
+ error: 'API error 401: invalid api key',
+ });
+
+ refreshStatsCache(getDb(), true);
+ expect(speedOf('refusing')).toBeCloseTo(speedOf('fast'), 10);
+ const { scores } = getRoutingScores();
+ const refusing = scores.find(s => s.modelId === 'refusing')!;
+ const fast = scores.find(s => s.modelId === 'fast')!;
+ expect(refusing.reliability).toBeLessThan(fast.reliability);
+ });
+});
+
+describe('observed speed_rank writeback (#619)', () => {
+ beforeEach(() => {
+ process.env.DEV_MODE = 'true';
+ process.env.NODE_ENV = 'test';
+ initDb(':memory:');
+ getDb().exec('DELETE FROM fallback_config; DELETE FROM api_keys; DELETE FROM models; DELETE FROM requests; DELETE FROM model_overrides;');
+ setRoutingStrategy('balanced');
+ // Drain the periodic writeback against the now-empty DB so its interval gate
+ // is closed: each test below drives writeObservedSpeedRanks itself and wants
+ // an exact count. The one test that checks the automatic tail re-opens it.
+ resetSpeedRankWriteback();
+ refreshStatsCache(getDb(), true);
+ });
+
+ afterEach(() => {
+ if (ORIGINAL_DEV_MODE === undefined) delete process.env.DEV_MODE; else process.env.DEV_MODE = ORIGINAL_DEV_MODE;
+ if (ORIGINAL_NODE_ENV === undefined) delete process.env.NODE_ENV; else process.env.NODE_ENV = ORIGINAL_NODE_ENV;
+ });
+
+ it('maps the speed axis onto the catalog rank scale (1 = fastest, low is fast)', () => {
+ expect(observedSpeedRank(1)).toBe(1);
+ expect(observedSpeedRank(0)).toBe(10);
+ expect(observedSpeedRank(0.5)).toBeGreaterThan(observedSpeedRank(0.9));
+ // Out-of-range inputs are clamped, never NaN.
+ expect(observedSpeedRank(5)).toBe(1);
+ expect(observedSpeedRank(-5)).toBe(10);
+ });
+
+ it('writes an observed rank for a well-sampled model', () => {
+ addModel({ platform: 'google', modelId: 'quick', name: 'Quick', speedRank: 9 });
+ addRequests('google', 'quick', { count: 40, status: 'success', outTokens: 4000, latencyMs: 1000, ttfbMs: 100 });
+
+ refreshStatsCache(getDb(), true);
+ expect(writeObservedSpeedRanks(getDb())).toBe(1);
+ // Genuinely fast traffic (4000 tok/s, 100ms TTFB) → the top of the scale.
+ expect(speedRankOf('google', 'quick')).toBe(1);
+ });
+
+ it('demotes a model whose observed speed is dragged down by timeouts', () => {
+ addModel({ platform: 'google', modelId: 'slowish', name: 'Slowish', speedRank: 1 });
+ addRequests('google', 'slowish', { count: 30, status: 'success', outTokens: 100, latencyMs: 1000, ttfbMs: 200 });
+ addRequests('google', 'slowish', {
+ count: 30, status: 'error', outTokens: 0, latencyMs: 120_000,
+ error: 'The operation was aborted (google, chat, 120s)',
+ });
+
+ refreshStatsCache(getDb(), true);
+ writeObservedSpeedRanks(getDb());
+ expect(speedRankOf('google', 'slowish')).toBeGreaterThan(5);
+ });
+
+ it('leaves a low-sample model on its catalog rank', () => {
+ addModel({ platform: 'google', modelId: 'shy', name: 'Shy', speedRank: 7 });
+ addRequests('google', 'shy', { count: 3, status: 'success', outTokens: 4000, latencyMs: 1000, ttfbMs: 100 });
+
+ refreshStatsCache(getDb(), true);
+ expect(writeObservedSpeedRanks(getDb())).toBe(0);
+ expect(speedRankOf('google', 'shy')).toBe(7);
+ });
+
+ it('never clobbers a user-set speed_rank override', () => {
+ addModel({ platform: 'google', modelId: 'pinned', name: 'Pinned', speedRank: 2 });
+ upsertModelOverrides(getDb(), 'google', 'pinned', { speedRank: 2 });
+ addRequests('google', 'pinned', { count: 40, status: 'success', outTokens: 4000, latencyMs: 1000, ttfbMs: 100 });
+
+ refreshStatsCache(getDb(), true);
+ expect(writeObservedSpeedRanks(getDb())).toBe(0);
+ expect(speedRankOf('google', 'pinned')).toBe(2);
+ });
+
+ it('an override on a DIFFERENT field does not block the observed rank', () => {
+ addModel({ platform: 'google', modelId: 'renamed', name: 'Renamed', speedRank: 9 });
+ upsertModelOverrides(getDb(), 'google', 'renamed', { displayName: 'My Name' });
+ addRequests('google', 'renamed', { count: 40, status: 'success', outTokens: 4000, latencyMs: 1000, ttfbMs: 100 });
+
+ refreshStatsCache(getDb(), true);
+ expect(writeObservedSpeedRanks(getDb())).toBe(1);
+ expect(speedRankOf('google', 'renamed')).toBe(1);
+ });
+
+ it('is idempotent — a second pass writes nothing', () => {
+ addModel({ platform: 'google', modelId: 'steady', name: 'Steady', speedRank: 9 });
+ addRequests('google', 'steady', { count: 40, status: 'success', outTokens: 4000, latencyMs: 1000, ttfbMs: 100 });
+
+ refreshStatsCache(getDb(), true);
+ expect(writeObservedSpeedRanks(getDb())).toBe(1);
+ expect(writeObservedSpeedRanks(getDb())).toBe(0);
+ });
+
+ it('refreshStatsCache drives the writeback on its own', () => {
+ addModel({ platform: 'google', modelId: 'auto', name: 'Auto', speedRank: 9 });
+ addRequests('google', 'auto', { count: 40, status: 'success', outTokens: 4000, latencyMs: 1000, ttfbMs: 100 });
+
+ resetSpeedRankWriteback();
+ refreshStatsCache(getDb(), true);
+ expect(speedRankOf('google', 'auto')).toBe(1);
+ });
+});
diff --git a/server/src/lib/error-classify.ts b/server/src/lib/error-classify.ts
index 76b82874..3893e5e0 100644
--- a/server/src/lib/error-classify.ts
+++ b/server/src/lib/error-classify.ts
@@ -119,6 +119,30 @@ export function isRateLimitSignal(err: any): boolean {
|| msg.includes('quota') || msg.includes('resource_exhausted');
}
+// ── Timeout wording ─────────────────────────────────────────────────────────
+// The message markers that mean "this attempt ran out of time", in every
+// wording the stack produces: the per-attempt HTTP deadline ("The operation was
+// aborted (groq, chat, 120s)" — enrichAbort in lib/proxy.ts), the mid-stream
+// inactivity watchdog ("… stream stalled: no data for 90000ms (timeout)"), the
+// first-byte grace budget, and a raw socket ETIMEDOUT.
+//
+// A client hang-up is deliberately NOT in here: newClientAbortError's message
+// ("client disconnected — upstream request canceled") carries none of these
+// markers precisely so a vanished client is never read as provider slowness.
+//
+// Single source of truth for two consumers: the attempt-trail classifier
+// (classifyAttemptError) and the analytics query that folds timeouts into the
+// speed score (refreshStatsCache, #619) — so the trail and the score always
+// agree about what a timeout is.
+export const TIMEOUT_ERROR_MARKERS = ['timeout', 'stalled', 'etimedout', 'aborted'] as const;
+
+/** True when an error message reads as a timeout. Expects raw text; caller need
+ * not lowercase it. */
+export function isTimeoutErrorText(message: unknown): boolean {
+ const msg = (typeof message === 'string' ? message : String(message ?? '')).toLowerCase();
+ return TIMEOUT_ERROR_MARKERS.some(marker => msg.includes(marker));
+}
+
// ── Client-caused aborts ─────────────────────────────────────────────────────
// When the gateway's OWN client hangs up, the proxy surfaces abort the composed
// fetch signal with this marked error, and undici rejects the in-flight fetch /
diff --git a/server/src/lib/fallback-loop.ts b/server/src/lib/fallback-loop.ts
index 07c4dbf7..e9790cb8 100644
--- a/server/src/lib/fallback-loop.ts
+++ b/server/src/lib/fallback-loop.ts
@@ -41,6 +41,7 @@ import {
isProviderBadRequestError,
isProviderDegradedError,
isContextTooLargeError,
+ isTimeoutErrorText,
} from './error-classify.js';
import { sanitizeProviderErrorMessage, summarizeAttemptError } from './error-redaction.js';
import { checkKeyHealth, markKeyHealthyFromRequest } from '../services/health.js';
@@ -295,7 +296,7 @@ export function classifyAttemptError(err: any): AttemptErrorClass {
const msg = (err?.message ?? '').toLowerCase();
if (msg.includes('empty completion')) return 'empty_completion';
if (msg.includes('ignored response_format') || msg.includes('truncated json')) return 'format_ignored';
- if (msg.includes('timeout') || msg.includes('stalled') || msg.includes('etimedout') || msg.includes('aborted')) return 'timeout';
+ if (isTimeoutErrorText(msg)) return 'timeout';
if (msg.includes('429') || msg.includes('rate limit') || msg.includes('too many requests') || msg.includes('quota')) return 'rate_limited';
const status = typeof err?.status === 'number' ? err.status : 0;
if (status >= 500 || msg.includes('500') || msg.includes('502') || msg.includes('503') || msg.includes('unavailable') || msg.includes('internal server error')) return 'upstream_error';
diff --git a/server/src/services/model-state.ts b/server/src/services/model-state.ts
index cef802d9..360af803 100644
--- a/server/src/services/model-state.ts
+++ b/server/src/services/model-state.ts
@@ -138,6 +138,30 @@ export function getModelOverrides(
return parseOverrides(row?.overrides_json);
}
+/**
+ * Every model whose stored overrides pin ONE given field, as a set of
+ * "platform:model_id" keys. One query over a table that only ever holds the
+ * models a user has actually touched, so callers that need "is this field
+ * user-owned?" for a whole catalog don't do it per model.
+ *
+ * Note it keys off the field, not the row: a user who renamed a model has an
+ * override row but has said nothing about its speed_rank, so a derived value
+ * may still fill that column (#619).
+ */
+export function modelsWithOverriddenField(
+ db: Db,
+ field: keyof ModelOverridePatch,
+): Set<string> {
+ const rows = db.prepare('SELECT platform, model_id, overrides_json FROM model_overrides')
+ .all() as { platform: string; model_id: string; overrides_json: string }[];
+ const pinned = new Set<string>();
+ for (const row of rows) {
+ const overrides = parseOverrides(row.overrides_json);
+ if (overrides[field] !== undefined) pinned.add(`${row.platform}:${row.model_id}`);
+ }
+ return pinned;
+}
+
export function applyModelOverrides(
db: Db,
platform: string,
diff --git a/server/src/services/router.ts b/server/src/services/router.ts
index 4d95b587..b7b0ae7c 100644
--- a/server/src/services/router.ts
+++ b/server/src/services/router.ts
@@ -17,7 +17,10 @@ import {
BANDIT_PRESETS, DEFAULT_STRATEGY, type RoutingStrategy, type RoutingWeights,
reliabilityPosterior, expectedReliability, sampleBeta,
speedScore, intelligenceScore, headroomFactor, rateLimitFactor, combineScore,
+ observedSpeedRank, TIMEOUT_LATENCY_CAP_MS,
} from './scoring.js';
+import { TIMEOUT_ERROR_MARKERS } from '../lib/error-classify.js';
+import { modelsWithOverriddenField } from './model-state.js';
import { parseBudget } from '../lib/budget.js';
import { platformDropsResponseFormat } from '../lib/sampling-params.js';
import { isUnifyEnabled, getModelGroups, resolveRequestedIdToMembers } from './model-groups.js';
@@ -341,9 +344,16 @@ const CACHE_TTL_MS = 60 * 1000;
interface ModelStats {
successes: number; // decay-weighted pseudo-count
failures: number; // decay-weighted pseudo-count
- tokPerSec: number; // from successful requests only (0 = no data)
+ // Output tokens from successes over the time spent on successes AND timeouts
+ // (#619 — see the accumulator below); 0 = no data.
+ tokPerSec: number;
avgTtfbMs: number | null; // null = no first-byte timing yet
monthlyUsedTokens: number; // calendar-month usage, for the headroom guardrail
+ // Decay-weighted requests that actually SAY something about speed: successes
+ // plus timeouts. A model can have hundreds of 401s and still no speed signal,
+ // so this — not successes + failures — is what gates the observed speed_rank
+ // writeback.
+ speedSamples: number;
}
// Per-key slice of the same window (#580): reliability/speed observed through
@@ -366,6 +376,16 @@ function decayWeight(ageDays: number): number {
return Math.pow(0.5, Math.max(0, ageDays) / HALF_LIFE_DAYS);
}
+// SQL predicate for "this row is a timed-out request" (#619). `requests.status`
+// only ever holds 'success' or 'error' — a timeout is an error row whose text
+// carries one of the shared timeout markers (lib/error-classify.ts), which is
+// also what the failover attempt trail classifies on. The markers are
+// hard-coded lowercase identifiers from our own source, never user input, so
+// interpolating them into the LIKE list is safe.
+const IS_TIMEOUT_SQL = `(status != 'success' AND (${
+ TIMEOUT_ERROR_MARKERS.map(m => `LOWER(COALESCE(error, '')) LIKE '%${m}%'`).join(' OR ')
+}))`;
+
export function refreshStatsCache(db: Db, force = false): void {
if (!force && statsCache && Date.now() - statsCacheTime < CACHE_TTL_MS) return;
@@ -382,25 +402,37 @@ export function refreshStatsCache(db: Db, force = false): void {
SUM(CASE WHEN status = 'success' THEN output_tokens ELSE 0 END) AS succ_out,
SUM(CASE WHEN status = 'success' THEN latency_ms ELSE 0 END) AS succ_lat,
SUM(CASE WHEN status = 'success' AND ttfb_ms IS NOT NULL THEN ttfb_ms ELSE 0 END) AS succ_ttfb_sum,
- SUM(CASE WHEN status = 'success' AND ttfb_ms IS NOT NULL THEN 1 ELSE 0 END) AS succ_ttfb_cnt
+ SUM(CASE WHEN status = 'success' AND ttfb_ms IS NOT NULL THEN 1 ELSE 0 END) AS succ_ttfb_cnt,
+ SUM(CASE WHEN ${IS_TIMEOUT_SQL} THEN 1 ELSE 0 END) AS timeouts,
+ SUM(CASE WHEN ${IS_TIMEOUT_SQL} THEN MIN(MAX(latency_ms, 0), ${TIMEOUT_LATENCY_CAP_MS}) ELSE 0 END) AS timeout_lat
FROM requests
WHERE created_at >= ?
GROUP BY platform, model_id, key_id, age_days
`).all(since) as Array<{
platform: string; model_id: string; key_id: number | null; age_days: number; total: number; successes: number;
succ_out: number; succ_lat: number; succ_ttfb_sum: number; succ_ttfb_cnt: number;
+ timeouts: number; timeout_lat: number;
}>;
// Accumulate decay-weighted sums per model AND per key.
- interface Acc { wSucc: number; wFail: number; wOut: number; wLat: number; wTtfbSum: number; wTtfbCnt: number }
- const emptyAcc = (): Acc => ({ wSucc: 0, wFail: 0, wOut: 0, wLat: 0, wTtfbSum: 0, wTtfbCnt: 0 });
+ //
+ // Timeouts (#619) land in the SAME latency/TTFB accumulators as successes,
+ // because that is what they are: time spent, nothing produced. Each one adds
+ // its capped wall-clock latency to the throughput denominator with zero
+ // output tokens, and that same figure as a first-byte sample — which is past
+ // TTFB_WORST_MS, so it scores no latency credit. Net effect: a model that
+ // times out constantly can no longer keep a stellar speed number just because
+ // its handful of successes were quick.
+ interface Acc { wSucc: number; wFail: number; wOut: number; wLat: number; wTtfbSum: number; wTtfbCnt: number; wTimeouts: number }
+ const emptyAcc = (): Acc => ({ wSucc: 0, wFail: 0, wOut: 0, wLat: 0, wTtfbSum: 0, wTtfbCnt: 0, wTimeouts: 0 });
const addBucket = (a: Acc, w: number, b: (typeof buckets)[number]): void => {
a.wSucc += w * b.successes;
a.wFail += w * (b.total - b.successes);
a.wOut += w * b.succ_out;
- a.wLat += w * b.succ_lat;
- a.wTtfbSum += w * b.succ_ttfb_sum;
- a.wTtfbCnt += w * b.succ_ttfb_cnt;
+ a.wLat += w * (b.succ_lat + b.timeout_lat);
+ a.wTtfbSum += w * (b.succ_ttfb_sum + b.timeout_lat);
+ a.wTtfbCnt += w * (b.succ_ttfb_cnt + b.timeouts);
+ a.wTimeouts += w * b.timeouts;
};
const acc = new Map<string, Acc>();
const keyAcc = new Map<string, Acc>();
@@ -436,12 +468,13 @@ export function refreshStatsCache(db: Db, force = false): void {
tokPerSec: a.wLat > 0 ? (a.wOut * 1000) / a.wLat : 0,
avgTtfbMs: a.wTtfbCnt > 0 ? a.wTtfbSum / a.wTtfbCnt : null,
monthlyUsedTokens: usageMap.get(key) ?? 0,
+ speedSamples: a.wSucc + a.wTimeouts,
});
}
// Models with month usage but no recent window data still need a headroom number.
for (const [key, used] of usageMap) {
if (!next.has(key)) {
- next.set(key, { successes: 0, failures: 0, tokPerSec: 0, avgTtfbMs: null, monthlyUsedTokens: used });
+ next.set(key, { successes: 0, failures: 0, tokPerSec: 0, avgTtfbMs: null, monthlyUsedTokens: used, speedSamples: 0 });
}
}
@@ -458,6 +491,78 @@ export function refreshStatsCache(db: Db, force = false): void {
statsCache = next;
keyStatsCache = nextKeys;
statsCacheTime = Date.now();
+
+ // Natural tail of a recompute: fold what we just measured back into
+ // models.speed_rank (#619). Never allowed to break routing — the caches above
+ // are already published, and a failed write just means the column keeps its
+ // previous value until the next pass.
+ if (Date.now() - speedRankWriteTime >= SPEED_RANK_WRITE_INTERVAL_MS) {
+ speedRankWriteTime = Date.now();
+ try {
+ writeObservedSpeedRanks(db);
+ } catch (e) {
+ console.error('Failed to write observed speed ranks:', e);
+ }
+ }
+}
+
+// ── Observed speed_rank writeback (#619) ────────────────────────────────────
+// models.speed_rank is the catalog's hand-assigned speed ordering and drives
+// the dashboard's sort-by-speed preset. It was only ever WRITTEN by the seed
+// migrations, catalog sync, and an explicit user override — never by anything
+// that had actually watched the model run, so a relay model that hangs on half
+// its calls kept whatever rank the catalog guessed for it forever.
+//
+// This folds the live speed axis back into the column, under three rules:
+// - a model needs SPEED_RANK_MIN_SAMPLES decay-weighted speed-bearing
+// requests (successes + timeouts) before we claim to know anything; below
+// that it keeps its catalog value;
+// - a user-set speed_rank override always wins — we skip those models
+// entirely rather than fight applyModelOverrides for the column;
+// - the UPDATE is guarded on the value actually changing, so a steady system
+// writes nothing at all.
+//
+// A catalog sync re-stamps speed_rank from the catalog; the next pass simply
+// re-derives the observed value, which is why this is a periodic write rather
+// than a one-shot migration.
+export const SPEED_RANK_MIN_SAMPLES = 20;
+const SPEED_RANK_WRITE_INTERVAL_MS = 10 * 60 * 1000;
+let speedRankWriteTime = 0;
+
+/** Test hook: forget when the last writeback ran so the next refresh does one. */
+export function resetSpeedRankWriteback(): void {
+ speedRankWriteTime = 0;
+}
+
+/**
+ * Write an observed speed rank for every model with enough recent samples and
+ * no user-set speed_rank override. Returns how many rows actually changed.
+ * Reads the stats cache as-is — callers refresh it first (refreshStatsCache
+ * calls this from its own tail).
+ */
+export function writeObservedSpeedRanks(db: Db): number {
+ if (!statsCache || statsCache.size === 0) return 0;
+
+ const pinned = modelsWithOverriddenField(db, 'speedRank');
+ const rows = db.prepare('SELECT id, platform, model_id, speed_rank FROM models')
+ .all() as { id: number; platform: string; model_id: string; speed_rank: number }[];
+
+ const update = db.prepare('UPDATE models SET speed_rank = ? WHERE id = ?');
+ let written = 0;
+ const tx = db.transaction(() => {
+ for (const row of rows) {
+ const key = `${row.platform}:${row.model_id}`;
+ if (pinned.has(key)) continue;
+ const stats = statsCache!.get(key);
+ if (!stats || stats.speedSamples < SPEED_RANK_MIN_SAMPLES) continue;
+ const rank = observedSpeedRank(speedScore(stats.tokPerSec, stats.avgTtfbMs));
+ if (rank === row.speed_rank) continue;
+ update.run(rank, row.id);
+ written++;
+ }
+ });
+ tx();
+ return written;
}
// Composite intelligence: size_label is the cross-provider capability tier
diff --git a/server/src/services/scoring.ts b/server/src/services/scoring.ts
index d713176b..4520604c 100644
--- a/server/src/services/scoring.ts
+++ b/server/src/services/scoring.ts
@@ -106,6 +106,41 @@ export function speedScore(tokPerSec: number, ttfbMs: number | null): number {
return THROUGHPUT_WEIGHT * tp + TTFB_WEIGHT * ttfbScore(ttfbMs);
}
+// ── Speed: what a timeout costs (#619) ──────────────────────────────────────
+// A request that times out IS the model being slow — the reporter's exact
+// complaint: a relay model that hangs on half its calls loses reliability but
+// used to keep a pristine speed number, because the analytics query fed the
+// speed axis from `status = 'success'` rows ONLY. So a timeout now contributes
+// to the speed window as what it actually was: its wall-clock latency with ZERO
+// output tokens (dragging throughput down) and that same latency as its
+// first-byte time (which lands past TTFB_WORST_MS, i.e. no latency credit).
+//
+// Capped, because `latency_ms` on a timed-out row is unbounded — a provider
+// that holds a socket open for twenty minutes, or a per-platform
+// PROVIDER_TIMEOUT override in the hundreds of seconds, would otherwise let a
+// single hang dominate the whole 7-day window and flatten the axis for every
+// model on that platform. Two minutes is above the built-in per-attempt HTTP
+// deadline and the 90s stream-stall watchdog, so a normal timeout is counted in
+// full and only genuine outliers are clipped.
+export const TIMEOUT_LATENCY_CAP_MS = 120_000;
+
+// ── Observed speed rank ─────────────────────────────────────────────────────
+// `models.speed_rank` is the catalog's hand-assigned "how fast is this model"
+// ordering (1 = fastest; the shipped catalog spans 1..11) and it drives the
+// dashboard's sort-by-speed preset. Until now nothing ever wrote it at runtime,
+// so it stayed frozen at whatever the seed said no matter how the model
+// actually behaved (#619). observedSpeedRank projects the live [0,1] speed axis
+// back onto that same low-is-fast scale so an observed rank is directly
+// comparable with a catalog one.
+export const OBSERVED_SPEED_RANK_BEST = 1;
+export const OBSERVED_SPEED_RANK_WORST = 10;
+
+export function observedSpeedRank(speed: number): number {
+ const s = Math.min(1, Math.max(0, Number.isFinite(speed) ? speed : 0));
+ const span = OBSERVED_SPEED_RANK_WORST - OBSERVED_SPEED_RANK_BEST;
+ return OBSERVED_SPEED_RANK_BEST + Math.round((1 - s) * span);
+}
+
// ── Intelligence ────────────────────────────────────────────────────────────
// Caller supplies a composite (tier-first, rank-as-tiebreaker — see router) and
// the min/max across the enabled chain. We min-max normalize to [0,1], 1 = best.