Skip to content

7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.

TypeScript

Demote models that are burning through their rpd/tpd windows (#1001)

Tashfeen45a05c709a72
diff --git a/server/src/__tests__/services/router-window-headroom.test.ts b/server/src/__tests__/services/router-window-headroom.test.ts
new file mode 100644
index 00000000..fe76ad32
--- /dev/null
+++ b/server/src/__tests__/services/router-window-headroom.test.ts
@@ -0,0 +1,188 @@
+import { describe, it, expect, beforeAll, beforeEach } from 'vitest';
+import { initDb, getDb } from '../../db/index.js';
+import { encrypt } from '../../lib/crypto.js';
+import { getRoutingScores, refreshStatsCache, setHeadroomThresholds } from '../../services/router.js';
+import { invalidateWindowUsage, modelWindowUsedFraction } from '../../services/ratelimit.js';
+
+// #899: the headroom guardrail used to have an opinion only about models that
+// declare a monthly_token_budget. A model capped by rpd/tpd was binary-gated —
+// full score until the request that finally 429s it. These tests pin the graded
+// behaviour: demotion as the window fills, no opinion while it is empty,
+// automatic recovery when the window rolls, and operator-tunable thresholds.
+describe('rate-window headroom guardrail (#899)', () => {
+ const DAY_MS = 24 * 60 * 60 * 1000;
+ let busy: { id: number; modelId: string };
+ let idle: { id: number; modelId: string };
+
+ beforeAll(() => {
+ process.env.ENCRYPTION_KEY = '0'.repeat(64);
+ initDb(':memory:');
+ const db = getDb();
+
+ const rows = db.prepare(
+ "SELECT id, model_id FROM models WHERE platform = 'groq' ORDER BY id LIMIT 2"
+ ).all() as { id: number; model_id: string }[];
+ expect(rows.length).toBe(2);
+ [busy, idle] = rows.map(r => ({ id: r.id, modelId: r.model_id }));
+
+ // Two peers that differ ONLY in window utilization: same tier and rank so
+ // the intelligence axis ties, no monthly budget so that guardrail stays out
+ // of it, and the same daily request cap.
+ for (const m of rows) {
+ db.prepare(`
+ UPDATE models
+ SET monthly_token_budget = '', rpd_limit = 100, rpm_limit = NULL,
+ tpm_limit = NULL, tpd_limit = NULL,
+ size_label = 'medium', intelligence_rank = 10
+ WHERE id = ?
+ `).run(m.id);
+ }
+ // Every other model out of the chain, so the two peers are the whole
+ // ranking and the intelligence normalization cannot reorder them.
+ db.prepare("UPDATE models SET enabled = 0 WHERE id NOT IN (?, ?)").run(busy.id, idle.id);
+
+ const { encrypted, iv, authTag } = encrypt('groq-key');
+ db.prepare(`
+ INSERT INTO api_keys (platform, label, encrypted_key, iv, auth_tag, status, enabled)
+ VALUES ('groq', 'k0', ?, ?, ?, 'healthy', 1)
+ `).run(encrypted, iv, authTag);
+ });
+
+ beforeEach(() => {
+ getDb().prepare('DELETE FROM rate_limit_usage').run();
+ setHeadroomThresholds(null, null);
+ invalidateWindowUsage();
+ });
+
+ function keyId(): number {
+ return (getDb().prepare("SELECT id FROM api_keys WHERE platform = 'groq'").get() as { id: number }).id;
+ }
+
+ /** n recorded requests for `modelId`, aged `ageMs` into the past. */
+ function recordRequests(modelId: string, n: number, ageMs = 0) {
+ const db = getDb();
+ const stmt = db.prepare(`
+ INSERT INTO rate_limit_usage (platform, model_id, key_id, kind, tokens, created_at_ms)
+ VALUES ('groq', ?, ?, 'request', 0, ?)
+ `);
+ const at = Date.now() - ageMs;
+ for (let i = 0; i < n; i++) stmt.run(modelId, keyId(), at);
+ invalidateWindowUsage();
+ }
+
+ function headroomOf(modelDbId: number): number {
+ refreshStatsCache(getDb(), true);
+ const { scores } = getRoutingScores();
+ const row = scores.find(s => s.modelDbId === modelDbId);
+ if (!row) throw new Error('model missing from routing scores');
+ return row.headroom;
+ }
+
+ function scoreOf(modelDbId: number): number {
+ refreshStatsCache(getDb(), true);
+ const { scores } = getRoutingScores();
+ const row = scores.find(s => s.modelDbId === modelDbId);
+ if (!row) throw new Error('model missing from routing scores');
+ return row.score;
+ }
+
+ it('leaves an untouched model alone (factor 1)', () => {
+ expect(headroomOf(busy.id)).toBe(1);
+ expect(headroomOf(idle.id)).toBe(1);
+ });
+
+ it('still has no opinion at low utilization, below the ramp', () => {
+ recordRequests(busy.modelId, 50); // 50% of RPD — 50% left, ramp starts at 20%
+ expect(headroomOf(busy.id)).toBe(1);
+ });
+
+ it('demotes a model at 95% of its RPD below an idle peer', () => {
+ recordRequests(busy.modelId, 95);
+ const busyHeadroom = headroomOf(busy.id);
+ expect(busyHeadroom).toBeLessThan(1);
+ // 5% remaining, ramp 20%, floor 0.1 → 0.1 + 0.9·(0.05/0.2) = 0.325
+ expect(busyHeadroom).toBeCloseTo(0.325, 5);
+ expect(headroomOf(idle.id)).toBe(1);
+ expect(scoreOf(busy.id)).toBeLessThan(scoreOf(idle.id));
+
+ // And the ranking actually flips — the busy model is no longer first.
+ const { scores } = getRoutingScores();
+ expect(scores[0].modelDbId).toBe(idle.id);
+ });
+
+ it('recovers on its own once the day window rolls past the usage', () => {
+ recordRequests(busy.modelId, 95);
+ expect(headroomOf(busy.id)).toBeLessThan(1);
+
+ // Age the same rows out of the sliding day. Nothing resets a counter — the
+ // window simply stops counting them.
+ getDb().prepare(
+ "UPDATE rate_limit_usage SET created_at_ms = ? WHERE model_id = ?"
+ ).run(Date.now() - DAY_MS - 60_000, busy.modelId);
+ invalidateWindowUsage();
+
+ expect(headroomOf(busy.id)).toBe(1);
+ expect(scoreOf(busy.id)).toBeCloseTo(scoreOf(idle.id), 10);
+ });
+
+ it('honours the tunable thresholds from the settings API', () => {
+ recordRequests(busy.modelId, 60); // 40% left — outside the 20% default ramp
+ expect(headroomOf(busy.id)).toBe(1);
+
+ // Operator asks to start protecting at 50% remaining with a 0.3 floor.
+ setHeadroomThresholds(0.5, 0.3);
+ invalidateWindowUsage();
+ // 0.3 + 0.7·(0.4/0.5) = 0.86
+ expect(headroomOf(busy.id)).toBeCloseTo(0.86, 5);
+ expect(headroomOf(idle.id)).toBe(1);
+ });
+
+ it('takes the busiest window across rpd and tpd', () => {
+ const db = getDb();
+ db.prepare("UPDATE models SET tpd_limit = 1000 WHERE id = ?").run(busy.id);
+ recordRequests(busy.modelId, 10); // only 10% of RPD
+ db.prepare(`
+ INSERT INTO rate_limit_usage (platform, model_id, key_id, kind, tokens, created_at_ms)
+ VALUES ('groq', ?, ?, 'tokens', 900, ?)
+ `).run(busy.modelId, keyId(), Date.now());
+ invalidateWindowUsage();
+
+ // 90% of TPD binds, not 10% of RPD.
+ expect(headroomOf(busy.id)).toBeCloseTo(0.55, 5); // 0.1 + 0.9·(0.1/0.2)
+ db.prepare("UPDATE models SET tpd_limit = NULL WHERE id = ?").run(busy.id);
+ });
+
+ it('follows the eligible key with the most headroom, not the worst one', () => {
+ const db = getDb();
+ const second = encrypt('groq-key-2');
+ db.prepare(`
+ INSERT INTO api_keys (platform, label, encrypted_key, iv, auth_tag, status, enabled)
+ VALUES ('groq', 'k1', ?, ?, ?, 'healthy', 1)
+ `).run(second.encrypted, second.iv, second.authTag);
+ const ids = (db.prepare("SELECT id FROM api_keys WHERE platform = 'groq' ORDER BY id").all() as { id: number }[]).map(r => r.id);
+
+ const stmt = db.prepare(`
+ INSERT INTO rate_limit_usage (platform, model_id, key_id, kind, tokens, created_at_ms)
+ VALUES ('groq', ?, ?, 'request', 0, ?)
+ `);
+ for (let i = 0; i < 99; i++) stmt.run(busy.modelId, ids[0], Date.now());
+ invalidateWindowUsage();
+
+ // One key is all but exhausted; the other is untouched, and the router will
+ // land on it, so the model is not under any real pressure.
+ expect(headroomOf(busy.id)).toBe(1);
+ db.prepare('DELETE FROM api_keys WHERE id = ?').run(ids[1]);
+ invalidateWindowUsage();
+ });
+
+ it('has no opinion for a model that declares no window limits', () => {
+ getDb().prepare("UPDATE models SET rpd_limit = NULL WHERE id = ?").run(busy.id);
+ recordRequests(busy.modelId, 500);
+ expect(modelWindowUsedFraction(
+ { platform: 'groq', modelId: busy.modelId, keyId: null },
+ { rpm: null, rpd: null, tpm: null, tpd: null },
+ )).toBeNull();
+ expect(headroomOf(busy.id)).toBe(1);
+ getDb().prepare("UPDATE models SET rpd_limit = 100 WHERE id = ?").run(busy.id);
+ });
+});
diff --git a/server/src/__tests__/services/scoring.test.ts b/server/src/__tests__/services/scoring.test.ts
index 63d1ebf7..652aabf8 100644
--- a/server/src/__tests__/services/scoring.test.ts
+++ b/server/src/__tests__/services/scoring.test.ts
@@ -1,7 +1,7 @@
import { describe, it, expect } from 'vitest';
import {
BANDIT_PRESETS, combineScore, speedScore, intelligenceScore, intelligenceComposite,
- headroomFactor, rateLimitFactor, sampleBeta, reliabilityPosterior,
+ headroomFactor, rateWindowHeadroomFactor, rateLimitFactor, sampleBeta, reliabilityPosterior,
expectedReliability, SPEED_PRIOR, HEADROOM_FLOOR,
isPeakHours, peakAdjustedWeights, hourInTimezone, isValidPeakHour, isValidTimezone,
isPeakExemptStrategy, DEFAULT_PEAK_HOURS, PEAK_SPEED_TO_RELIABILITY,
@@ -149,6 +149,36 @@ describe('scoring: guardrails', () => {
expect(headroomFactor(123, 0)).toBe(1);
});
+ it('rate-window headroom rides the same ramp as the monthly one (#899)', () => {
+ expect(rateWindowHeadroomFactor(0)).toBe(1); // idle
+ expect(rateWindowHeadroomFactor(0.5)).toBe(1); // 50% left, ramp starts at 20%
+ // Exactly at the ramp start: 1 - 0.8 lands a float ulp under 0.2, so this
+ // is the very top of the ramp rather than the flat part. Same arithmetic
+ // the monthly guardrail has always done.
+ expect(rateWindowHeadroomFactor(0.8)).toBeCloseTo(1, 9);
+ // 5% of the window left → 0.1 + 0.9·(0.05/0.2)
+ expect(rateWindowHeadroomFactor(0.95)).toBeCloseTo(0.325, 5);
+ expect(rateWindowHeadroomFactor(1)).toBeCloseTo(HEADROOM_FLOOR, 5);
+ // Over-consumption (a provider counted more than we did) cannot go below
+ // the floor.
+ expect(rateWindowHeadroomFactor(1.4)).toBeCloseTo(HEADROOM_FLOOR, 5);
+ // The two guardrails agree wherever the remaining fraction agrees.
+ expect(rateWindowHeadroomFactor(0.95)).toBeCloseTo(headroomFactor(950, 1000), 10);
+ });
+
+ it('rate-window headroom takes the same tunable thresholds (#899)', () => {
+ const opts = { rampStart: 0.5, floor: 0.3 };
+ expect(rateWindowHeadroomFactor(0.4, opts)).toBe(1); // 60% left
+ expect(rateWindowHeadroomFactor(0.6, opts)).toBeCloseTo(0.86, 5); // 0.3 + 0.7·(0.4/0.5)
+ expect(rateWindowHeadroomFactor(0.6, { rampStart: 5 }))
+ .toBe(rateWindowHeadroomFactor(0.6)); // out of range → defaults
+ });
+
+ it('no measurable window yields no opinion (factor 1)', () => {
+ expect(rateWindowHeadroomFactor(null)).toBe(1);
+ expect(rateWindowHeadroomFactor(NaN)).toBe(1);
+ });
+
it('rate-limit factor is 1 at no penalty and damped but non-zero at max', () => {
expect(rateLimitFactor(0)).toBe(1);
expect(rateLimitFactor(10)).toBeCloseTo(0.4, 5);
diff --git a/server/src/services/ratelimit.ts b/server/src/services/ratelimit.ts
index dbb28eb8..747432ae 100644
--- a/server/src/services/ratelimit.ts
+++ b/server/src/services/ratelimit.ts
@@ -2,6 +2,7 @@
import { getDb } from '../db/index.js';
import { isLoopbackOrPrivateUrl } from '../lib/url-guard.js';
+import { parseModelScope, scopeAllows } from '../lib/model-scope.js';
interface Window {
timestamps: number[];
@@ -354,6 +355,199 @@ export function canUseTokens(
return true;
}
+// ── Window utilization for the routing guardrail (#899) ──────────────────────
+// Everything above answers a yes/no question on the hot path: may THIS key
+// serve THIS request. The router also needs a graded one — "how much of its
+// window quota has this model already burned" — so it can steer traffic away
+// from a model at 95% of its daily cap before that cap actually rejects
+// anything. See rateWindowHeadroomFactor in scoring.ts for what is done with
+// the number.
+//
+// Cost is the whole design constraint: this runs per chain entry per request,
+// and the naive shape is four counts per model × key. So it is one grouped scan
+// of rate_limit_usage plus one read of api_keys, memoised for a few seconds and
+// shared by every entry of every chain — the same trade getKeyQuotaHeadroom
+// makes in provider-quota.ts. Per-entry work is then pure in-memory arithmetic
+// over the model's own limit columns, which the router already has in hand.
+
+export interface WindowLimits {
+ rpm: number | null;
+ rpd: number | null;
+ tpm: number | null;
+ tpd: number | null;
+}
+
+interface KeyWindowUsage { rpm: number; rpd: number; tpm: number; tpd: number }
+const NO_WINDOW_USAGE: KeyWindowUsage = { rpm: 0, rpd: 0, tpm: 0, tpd: 0 };
+
+interface EligibleKey { id: number; baseUrl: string | null; scope: Set<string> | null }
+
+interface WindowUsageSnapshot {
+ db: unknown;
+ at: number;
+ usage: Map<string, KeyWindowUsage>;
+ keysByPlatform: Map<string, EligibleKey[]>;
+ baseUrlByKeyId: Map<number, string | null>;
+}
+
+/** Utilization moves on the timescale of a rate-limit window, not a request, so
+ * a few seconds of staleness is invisible to a guardrail while the query count
+ * drops to ~one per burst. The hard gates above are unaffected — they still
+ * read live counts, so nothing here can let a request through a real limit. */
+const WINDOW_USAGE_TTL_MS = 5_000;
+
+// The Db handle is part of the cache identity, so reconnecting (tests, a
+// restore) invalidates the snapshot rather than serving another database's
+// numbers.
+let windowUsageSnapshot: WindowUsageSnapshot | null = null;
+
+function usageCacheKey(platform: string, modelId: string, keyId: number): string {
+ return `${platform}\u0000${modelId}\u0000${keyId}`;
+}
+
+function buildWindowUsageSnapshot(now: number): WindowUsageSnapshot | null {
+ return withDb(db => {
+ // One grouped scan replaces four counts per model × key. Same windows the
+ // gates above enforce: a sliding minute for rpm/tpm, a sliding day for
+ // rpd/tpd. Rows older than a day are pruned on write, but the WHERE keeps
+ // this correct even when a prune has not run yet.
+ const rows = db.prepare(`
+ SELECT platform, model_id, key_id,
+ SUM(CASE WHEN kind = 'request' AND created_at_ms > ? THEN 1 ELSE 0 END) AS rpm_used,
+ SUM(CASE WHEN kind = 'request' THEN 1 ELSE 0 END) AS rpd_used,
+ SUM(CASE WHEN kind = 'tokens' AND created_at_ms > ? THEN tokens ELSE 0 END) AS tpm_used,
+ SUM(CASE WHEN kind = 'tokens' THEN tokens ELSE 0 END) AS tpd_used
+ FROM rate_limit_usage
+ WHERE created_at_ms > ?
+ GROUP BY platform, model_id, key_id
+ `).all(now - MINUTE, now - MINUTE, now - DAY) as Array<{
+ platform: string; model_id: string; key_id: number;
+ rpm_used: number; rpd_used: number; tpm_used: number; tpd_used: number;
+ }>;
+
+ const usage = new Map<string, KeyWindowUsage>();
+ for (const r of rows) {
+ usage.set(usageCacheKey(r.platform, r.model_id, r.key_id), {
+ rpm: r.rpm_used, rpd: r.rpd_used, tpm: r.tpm_used, tpd: r.tpd_used,
+ });
+ }
+
+ // Mirror the router's key eligibility (selectKeyForModel): enabled +
+ // healthy/unknown, and the model scope must admit the model. The custom
+ // endpoint check needs base_url for every row, disabled ones included, so
+ // that map is built from the unfiltered read.
+ const keyRows = db.prepare(
+ 'SELECT id, platform, base_url, model_scope_json, enabled, status FROM api_keys'
+ ).all() as Array<{
+ id: number; platform: string; base_url: string | null;
+ model_scope_json: string | null; enabled: number; status: string;
+ }>;
+
+ const baseUrlByKeyId = new Map<number, string | null>();
+ const keysByPlatform = new Map<string, EligibleKey[]>();
+ for (const k of keyRows) {
+ baseUrlByKeyId.set(k.id, k.base_url);
+ if (k.enabled !== 1 || (k.status !== 'healthy' && k.status !== 'unknown')) continue;
+ const list = keysByPlatform.get(k.platform) ?? [];
+ list.push({ id: k.id, baseUrl: k.base_url, scope: parseModelScope(k.model_scope_json) });
+ keysByPlatform.set(k.platform, list);
+ }
+
+ return { db, at: now, usage, keysByPlatform, baseUrlByKeyId };
+ }) ?? null;
+}
+
+function getWindowUsageSnapshot(now: number): WindowUsageSnapshot | null {
+ let db;
+ try {
+ db = getDb();
+ } catch {
+ return null;
+ }
+ const cached = windowUsageSnapshot;
+ if (cached && cached.db === db && now - cached.at < WINDOW_USAGE_TTL_MS && now >= cached.at) {
+ return cached;
+ }
+ const built = buildWindowUsageSnapshot(now);
+ windowUsageSnapshot = built;
+ return built;
+}
+
+/** Drop the memoised window snapshot. Tests use it to make a write visible
+ * immediately; production relies on the TTL. */
+export function invalidateWindowUsage(): void {
+ windowUsageSnapshot = null;
+}
+
+/** The busiest window of one key: the ratio that would reject its next request
+ * first. A key metered on several windows takes the WORST of them, because the
+ * binding constraint is what 429s. */
+function keyWindowPressure(usage: KeyWindowUsage, limits: WindowLimits): number {
+ let worst = 0;
+ const ratios: Array<[number, number | null]> = [
+ [usage.rpm, limits.rpm],
+ [usage.rpd, limits.rpd],
+ [usage.tpm, limits.tpm],
+ [usage.tpd, limits.tpd],
+ ];
+ for (const [used, limit] of ratios) {
+ if (limit == null || limit <= 0) continue;
+ const r = used / limit;
+ if (r > worst) worst = r;
+ }
+ return worst;
+}
+
+/**
+ * Share of its binding rate-limit window a model has already consumed, as a
+ * 0..1 fraction (0 idle, 1 exhausted). null = no opinion: the model declares no
+ * window limits, has no routable key to measure, or the database is unreachable.
+ *
+ * Which key's numbers to report matters, and the answer is the same one the
+ * dashboard's usage badge settled on (#921): the guardrail asks "how close is
+ * this model to being unroutable", so it must follow the key the router would
+ * pick NEXT, not the worst key on the account. A platform with one exhausted
+ * key and one idle key routes perfectly well, so we take the ELIGIBLE key with
+ * the MOST headroom.
+ *
+ * In-flight leases are deliberately not counted. They exist to close the
+ * check-then-act race on the hard gates; this is a steering signal averaged over
+ * a whole window, and folding a per-request quantity into a cached snapshot
+ * would buy noise, not accuracy.
+ */
+export function modelWindowUsedFraction(
+ model: { platform: string; modelId: string; keyId?: number | null },
+ limits: WindowLimits,
+ now = Date.now(),
+): number | null {
+ if (limits.rpm == null && limits.rpd == null && limits.tpm == null && limits.tpd == null) return null;
+ const snap = getWindowUsageSnapshot(now);
+ if (!snap) return null;
+
+ const pool = snap.keysByPlatform.get(model.platform);
+ if (!pool || pool.length === 0) return null; // nothing routable → a number would be fiction
+
+ // A custom model belongs to one endpoint; only that endpoint's credentials can
+ // serve it. Legacy rows (key_id NULL) keep the any-key match.
+ const endpointBaseUrl = model.platform === 'custom' && model.keyId != null
+ ? snap.baseUrlByKeyId.get(model.keyId) ?? null
+ : null;
+
+ let best: number | null = null;
+ for (const k of pool) {
+ if (!scopeAllows(k.scope, model.modelId)) continue;
+ if (model.platform === 'custom' && model.keyId != null) {
+ if (endpointBaseUrl == null ? k.id !== model.keyId : k.baseUrl !== endpointBaseUrl) continue;
+ }
+ const usage = snap.usage.get(usageCacheKey(model.platform, model.modelId, k.id)) ?? NO_WINDOW_USAGE;
+ const pressure = keyWindowPressure(usage, limits);
+ if (best === null || pressure < best) best = pressure;
+ if (best === 0) break; // an untouched key is as good as it gets
+ }
+ if (best === null) return null; // every key scoped away from this model
+ return Math.min(1, best);
+}
+
// ── Provider-wide daily request caps (#162) ──
// Some providers enforce one daily REQUEST quota across the WHOLE account,
// shared by every model — not per model. OpenRouter's free tier is the classic
diff --git a/server/src/services/router.ts b/server/src/services/router.ts
index c68873fb..cf73540e 100644
--- a/server/src/services/router.ts
+++ b/server/src/services/router.ts
@@ -13,12 +13,14 @@ import {
acquireLease,
releaseLease,
getSoonestCooldownExpiry,
+ modelWindowUsedFraction,
} from './ratelimit.js';
import {
BANDIT_PRESETS, DEFAULT_STRATEGY, type RoutingStrategy, type RoutingWeights,
type KeySelectionStrategy,
reliabilityPosterior, expectedReliability, sampleBeta,
- speedScore, intelligenceScore, intelligenceComposite, headroomFactor, rateLimitFactor, combineScore,
+ speedScore, intelligenceScore, intelligenceComposite, headroomFactor, rateWindowHeadroomFactor,
+ rateLimitFactor, combineScore,
peakAdjustedWeights, isValidPeakHour, isValidTimezone,
DEFAULT_PEAK_HOURS, type PeakHoursConfig,
observedSpeedRank, TIMEOUT_LATENCY_CAP_MS,
@@ -974,7 +976,27 @@ function scoreChainEntry(
// ONCE per chain by the caller, not per entry — getSetting is an uncached
// SELECT, so reading it here cost two extra SQLite round-trips per model per
// request, the same reason `weights` and `keyCounts` are hoisted.
- const headroom = headroomFactor(stats?.monthlyUsedTokens ?? 0, budget, headroomCfg);
+ const monthlyHeadroom = headroomFactor(stats?.monthlyUsedTokens ?? 0, budget, headroomCfg);
+
+ // The same guardrail, driven by live rpm/rpd/tpm/tpd utilization instead of
+ // the monthly budget (#899). Most free tiers publish a daily request or token
+ // cap and no monthly figure at all, so without this a model sits at score #1
+ // until the request that finally 429s it. Reads a snapshot memoised inside
+ // ratelimit.ts, so this costs no query per model per request.
+ const windowHeadroom = rateWindowHeadroomFactor(
+ modelWindowUsedFraction(
+ { platform: entry.platform, modelId: entry.model_id, keyId: entry.key_id },
+ { rpm: entry.rpm_limit, rpd: entry.rpd_limit, tpm: entry.tpm_limit, tpd: entry.tpd_limit },
+ ),
+ headroomCfg,
+ );
+
+ // The WORSE of the two, not their product: both express the same "this model
+ // is close to burning out" opinion on different meters, and multiplying them
+ // would push a model that is low on both to floor², below the floor the
+ // operator configured. Taking the binding constraint keeps the floor meaning
+ // what it says — the same rule getKeyQuotaHeadroom applies across metrics.
+ const headroom = Math.min(monthlyHeadroom, windowHeadroom);
const rl = rateLimitFactor(getPenalty(entry.model_db_id));
// Per-model env overrides (#738) scale the final score so a slow or
diff --git a/server/src/services/scoring.ts b/server/src/services/scoring.ts
index 7cbaecb2..8b0324ed 100644
--- a/server/src/services/scoring.ts
+++ b/server/src/services/scoring.ts
@@ -365,14 +365,47 @@ export interface HeadroomThresholds {
export function headroomFactor(usedTokens: number, budgetTokens: number, opts?: HeadroomThresholds): number {
if (!budgetTokens || budgetTokens <= 0) return 1; // unknown budget → no opinion
+ return headroomRamp(1 - usedTokens / budgetTokens, opts);
+}
+
+/** The shared ramp both headroom guardrails ride: 1 while `remaining` (a 0..1
+ * fraction of the quota still available) is comfortable, then linear down to
+ * `floor` as it reaches zero. Factored out so the monthly-budget guardrail and
+ * the rate-window one below cannot drift apart — and so a single pair of
+ * operator-tuned thresholds governs both (#899). */
+function headroomRamp(remainingRaw: number, opts?: HeadroomThresholds): number {
const rampStart = clampUnit(opts?.rampStart, HEADROOM_RAMP_START);
const floor = clampUnit(opts?.floor, HEADROOM_FLOOR);
- const remaining = Math.max(0, 1 - usedTokens / budgetTokens);
+ const remaining = Math.max(0, Math.min(1, remainingRaw));
if (remaining >= rampStart) return 1;
// Linear from (0 remaining → floor) to (rampStart remaining → 1).
return floor + (1 - floor) * (remaining / rampStart);
}
+// ── Guardrail: rate-window headroom (#899) ──────────────────────────────────
+// The monthly-budget guardrail above only has an opinion about models that
+// declare a `monthly_token_budget`. The far more common free-tier shape is a
+// per-day request or token cap (rpd/tpd, plus the per-minute rpm/tpm), and
+// those were purely BINARY: canMakeRequest/canUseTokens reject at 100% and the
+// router happily keeps a model pinned at #1 until the very request that
+// exhausts it, then eats a 429 and falls through. That is exactly the
+// "deprioritize at 82% utilization, prefer the idle peer" behavior the issue
+// asks for.
+//
+// So: same ramp, same tunable thresholds, driven by live window utilization
+// instead of monthly tokens. `usedFraction` is the share of the binding window
+// limit already consumed (0 = idle, 1 = exhausted); null means the model
+// declares no window limits, or has no routable key to measure — in both cases
+// the guardrail has no opinion and returns 1.
+//
+// Recovery is automatic and needs no bookkeeping: the windows are sliding, so a
+// demoted model's utilization falls on its own as the minute or the day rolls
+// past, and the factor climbs straight back to 1.
+export function rateWindowHeadroomFactor(usedFraction: number | null, opts?: HeadroomThresholds): number {
+ if (usedFraction === null || !Number.isFinite(usedFraction)) return 1; // unknown → no opinion
+ return headroomRamp(1 - usedFraction, opts);
+}
+
// Out-of-range / non-finite operator input falls back to the default rather
// than silently clamping to a legal-but-unintended value (#899).
function clampUnit(n: number | undefined, fallback: number): number {