7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.
TypeScript
Demote models that are burning through their rpd/tpd windows (#1001)
Tashfeen45a05c709a72
diff --git a/server/src/__tests__/services/router-window-headroom.test.ts b/server/src/__tests__/services/router-window-headroom.test.tsnew file mode 100644index 00000000..fe76ad32--- /dev/null+++ b/server/src/__tests__/services/router-window-headroom.test.ts@@ -0,0 +1,188 @@+import { describe, it, expect, beforeAll, beforeEach } from 'vitest';+import { initDb, getDb } from '../../db/index.js';+import { encrypt } from '../../lib/crypto.js';+import { getRoutingScores, refreshStatsCache, setHeadroomThresholds } from '../../services/router.js';+import { invalidateWindowUsage, modelWindowUsedFraction } from '../../services/ratelimit.js';++// #899: the headroom guardrail used to have an opinion only about models that+// declare a monthly_token_budget. A model capped by rpd/tpd was binary-gated —+// full score until the request that finally 429s it. These tests pin the graded+// behaviour: demotion as the window fills, no opinion while it is empty,+// automatic recovery when the window rolls, and operator-tunable thresholds.+describe('rate-window headroom guardrail (#899)', () => {+ const DAY_MS = 24 * 60 * 60 * 1000;+ let busy: { id: number; modelId: string };+ let idle: { id: number; modelId: string };++ beforeAll(() => {+ process.env.ENCRYPTION_KEY = '0'.repeat(64);+ initDb(':memory:');+ const db = getDb();++ const rows = db.prepare(+ "SELECT id, model_id FROM models WHERE platform = 'groq' ORDER BY id LIMIT 2"+ ).all() as { id: number; model_id: string }[];+ expect(rows.length).toBe(2);+ [busy, idle] = rows.map(r => ({ id: r.id, modelId: r.model_id }));++ // Two peers that differ ONLY in window utilization: same tier and rank so+ // the intelligence axis ties, no monthly budget so that guardrail stays out+ // of it, and the same daily request cap.+ for (const m of rows) {+ db.prepare(`+ UPDATE models+ SET monthly_token_budget = '', rpd_limit = 100, rpm_limit = NULL,+ tpm_limit = NULL, tpd_limit = NULL,+ size_label = 'medium', intelligence_rank = 10+ WHERE id = ?+ `).run(m.id);+ }+ // Every other model out of the chain, so the two peers are the whole+ // ranking and the intelligence normalization cannot reorder them.+ db.prepare("UPDATE models SET enabled = 0 WHERE id NOT IN (?, ?)").run(busy.id, idle.id);++ const { encrypted, iv, authTag } = encrypt('groq-key');+ db.prepare(`+ INSERT INTO api_keys (platform, label, encrypted_key, iv, auth_tag, status, enabled)+ VALUES ('groq', 'k0', ?, ?, ?, 'healthy', 1)+ `).run(encrypted, iv, authTag);+ });++ beforeEach(() => {+ getDb().prepare('DELETE FROM rate_limit_usage').run();+ setHeadroomThresholds(null, null);+ invalidateWindowUsage();+ });++ function keyId(): number {+ return (getDb().prepare("SELECT id FROM api_keys WHERE platform = 'groq'").get() as { id: number }).id;+ }++ /** n recorded requests for `modelId`, aged `ageMs` into the past. */+ function recordRequests(modelId: string, n: number, ageMs = 0) {+ const db = getDb();+ const stmt = db.prepare(`+ INSERT INTO rate_limit_usage (platform, model_id, key_id, kind, tokens, created_at_ms)+ VALUES ('groq', ?, ?, 'request', 0, ?)+ `);+ const at = Date.now() - ageMs;+ for (let i = 0; i < n; i++) stmt.run(modelId, keyId(), at);+ invalidateWindowUsage();+ }++ function headroomOf(modelDbId: number): number {+ refreshStatsCache(getDb(), true);+ const { scores } = getRoutingScores();+ const row = scores.find(s => s.modelDbId === modelDbId);+ if (!row) throw new Error('model missing from routing scores');+ return row.headroom;+ }++ function scoreOf(modelDbId: number): number {+ refreshStatsCache(getDb(), true);+ const { scores } = getRoutingScores();+ const row = scores.find(s => s.modelDbId === modelDbId);+ if (!row) throw new Error('model missing from routing scores');+ return row.score;+ }++ it('leaves an untouched model alone (factor 1)', () => {+ expect(headroomOf(busy.id)).toBe(1);+ expect(headroomOf(idle.id)).toBe(1);+ });++ it('still has no opinion at low utilization, below the ramp', () => {+ recordRequests(busy.modelId, 50); // 50% of RPD — 50% left, ramp starts at 20%+ expect(headroomOf(busy.id)).toBe(1);+ });++ it('demotes a model at 95% of its RPD below an idle peer', () => {+ recordRequests(busy.modelId, 95);+ const busyHeadroom = headroomOf(busy.id);+ expect(busyHeadroom).toBeLessThan(1);+ // 5% remaining, ramp 20%, floor 0.1 → 0.1 + 0.9·(0.05/0.2) = 0.325+ expect(busyHeadroom).toBeCloseTo(0.325, 5);+ expect(headroomOf(idle.id)).toBe(1);+ expect(scoreOf(busy.id)).toBeLessThan(scoreOf(idle.id));++ // And the ranking actually flips — the busy model is no longer first.+ const { scores } = getRoutingScores();+ expect(scores[0].modelDbId).toBe(idle.id);+ });++ it('recovers on its own once the day window rolls past the usage', () => {+ recordRequests(busy.modelId, 95);+ expect(headroomOf(busy.id)).toBeLessThan(1);++ // Age the same rows out of the sliding day. Nothing resets a counter — the+ // window simply stops counting them.+ getDb().prepare(+ "UPDATE rate_limit_usage SET created_at_ms = ? WHERE model_id = ?"+ ).run(Date.now() - DAY_MS - 60_000, busy.modelId);+ invalidateWindowUsage();++ expect(headroomOf(busy.id)).toBe(1);+ expect(scoreOf(busy.id)).toBeCloseTo(scoreOf(idle.id), 10);+ });++ it('honours the tunable thresholds from the settings API', () => {+ recordRequests(busy.modelId, 60); // 40% left — outside the 20% default ramp+ expect(headroomOf(busy.id)).toBe(1);++ // Operator asks to start protecting at 50% remaining with a 0.3 floor.+ setHeadroomThresholds(0.5, 0.3);+ invalidateWindowUsage();+ // 0.3 + 0.7·(0.4/0.5) = 0.86+ expect(headroomOf(busy.id)).toBeCloseTo(0.86, 5);+ expect(headroomOf(idle.id)).toBe(1);+ });++ it('takes the busiest window across rpd and tpd', () => {+ const db = getDb();+ db.prepare("UPDATE models SET tpd_limit = 1000 WHERE id = ?").run(busy.id);+ recordRequests(busy.modelId, 10); // only 10% of RPD+ db.prepare(`+ INSERT INTO rate_limit_usage (platform, model_id, key_id, kind, tokens, created_at_ms)+ VALUES ('groq', ?, ?, 'tokens', 900, ?)+ `).run(busy.modelId, keyId(), Date.now());+ invalidateWindowUsage();++ // 90% of TPD binds, not 10% of RPD.+ expect(headroomOf(busy.id)).toBeCloseTo(0.55, 5); // 0.1 + 0.9·(0.1/0.2)+ db.prepare("UPDATE models SET tpd_limit = NULL WHERE id = ?").run(busy.id);+ });++ it('follows the eligible key with the most headroom, not the worst one', () => {+ const db = getDb();+ const second = encrypt('groq-key-2');+ db.prepare(`+ INSERT INTO api_keys (platform, label, encrypted_key, iv, auth_tag, status, enabled)+ VALUES ('groq', 'k1', ?, ?, ?, 'healthy', 1)+ `).run(second.encrypted, second.iv, second.authTag);+ const ids = (db.prepare("SELECT id FROM api_keys WHERE platform = 'groq' ORDER BY id").all() as { id: number }[]).map(r => r.id);++ const stmt = db.prepare(`+ INSERT INTO rate_limit_usage (platform, model_id, key_id, kind, tokens, created_at_ms)+ VALUES ('groq', ?, ?, 'request', 0, ?)+ `);+ for (let i = 0; i < 99; i++) stmt.run(busy.modelId, ids[0], Date.now());+ invalidateWindowUsage();++ // One key is all but exhausted; the other is untouched, and the router will+ // land on it, so the model is not under any real pressure.+ expect(headroomOf(busy.id)).toBe(1);+ db.prepare('DELETE FROM api_keys WHERE id = ?').run(ids[1]);+ invalidateWindowUsage();+ });++ it('has no opinion for a model that declares no window limits', () => {+ getDb().prepare("UPDATE models SET rpd_limit = NULL WHERE id = ?").run(busy.id);+ recordRequests(busy.modelId, 500);+ expect(modelWindowUsedFraction(+ { platform: 'groq', modelId: busy.modelId, keyId: null },+ { rpm: null, rpd: null, tpm: null, tpd: null },+ )).toBeNull();+ expect(headroomOf(busy.id)).toBe(1);+ getDb().prepare("UPDATE models SET rpd_limit = 100 WHERE id = ?").run(busy.id);+ });+});diff --git a/server/src/__tests__/services/scoring.test.ts b/server/src/__tests__/services/scoring.test.tsindex 63d1ebf7..652aabf8 100644--- a/server/src/__tests__/services/scoring.test.ts+++ b/server/src/__tests__/services/scoring.test.ts@@ -1,7 +1,7 @@import { describe, it, expect } from 'vitest';import {BANDIT_PRESETS, combineScore, speedScore, intelligenceScore, intelligenceComposite,- headroomFactor, rateLimitFactor, sampleBeta, reliabilityPosterior,+ headroomFactor, rateWindowHeadroomFactor, rateLimitFactor, sampleBeta, reliabilityPosterior,expectedReliability, SPEED_PRIOR, HEADROOM_FLOOR,isPeakHours, peakAdjustedWeights, hourInTimezone, isValidPeakHour, isValidTimezone,isPeakExemptStrategy, DEFAULT_PEAK_HOURS, PEAK_SPEED_TO_RELIABILITY,@@ -149,6 +149,36 @@ describe('scoring: guardrails', () => {expect(headroomFactor(123, 0)).toBe(1);});+ it('rate-window headroom rides the same ramp as the monthly one (#899)', () => {+ expect(rateWindowHeadroomFactor(0)).toBe(1); // idle+ expect(rateWindowHeadroomFactor(0.5)).toBe(1); // 50% left, ramp starts at 20%+ // Exactly at the ramp start: 1 - 0.8 lands a float ulp under 0.2, so this+ // is the very top of the ramp rather than the flat part. Same arithmetic+ // the monthly guardrail has always done.+ expect(rateWindowHeadroomFactor(0.8)).toBeCloseTo(1, 9);+ // 5% of the window left → 0.1 + 0.9·(0.05/0.2)+ expect(rateWindowHeadroomFactor(0.95)).toBeCloseTo(0.325, 5);+ expect(rateWindowHeadroomFactor(1)).toBeCloseTo(HEADROOM_FLOOR, 5);+ // Over-consumption (a provider counted more than we did) cannot go below+ // the floor.+ expect(rateWindowHeadroomFactor(1.4)).toBeCloseTo(HEADROOM_FLOOR, 5);+ // The two guardrails agree wherever the remaining fraction agrees.+ expect(rateWindowHeadroomFactor(0.95)).toBeCloseTo(headroomFactor(950, 1000), 10);+ });++ it('rate-window headroom takes the same tunable thresholds (#899)', () => {+ const opts = { rampStart: 0.5, floor: 0.3 };+ expect(rateWindowHeadroomFactor(0.4, opts)).toBe(1); // 60% left+ expect(rateWindowHeadroomFactor(0.6, opts)).toBeCloseTo(0.86, 5); // 0.3 + 0.7·(0.4/0.5)+ expect(rateWindowHeadroomFactor(0.6, { rampStart: 5 }))+ .toBe(rateWindowHeadroomFactor(0.6)); // out of range → defaults+ });++ it('no measurable window yields no opinion (factor 1)', () => {+ expect(rateWindowHeadroomFactor(null)).toBe(1);+ expect(rateWindowHeadroomFactor(NaN)).toBe(1);+ });+it('rate-limit factor is 1 at no penalty and damped but non-zero at max', () => {expect(rateLimitFactor(0)).toBe(1);expect(rateLimitFactor(10)).toBeCloseTo(0.4, 5);diff --git a/server/src/services/ratelimit.ts b/server/src/services/ratelimit.tsindex dbb28eb8..747432ae 100644--- a/server/src/services/ratelimit.ts+++ b/server/src/services/ratelimit.ts@@ -2,6 +2,7 @@import { getDb } from '../db/index.js';import { isLoopbackOrPrivateUrl } from '../lib/url-guard.js';+import { parseModelScope, scopeAllows } from '../lib/model-scope.js';interface Window {timestamps: number[];@@ -354,6 +355,199 @@ export function canUseTokens(return true;}+// ── Window utilization for the routing guardrail (#899) ──────────────────────+// Everything above answers a yes/no question on the hot path: may THIS key+// serve THIS request. The router also needs a graded one — "how much of its+// window quota has this model already burned" — so it can steer traffic away+// from a model at 95% of its daily cap before that cap actually rejects+// anything. See rateWindowHeadroomFactor in scoring.ts for what is done with+// the number.+//+// Cost is the whole design constraint: this runs per chain entry per request,+// and the naive shape is four counts per model × key. So it is one grouped scan+// of rate_limit_usage plus one read of api_keys, memoised for a few seconds and+// shared by every entry of every chain — the same trade getKeyQuotaHeadroom+// makes in provider-quota.ts. Per-entry work is then pure in-memory arithmetic+// over the model's own limit columns, which the router already has in hand.++export interface WindowLimits {+ rpm: number | null;+ rpd: number | null;+ tpm: number | null;+ tpd: number | null;+}++interface KeyWindowUsage { rpm: number; rpd: number; tpm: number; tpd: number }+const NO_WINDOW_USAGE: KeyWindowUsage = { rpm: 0, rpd: 0, tpm: 0, tpd: 0 };++interface EligibleKey { id: number; baseUrl: string | null; scope: Set<string> | null }++interface WindowUsageSnapshot {+ db: unknown;+ at: number;+ usage: Map<string, KeyWindowUsage>;+ keysByPlatform: Map<string, EligibleKey[]>;+ baseUrlByKeyId: Map<number, string | null>;+}++/** Utilization moves on the timescale of a rate-limit window, not a request, so+ * a few seconds of staleness is invisible to a guardrail while the query count+ * drops to ~one per burst. The hard gates above are unaffected — they still+ * read live counts, so nothing here can let a request through a real limit. */+const WINDOW_USAGE_TTL_MS = 5_000;++// The Db handle is part of the cache identity, so reconnecting (tests, a+// restore) invalidates the snapshot rather than serving another database's+// numbers.+let windowUsageSnapshot: WindowUsageSnapshot | null = null;++function usageCacheKey(platform: string, modelId: string, keyId: number): string {+ return `${platform}\u0000${modelId}\u0000${keyId}`;+}++function buildWindowUsageSnapshot(now: number): WindowUsageSnapshot | null {+ return withDb(db => {+ // One grouped scan replaces four counts per model × key. Same windows the+ // gates above enforce: a sliding minute for rpm/tpm, a sliding day for+ // rpd/tpd. Rows older than a day are pruned on write, but the WHERE keeps+ // this correct even when a prune has not run yet.+ const rows = db.prepare(`+ SELECT platform, model_id, key_id,+ SUM(CASE WHEN kind = 'request' AND created_at_ms > ? THEN 1 ELSE 0 END) AS rpm_used,+ SUM(CASE WHEN kind = 'request' THEN 1 ELSE 0 END) AS rpd_used,+ SUM(CASE WHEN kind = 'tokens' AND created_at_ms > ? THEN tokens ELSE 0 END) AS tpm_used,+ SUM(CASE WHEN kind = 'tokens' THEN tokens ELSE 0 END) AS tpd_used+ FROM rate_limit_usage+ WHERE created_at_ms > ?+ GROUP BY platform, model_id, key_id+ `).all(now - MINUTE, now - MINUTE, now - DAY) as Array<{+ platform: string; model_id: string; key_id: number;+ rpm_used: number; rpd_used: number; tpm_used: number; tpd_used: number;+ }>;++ const usage = new Map<string, KeyWindowUsage>();+ for (const r of rows) {+ usage.set(usageCacheKey(r.platform, r.model_id, r.key_id), {+ rpm: r.rpm_used, rpd: r.rpd_used, tpm: r.tpm_used, tpd: r.tpd_used,+ });+ }++ // Mirror the router's key eligibility (selectKeyForModel): enabled ++ // healthy/unknown, and the model scope must admit the model. The custom+ // endpoint check needs base_url for every row, disabled ones included, so+ // that map is built from the unfiltered read.+ const keyRows = db.prepare(+ 'SELECT id, platform, base_url, model_scope_json, enabled, status FROM api_keys'+ ).all() as Array<{+ id: number; platform: string; base_url: string | null;+ model_scope_json: string | null; enabled: number; status: string;+ }>;++ const baseUrlByKeyId = new Map<number, string | null>();+ const keysByPlatform = new Map<string, EligibleKey[]>();+ for (const k of keyRows) {+ baseUrlByKeyId.set(k.id, k.base_url);+ if (k.enabled !== 1 || (k.status !== 'healthy' && k.status !== 'unknown')) continue;+ const list = keysByPlatform.get(k.platform) ?? [];+ list.push({ id: k.id, baseUrl: k.base_url, scope: parseModelScope(k.model_scope_json) });+ keysByPlatform.set(k.platform, list);+ }++ return { db, at: now, usage, keysByPlatform, baseUrlByKeyId };+ }) ?? null;+}++function getWindowUsageSnapshot(now: number): WindowUsageSnapshot | null {+ let db;+ try {+ db = getDb();+ } catch {+ return null;+ }+ const cached = windowUsageSnapshot;+ if (cached && cached.db === db && now - cached.at < WINDOW_USAGE_TTL_MS && now >= cached.at) {+ return cached;+ }+ const built = buildWindowUsageSnapshot(now);+ windowUsageSnapshot = built;+ return built;+}++/** Drop the memoised window snapshot. Tests use it to make a write visible+ * immediately; production relies on the TTL. */+export function invalidateWindowUsage(): void {+ windowUsageSnapshot = null;+}++/** The busiest window of one key: the ratio that would reject its next request+ * first. A key metered on several windows takes the WORST of them, because the+ * binding constraint is what 429s. */+function keyWindowPressure(usage: KeyWindowUsage, limits: WindowLimits): number {+ let worst = 0;+ const ratios: Array<[number, number | null]> = [+ [usage.rpm, limits.rpm],+ [usage.rpd, limits.rpd],+ [usage.tpm, limits.tpm],+ [usage.tpd, limits.tpd],+ ];+ for (const [used, limit] of ratios) {+ if (limit == null || limit <= 0) continue;+ const r = used / limit;+ if (r > worst) worst = r;+ }+ return worst;+}++/**+ * Share of its binding rate-limit window a model has already consumed, as a+ * 0..1 fraction (0 idle, 1 exhausted). null = no opinion: the model declares no+ * window limits, has no routable key to measure, or the database is unreachable.+ *+ * Which key's numbers to report matters, and the answer is the same one the+ * dashboard's usage badge settled on (#921): the guardrail asks "how close is+ * this model to being unroutable", so it must follow the key the router would+ * pick NEXT, not the worst key on the account. A platform with one exhausted+ * key and one idle key routes perfectly well, so we take the ELIGIBLE key with+ * the MOST headroom.+ *+ * In-flight leases are deliberately not counted. They exist to close the+ * check-then-act race on the hard gates; this is a steering signal averaged over+ * a whole window, and folding a per-request quantity into a cached snapshot+ * would buy noise, not accuracy.+ */+export function modelWindowUsedFraction(+ model: { platform: string; modelId: string; keyId?: number | null },+ limits: WindowLimits,+ now = Date.now(),+): number | null {+ if (limits.rpm == null && limits.rpd == null && limits.tpm == null && limits.tpd == null) return null;+ const snap = getWindowUsageSnapshot(now);+ if (!snap) return null;++ const pool = snap.keysByPlatform.get(model.platform);+ if (!pool || pool.length === 0) return null; // nothing routable → a number would be fiction++ // A custom model belongs to one endpoint; only that endpoint's credentials can+ // serve it. Legacy rows (key_id NULL) keep the any-key match.+ const endpointBaseUrl = model.platform === 'custom' && model.keyId != null+ ? snap.baseUrlByKeyId.get(model.keyId) ?? null+ : null;++ let best: number | null = null;+ for (const k of pool) {+ if (!scopeAllows(k.scope, model.modelId)) continue;+ if (model.platform === 'custom' && model.keyId != null) {+ if (endpointBaseUrl == null ? k.id !== model.keyId : k.baseUrl !== endpointBaseUrl) continue;+ }+ const usage = snap.usage.get(usageCacheKey(model.platform, model.modelId, k.id)) ?? NO_WINDOW_USAGE;+ const pressure = keyWindowPressure(usage, limits);+ if (best === null || pressure < best) best = pressure;+ if (best === 0) break; // an untouched key is as good as it gets+ }+ if (best === null) return null; // every key scoped away from this model+ return Math.min(1, best);+}+// ── Provider-wide daily request caps (#162) ──// Some providers enforce one daily REQUEST quota across the WHOLE account,// shared by every model — not per model. OpenRouter's free tier is the classicdiff --git a/server/src/services/router.ts b/server/src/services/router.tsindex c68873fb..cf73540e 100644--- a/server/src/services/router.ts+++ b/server/src/services/router.ts@@ -13,12 +13,14 @@ import {acquireLease,releaseLease,getSoonestCooldownExpiry,+ modelWindowUsedFraction,} from './ratelimit.js';import {BANDIT_PRESETS, DEFAULT_STRATEGY, type RoutingStrategy, type RoutingWeights,type KeySelectionStrategy,reliabilityPosterior, expectedReliability, sampleBeta,- speedScore, intelligenceScore, intelligenceComposite, headroomFactor, rateLimitFactor, combineScore,+ speedScore, intelligenceScore, intelligenceComposite, headroomFactor, rateWindowHeadroomFactor,+ rateLimitFactor, combineScore,peakAdjustedWeights, isValidPeakHour, isValidTimezone,DEFAULT_PEAK_HOURS, type PeakHoursConfig,observedSpeedRank, TIMEOUT_LATENCY_CAP_MS,@@ -974,7 +976,27 @@ function scoreChainEntry(// ONCE per chain by the caller, not per entry — getSetting is an uncached// SELECT, so reading it here cost two extra SQLite round-trips per model per// request, the same reason `weights` and `keyCounts` are hoisted.- const headroom = headroomFactor(stats?.monthlyUsedTokens ?? 0, budget, headroomCfg);+ const monthlyHeadroom = headroomFactor(stats?.monthlyUsedTokens ?? 0, budget, headroomCfg);++ // The same guardrail, driven by live rpm/rpd/tpm/tpd utilization instead of+ // the monthly budget (#899). Most free tiers publish a daily request or token+ // cap and no monthly figure at all, so without this a model sits at score #1+ // until the request that finally 429s it. Reads a snapshot memoised inside+ // ratelimit.ts, so this costs no query per model per request.+ const windowHeadroom = rateWindowHeadroomFactor(+ modelWindowUsedFraction(+ { platform: entry.platform, modelId: entry.model_id, keyId: entry.key_id },+ { rpm: entry.rpm_limit, rpd: entry.rpd_limit, tpm: entry.tpm_limit, tpd: entry.tpd_limit },+ ),+ headroomCfg,+ );++ // The WORSE of the two, not their product: both express the same "this model+ // is close to burning out" opinion on different meters, and multiplying them+ // would push a model that is low on both to floor², below the floor the+ // operator configured. Taking the binding constraint keeps the floor meaning+ // what it says — the same rule getKeyQuotaHeadroom applies across metrics.+ const headroom = Math.min(monthlyHeadroom, windowHeadroom);const rl = rateLimitFactor(getPenalty(entry.model_db_id));// Per-model env overrides (#738) scale the final score so a slow ordiff --git a/server/src/services/scoring.ts b/server/src/services/scoring.tsindex 7cbaecb2..8b0324ed 100644--- a/server/src/services/scoring.ts+++ b/server/src/services/scoring.ts@@ -365,14 +365,47 @@ export interface HeadroomThresholds {export function headroomFactor(usedTokens: number, budgetTokens: number, opts?: HeadroomThresholds): number {if (!budgetTokens || budgetTokens <= 0) return 1; // unknown budget → no opinion+ return headroomRamp(1 - usedTokens / budgetTokens, opts);+}++/** The shared ramp both headroom guardrails ride: 1 while `remaining` (a 0..1+ * fraction of the quota still available) is comfortable, then linear down to+ * `floor` as it reaches zero. Factored out so the monthly-budget guardrail and+ * the rate-window one below cannot drift apart — and so a single pair of+ * operator-tuned thresholds governs both (#899). */+function headroomRamp(remainingRaw: number, opts?: HeadroomThresholds): number {const rampStart = clampUnit(opts?.rampStart, HEADROOM_RAMP_START);const floor = clampUnit(opts?.floor, HEADROOM_FLOOR);- const remaining = Math.max(0, 1 - usedTokens / budgetTokens);+ const remaining = Math.max(0, Math.min(1, remainingRaw));if (remaining >= rampStart) return 1;// Linear from (0 remaining → floor) to (rampStart remaining → 1).return floor + (1 - floor) * (remaining / rampStart);}+// ── Guardrail: rate-window headroom (#899) ──────────────────────────────────+// The monthly-budget guardrail above only has an opinion about models that+// declare a `monthly_token_budget`. The far more common free-tier shape is a+// per-day request or token cap (rpd/tpd, plus the per-minute rpm/tpm), and+// those were purely BINARY: canMakeRequest/canUseTokens reject at 100% and the+// router happily keeps a model pinned at #1 until the very request that+// exhausts it, then eats a 429 and falls through. That is exactly the+// "deprioritize at 82% utilization, prefer the idle peer" behavior the issue+// asks for.+//+// So: same ramp, same tunable thresholds, driven by live window utilization+// instead of monthly tokens. `usedFraction` is the share of the binding window+// limit already consumed (0 = idle, 1 = exhausted); null means the model+// declares no window limits, or has no routable key to measure — in both cases+// the guardrail has no opinion and returns 1.+//+// Recovery is automatic and needs no bookkeeping: the windows are sliding, so a+// demoted model's utilization falls on its own as the minute or the day rolls+// past, and the factor climbs straight back to 1.+export function rateWindowHeadroomFactor(usedFraction: number | null, opts?: HeadroomThresholds): number {+ if (usedFraction === null || !Number.isFinite(usedFraction)) return 1; // unknown → no opinion+ return headroomRamp(1 - usedFraction, opts);+}+// Out-of-range / non-finite operator input falls back to the default rather// than silently clamping to a legal-but-unintended value (#899).function clampUnit(n: number | undefined, fallback: number): number {