7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.
TypeScript
perf: stop the quota panel from ranking the whole observation log on every poll (#1123)
Tashfeen77e0ecc2410f
diff --git a/.env.example b/.env.exampleindex b34a16d7..80e71a11 100644--- a/.env.example+++ b/.env.example@@ -227,6 +227,12 @@ PORT=3001REQUEST_ANALYTICS_RETENTION_DAYS=90REQUEST_ANALYTICS_MAX_ROWS=100000+# Provider quota observation log (the audit trail behind the per-key quota+# state the dashboard shows). Only the newest row per pool is ever read, so the+# history is pruned daily by age and by count. 0 disables that half.+# QUOTA_OBSERVATIONS_RETENTION_DAYS=30+# QUOTA_OBSERVATIONS_MAX_ROWS=200000+# Persisted server logs behind the dashboard's log viewer. Only warn/error# lines are written to the database (the live view is an in-memory ring), so# these bounds are far tighter than the analytics ones above. Either value at 0diff --git a/client/src/App.tsx b/client/src/App.tsxindex 92b698b0..a3fb8004 100644--- a/client/src/App.tsx+++ b/client/src/App.tsx@@ -47,12 +47,15 @@ import AgentsPage from '@/pages/AgentsPage'// silently. A page that already shows the failure inline can opt out with// `meta: { silenceToast: true }` on the mutation.const queryClient = new QueryClient({- // A short staleTime dedupes the mount storm (#1047): the Models page alone- // mounts ~13 queries, several of them the same endpoint from different- // components, and staleTime 0 refetched every one of them on every- // navigation and window focus. Pollers use refetchInterval and mutations+ // staleTime dedupes the mount storm (#1047) and keeps page-to-page+ // navigation off the network: the Models page alone mounts ~13 queries,+ // several of them the same endpoint from different components, and+ // staleTime 0 refetched every one of them on every navigation and window+ // focus. Cached data renders immediately either way; this only decides+ // whether a background refetch follows. Thirty seconds is shorter than any+ // poller here (refetchInterval still fires on its own clock), and mutations// invalidate explicitly, so nothing user-visible goes stale.- defaultOptions: { queries: { staleTime: 5_000 } },+ defaultOptions: { queries: { staleTime: 30_000 } },mutationCache: new MutationCache({onError: (error, _variables, _context, mutation) => {if (mutation.meta?.silenceToast) returndiff --git a/server/src/__tests__/db/migrate/roundtrip.test.ts b/server/src/__tests__/db/migrate/roundtrip.test.tsindex 63a493fd..02bd05ee 100644--- a/server/src/__tests__/db/migrate/roundtrip.test.ts+++ b/server/src/__tests__/db/migrate/roundtrip.test.ts@@ -33,6 +33,7 @@ const BACKUPS_TABLE_FILENAME = '20260823_000002_backups_table.ts';const ATTEMPT_KEY_LABEL_FILENAME = '20260823_000003_attempt_key_label.ts';const PROFILE_AUTO_INCLUDE_FILENAME = '20260823_000004_profile_auto_include.ts';const IDEMPOTENCY_CLAIMS_FILENAME = '20260901_000001_idempotency_claims.ts';+const QUOTA_OBSERVATION_LOOKUP_FILENAME = '20260901_000002_quota_observation_lookup.ts';interface SchemaRow {type: string;@@ -112,6 +113,7 @@ describe('migration round trip', () => {ATTEMPT_KEY_LABEL_FILENAME,PROFILE_AUTO_INCLUDE_FILENAME,IDEMPOTENCY_CLAIMS_FILENAME,+ QUOTA_OBSERVATION_LOOKUP_FILENAME,]);} finally {db.close();diff --git a/server/src/__tests__/services/provider-quota.test.ts b/server/src/__tests__/services/provider-quota.test.tsindex 88295f3d..c8be34ed 100644--- a/server/src/__tests__/services/provider-quota.test.ts+++ b/server/src/__tests__/services/provider-quota.test.ts@@ -6,6 +6,7 @@ import {parseQuotaObservationsFromResponse,inferQuotaPoolKey,} from '../../services/provider-quota.js';+import { pruneQuotaObservations } from '../../services/request-retention.js';function insertState(row: {platform: string;@@ -62,6 +63,83 @@ describe('provider-quota: record + read round-trip', () => {getDb().prepare('DELETE FROM provider_quota_observations').run();});+ it('surfaces the newest observation per pool when the log holds many', () => {+ // Older rows for the same pool must never win, and rows for a sibling pool+ // must never bleed across. Mirrors the dashboard poll on a long-lived+ // install whose log holds hundreds of thousands of rows per pool.+ for (let i = 0; i < 25; i++) {+ recordQuotaObservation({+ platform: 'groq', keyId: 7, quotaPoolKey: 'groq::account', metric: 'tokens',+ limit: 1000, remaining: 1000 - i, modelId: `old-${i}`, source: 'header',+ observedAt: new Date(Date.UTC(2026, 0, 1, 0, i)).toISOString(),+ });+ }+ recordQuotaObservation({+ platform: 'groq', keyId: 7, quotaPoolKey: 'groq::account', metric: 'tokens',+ limit: 1000, remaining: 5, modelId: 'newest', source: 'header',+ observedAt: new Date(Date.UTC(2026, 0, 2)).toISOString(),+ });+ recordQuotaObservation({+ platform: 'groq', keyId: 7, quotaPoolKey: 'groq::account', metric: 'requests',+ limit: 30, remaining: 1, modelId: 'other-metric', source: 'header',+ observedAt: new Date(Date.UTC(2026, 0, 3)).toISOString(),+ });+ const rows = getQuotaStateForKeys().filter(r => r.platform === 'groq' && r.keyId === 7);+ expect(rows).toHaveLength(2);+ expect(rows.find(r => r.metric === 'tokens')?.modelId).toBe('newest');+ expect(rows.find(r => r.metric === 'tokens')?.remaining).toBe(5);+ expect(rows.find(r => r.metric === 'requests')?.modelId).toBe('other-metric');+ });++ it('prunes the observation log by age and count without touching state', () => {+ const db = getDb();+ const insert = db.prepare(`+ INSERT INTO provider_quota_observations+ (id, platform, key_id, quota_pool_key, metric, observed_at, created_at)+ VALUES (?, 'groq', 7, 'groq::account', 'tokens', ?, ?)+ `);+ const now = Date.UTC(2026, 8, 1);+ const stamp = (ms: number) => new Date(ms).toISOString().slice(0, 19).replace('T', ' ');+ for (let i = 0; i < 10; i++) {+ // 5 rows older than 30 days, 5 fresh ones.+ const at = stamp(now - (i < 5 ? 40 : 1) * 86_400_000 - i * 1000);+ insert.run(`obs-${i}`, at, at);+ }+ insertState({ platform: 'groq', keyId: 7, pool: 'groq::account', metric: 'tokens', limit: 1000, remaining: 10, resetAt: null });++ expect(pruneQuotaObservations(db, now)).toEqual({ deleted: 5, done: true });+ expect(db.prepare('SELECT COUNT(*) AS n FROM provider_quota_observations').get()).toEqual({ n: 5 });++ process.env.QUOTA_OBSERVATIONS_MAX_ROWS = '2';+ try {+ expect(pruneQuotaObservations(db, now)).toEqual({ deleted: 3, done: true });+ } finally {+ delete process.env.QUOTA_OBSERVATIONS_MAX_ROWS;+ }+ const left = db.prepare('SELECT id FROM provider_quota_observations ORDER BY created_at DESC').all() as { id: string }[];+ expect(left.map(r => r.id)).toEqual(['obs-5', 'obs-6']);+ expect(readState('groq', 7, 'groq::account', 'tokens')?.remaining).toBe(10);+ });++ it('stops a large sweep at its time budget and reports it unfinished', () => {+ const db = getDb();+ const insert = db.prepare(`+ INSERT INTO provider_quota_observations+ (id, platform, key_id, quota_pool_key, metric, observed_at, created_at)+ VALUES (?, 'groq', 7, 'groq::account', 'tokens', ?, ?)+ `);+ const now = Date.UTC(2026, 8, 1);+ const old = new Date(now - 60 * 86_400_000).toISOString().slice(0, 19).replace('T', ' ');+ const tx = db.transaction(() => { for (let i = 0; i < 45_000; i++) insert.run(`o-${i}`, old, old); });+ tx();+ // A zero budget allows exactly one chunk before the check trips.+ const first = pruneQuotaObservations(db, now, 0);+ expect(first.done).toBe(false);+ expect(first.deleted).toBe(20_000);+ const rest = pruneQuotaObservations(db, now, 60_000);+ expect(rest).toEqual({ deleted: 25_000, done: true });+ });+it('records an observation and surfaces it via getQuotaStateForKeys', () => {const rec = recordQuotaObservation({platform: 'groq',diff --git a/server/src/db/migrate/defaults.ts b/server/src/db/migrate/defaults.tsindex 6c4edbf7..6ccdc19e 100644--- a/server/src/db/migrate/defaults.ts+++ b/server/src/db/migrate/defaults.ts@@ -28,6 +28,7 @@ import * as backupsTable from '../migrations/20260823_000002_backups_table.js';import * as attemptKeyLabel from '../migrations/20260823_000003_attempt_key_label.js';import * as profileAutoInclude from '../migrations/20260823_000004_profile_auto_include.js';import * as idempotencyClaims from '../migrations/20260901_000001_idempotency_claims.js';+import * as quotaObservationLookup from '../migrations/20260901_000002_quota_observation_lookup.js';export interface MigrationModule {up(db: Db): void;@@ -68,6 +69,7 @@ export const BACKUPS_TABLE_FILENAME = '20260823_000002_backups_table.ts';export const ATTEMPT_KEY_LABEL_FILENAME = '20260823_000003_attempt_key_label.ts';export const PROFILE_AUTO_INCLUDE_FILENAME = '20260823_000004_profile_auto_include.ts';export const IDEMPOTENCY_CLAIMS_FILENAME = '20260901_000001_idempotency_claims.ts';+export const QUOTA_OBSERVATION_LOOKUP_FILENAME = '20260901_000002_quota_observation_lookup.ts';export const DEFAULT_MIGRATIONS: readonly DefaultMigration[] = [{ filename: LEGACY_BASELINE_FILENAME, module: legacyBaseline },@@ -99,4 +101,5 @@ export const DEFAULT_MIGRATIONS: readonly DefaultMigration[] = [{ filename: ATTEMPT_KEY_LABEL_FILENAME, module: attemptKeyLabel },{ filename: PROFILE_AUTO_INCLUDE_FILENAME, module: profileAutoInclude },{ filename: IDEMPOTENCY_CLAIMS_FILENAME, module: idempotencyClaims },+ { filename: QUOTA_OBSERVATION_LOOKUP_FILENAME, module: quotaObservationLookup },];diff --git a/server/src/db/migrations/20260901_000002_quota_observation_lookup.ts b/server/src/db/migrations/20260901_000002_quota_observation_lookup.tsnew file mode 100644index 00000000..9c3683dd--- /dev/null+++ b/server/src/db/migrations/20260901_000002_quota_observation_lookup.ts@@ -0,0 +1,40 @@+// Migration: lookup indexes for the provider quota observation log+// Created: 2026-09-01+//+// DOWN: reversible+//+// provider_quota_observations is an append-only log: every quota header, 429+// body and probe result lands here, so a busy install accumulates hundreds of+// thousands of rows within weeks. Two readers were paying for that on every+// dashboard poll:+//+// - getQuotaStateForKeys() needs the NEWEST observation per+// (platform, key_id, quota_pool_key, metric) — one row for each of the few+// dozen provider_quota_state rows. The baseline index+// (platform, key_id, observed_at) cannot answer that per pool+metric, so+// the query window-functioned the whole log (raw_json included) and held+// the event loop for seconds. GET /api/health and GET /api/free-tier call+// it, and the Keys page polls the former every 30s.+// - the retention sweep deletes by created_at, which had no index at all.+//+// The composite index turns the per-state lookup into a single seek; the+// created_at index keeps the prune a range delete.++import type { Db } from '../types.js';++export function up(db: Db): void {+ db.exec(`+ CREATE INDEX IF NOT EXISTS idx_provider_quota_observations_latest+ ON provider_quota_observations(platform, key_id, quota_pool_key, metric, observed_at DESC, created_at DESC);++ CREATE INDEX IF NOT EXISTS idx_provider_quota_observations_created_at+ ON provider_quota_observations(created_at);+ `);+}++export function down(db: Db): void {+ db.exec(`+ DROP INDEX IF EXISTS idx_provider_quota_observations_created_at;+ DROP INDEX IF EXISTS idx_provider_quota_observations_latest;+ `);+}diff --git a/server/src/services/provider-quota.ts b/server/src/services/provider-quota.tsindex c6b34936..5835e48a 100644--- a/server/src/services/provider-quota.ts+++ b/server/src/services/provider-quota.ts@@ -532,16 +532,13 @@ export function getQuotaStateForKeys(): QuotaObservationView[] {return [];}normalizeExpiredQuotaState(db);+ // One seek per state row for its newest observation. The log is append-only+ // and grows into the hundreds of thousands of rows, so this must never scan+ // it: the correlated subquery walks idx_provider_quota_observations_latest+ // (platform, key_id, quota_pool_key, metric, observed_at DESC, created_at+ // DESC) and stops at the first entry. The window-function form it replaces+ // ranked the entire table, raw_json included, on every dashboard poll.return db.prepare(`- WITH latest AS (- SELECT- oq.*,- ROW_NUMBER() OVER (- PARTITION BY oq.platform, oq.key_id, oq.quota_pool_key, oq.metric- ORDER BY oq.observed_at DESC, oq.created_at DESC- ) AS rn- FROM provider_quota_observations oq- )SELECTpqs.platform,pqs.key_id AS keyId,@@ -568,12 +565,17 @@ export function getQuotaStateForKeys(): QuotaObservationView[] {latest.created_at AS createdAtFROM provider_quota_state pqsLEFT JOIN api_keys k ON k.id = pqs.key_id- LEFT JOIN latest- ON latest.platform = pqs.platform- AND latest.key_id = pqs.key_id- AND latest.quota_pool_key = pqs.quota_pool_key- AND latest.metric = pqs.metric- AND latest.rn = 1+ LEFT JOIN provider_quota_observations latest+ ON latest.id = (+ SELECT o.id+ FROM provider_quota_observations o+ WHERE o.platform = pqs.platform+ AND o.key_id = pqs.key_id+ AND o.quota_pool_key = pqs.quota_pool_key+ AND o.metric = pqs.metric+ ORDER BY o.observed_at DESC, o.created_at DESC+ LIMIT 1+ )ORDER BY pqs.platform ASC, pqs.key_id ASC, pqs.quota_pool_key ASC, pqs.metric ASC`).all() as QuotaObservationView[];}diff --git a/server/src/services/request-retention.ts b/server/src/services/request-retention.tsindex 114f3d51..2829fc10 100644--- a/server/src/services/request-retention.ts+++ b/server/src/services/request-retention.ts@@ -14,6 +14,13 @@ const HOURLY_PRUNE_INTERVAL_MS = 24 * 60 * 60 * 1000;// rows carry no aggregate that would silently change if they were pruned.const DEFAULT_SERVER_LOGS_RETENTION_DAYS = 7;const DEFAULT_SERVER_LOGS_MAX_ROWS = 50_000;+// provider_quota_observations is the append-only audit trail behind+// provider_quota_state (which always holds the current reading). Nothing reads+// further back than the newest row per pool, so the history is diagnostics+// only — yet a busy install wrote ~15k rows a day and reached 470k rows in a+// month. Pruned on the daily gate: the delete is a range over created_at.+const DEFAULT_QUOTA_OBSERVATIONS_RETENTION_DAYS = 30;+const DEFAULT_QUOTA_OBSERVATIONS_MAX_ROWS = 200_000;type RetentionDb = ReturnType<typeof getDb>;@@ -24,6 +31,7 @@ export interface RequestAnalyticsRetentionConfig {let nextPruneAtMs = 0;let nextHourlyPruneAtMs = 0;+let nextQuotaObservationsPruneAtMs = 0;function readNonNegativeInt(name: string, defaultValue: number): number {const raw = process.env[name];@@ -59,6 +67,95 @@ export function getServerLogRetentionConfig(): ServerLogRetentionConfig {};}+export interface QuotaObservationRetentionConfig {+ retentionDays: number;+ maxRows: number;+}++/** Same env convention as the pairs above; 0 on either knob disables that half. */+export function getQuotaObservationRetentionConfig(): QuotaObservationRetentionConfig {+ return {+ retentionDays: readNonNegativeInt('QUOTA_OBSERVATIONS_RETENTION_DAYS', DEFAULT_QUOTA_OBSERVATIONS_RETENTION_DAYS),+ maxRows: readNonNegativeInt('QUOTA_OBSERVATIONS_MAX_ROWS', DEFAULT_QUOTA_OBSERVATIONS_MAX_ROWS),+ };+}++// The first sweep on an install that predates the prune may face hundreds of+// thousands of rows; deleting them in one statement held the loop for ~4.5s on+// a 470k-row log. Work is chunked under a wall-clock budget instead, and a+// sweep that runs out of budget is resumed on the next 60s tick rather than+// tomorrow.+const QUOTA_OBSERVATIONS_PRUNE_CHUNK = 20_000;+const QUOTA_OBSERVATIONS_PRUNE_BUDGET_MS = 250;++/**+ * Age- and count-bound the provider quota observation log.+ *+ * provider_quota_state is the source of truth for "how much is left"; this+ * table only explains how it got there. Keeping the newest row per pool is+ * guaranteed by the count bound being far above the number of live pools.+ * created_at is SQLite datetime text, indexed since the 20260901_000002+ * migration, so both deletes are range/ordered walks, not scans. Guarded+ * against the table being absent for the same reason as the hourly prune.+ *+ * Returns how many rows went and whether the sweep finished; `done: false`+ * means the budget ran out and the caller should come back soon.+ */+export function pruneQuotaObservations(+ db: RetentionDb,+ nowMs: number,+ budgetMs = QUOTA_OBSERVATIONS_PRUNE_BUDGET_MS,+): { deleted: number; done: boolean } {+ const hasTable = !!db+ .prepare("SELECT name FROM sqlite_master WHERE type='table' AND name='provider_quota_observations'")+ .get();+ if (!hasTable) return { deleted: 0, done: true };++ const { retentionDays, maxRows } = getQuotaObservationRetentionConfig();+ const started = Date.now();+ const inBudget = () => Date.now() - started < budgetMs;+ let deleted = 0;++ if (retentionDays > 0) {+ const cutoff = toSqliteTimestamp(new Date(nowMs - retentionDays * DAY_MS));+ const byAge = db.prepare(`+ DELETE FROM provider_quota_observations+ WHERE rowid IN (+ SELECT rowid FROM provider_quota_observations+ WHERE created_at < ?+ ORDER BY created_at ASC+ LIMIT ?+ )+ `);+ for (;;) {+ const changes = byAge.run(cutoff, QUOTA_OBSERVATIONS_PRUNE_CHUNK).changes;+ deleted += changes;+ if (changes < QUOTA_OBSERVATIONS_PRUNE_CHUNK) break;+ if (!inBudget()) return { deleted, done: false };+ }+ }++ if (maxRows > 0) {+ const byCount = db.prepare(`+ DELETE FROM provider_quota_observations+ WHERE rowid IN (+ SELECT rowid+ FROM provider_quota_observations+ ORDER BY created_at DESC, rowid DESC+ LIMIT ? OFFSET ?+ )+ `);+ for (;;) {+ const changes = byCount.run(QUOTA_OBSERVATIONS_PRUNE_CHUNK, maxRows).changes;+ deleted += changes;+ if (changes < QUOTA_OBSERVATIONS_PRUNE_CHUNK) break;+ if (!inBudget()) return { deleted, done: false };+ }+ }++ return { deleted, done: true };+}+/*** Age- and count-bound the persisted warn/error log rows.*@@ -144,6 +241,14 @@ export function pruneRequestAnalytics(options: {// than the UI range, or the 30d count will silently drop again.// Guarded against the table being absent (tests that init a DB before the// migration runs would otherwise crash the prune loop).+ // Quota observation log, once a day — or again next tick while a large+ // backlog is still being worked off in budgeted chunks.+ if (nowMs >= nextQuotaObservationsPruneAtMs) {+ const sweep = pruneQuotaObservations(db, nowMs);+ deleted += sweep.deleted;+ nextQuotaObservationsPruneAtMs = nowMs + (sweep.done ? HOURLY_PRUNE_INTERVAL_MS : PRUNE_INTERVAL_MS);+ }+if (nowMs >= nextHourlyPruneAtMs) {nextHourlyPruneAtMs = nowMs + HOURLY_PRUNE_INTERVAL_MS;const hasHourly = !!db