Skip to content

7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.

TypeScript

perf: stop the quota panel from ranking the whole observation log on every poll (#1123)

Tashfeen77e0ecc2410f
diff --git a/.env.example b/.env.example
index b34a16d7..80e71a11 100644
--- a/.env.example
+++ b/.env.example
@@ -227,6 +227,12 @@ PORT=3001
REQUEST_ANALYTICS_RETENTION_DAYS=90
REQUEST_ANALYTICS_MAX_ROWS=100000
+# Provider quota observation log (the audit trail behind the per-key quota
+# state the dashboard shows). Only the newest row per pool is ever read, so the
+# history is pruned daily by age and by count. 0 disables that half.
+# QUOTA_OBSERVATIONS_RETENTION_DAYS=30
+# QUOTA_OBSERVATIONS_MAX_ROWS=200000
+
# Persisted server logs behind the dashboard's log viewer. Only warn/error
# lines are written to the database (the live view is an in-memory ring), so
# these bounds are far tighter than the analytics ones above. Either value at 0
diff --git a/client/src/App.tsx b/client/src/App.tsx
index 92b698b0..a3fb8004 100644
--- a/client/src/App.tsx
+++ b/client/src/App.tsx
@@ -47,12 +47,15 @@ import AgentsPage from '@/pages/AgentsPage'
// silently. A page that already shows the failure inline can opt out with
// `meta: { silenceToast: true }` on the mutation.
const queryClient = new QueryClient({
- // A short staleTime dedupes the mount storm (#1047): the Models page alone
- // mounts ~13 queries, several of them the same endpoint from different
- // components, and staleTime 0 refetched every one of them on every
- // navigation and window focus. Pollers use refetchInterval and mutations
+ // staleTime dedupes the mount storm (#1047) and keeps page-to-page
+ // navigation off the network: the Models page alone mounts ~13 queries,
+ // several of them the same endpoint from different components, and
+ // staleTime 0 refetched every one of them on every navigation and window
+ // focus. Cached data renders immediately either way; this only decides
+ // whether a background refetch follows. Thirty seconds is shorter than any
+ // poller here (refetchInterval still fires on its own clock), and mutations
// invalidate explicitly, so nothing user-visible goes stale.
- defaultOptions: { queries: { staleTime: 5_000 } },
+ defaultOptions: { queries: { staleTime: 30_000 } },
mutationCache: new MutationCache({
onError: (error, _variables, _context, mutation) => {
if (mutation.meta?.silenceToast) return
diff --git a/server/src/__tests__/db/migrate/roundtrip.test.ts b/server/src/__tests__/db/migrate/roundtrip.test.ts
index 63a493fd..02bd05ee 100644
--- a/server/src/__tests__/db/migrate/roundtrip.test.ts
+++ b/server/src/__tests__/db/migrate/roundtrip.test.ts
@@ -33,6 +33,7 @@ const BACKUPS_TABLE_FILENAME = '20260823_000002_backups_table.ts';
const ATTEMPT_KEY_LABEL_FILENAME = '20260823_000003_attempt_key_label.ts';
const PROFILE_AUTO_INCLUDE_FILENAME = '20260823_000004_profile_auto_include.ts';
const IDEMPOTENCY_CLAIMS_FILENAME = '20260901_000001_idempotency_claims.ts';
+const QUOTA_OBSERVATION_LOOKUP_FILENAME = '20260901_000002_quota_observation_lookup.ts';
interface SchemaRow {
type: string;
@@ -112,6 +113,7 @@ describe('migration round trip', () => {
ATTEMPT_KEY_LABEL_FILENAME,
PROFILE_AUTO_INCLUDE_FILENAME,
IDEMPOTENCY_CLAIMS_FILENAME,
+ QUOTA_OBSERVATION_LOOKUP_FILENAME,
]);
} finally {
db.close();
diff --git a/server/src/__tests__/services/provider-quota.test.ts b/server/src/__tests__/services/provider-quota.test.ts
index 88295f3d..c8be34ed 100644
--- a/server/src/__tests__/services/provider-quota.test.ts
+++ b/server/src/__tests__/services/provider-quota.test.ts
@@ -6,6 +6,7 @@ import {
parseQuotaObservationsFromResponse,
inferQuotaPoolKey,
} from '../../services/provider-quota.js';
+import { pruneQuotaObservations } from '../../services/request-retention.js';
function insertState(row: {
platform: string;
@@ -62,6 +63,83 @@ describe('provider-quota: record + read round-trip', () => {
getDb().prepare('DELETE FROM provider_quota_observations').run();
});
+ it('surfaces the newest observation per pool when the log holds many', () => {
+ // Older rows for the same pool must never win, and rows for a sibling pool
+ // must never bleed across. Mirrors the dashboard poll on a long-lived
+ // install whose log holds hundreds of thousands of rows per pool.
+ for (let i = 0; i < 25; i++) {
+ recordQuotaObservation({
+ platform: 'groq', keyId: 7, quotaPoolKey: 'groq::account', metric: 'tokens',
+ limit: 1000, remaining: 1000 - i, modelId: `old-${i}`, source: 'header',
+ observedAt: new Date(Date.UTC(2026, 0, 1, 0, i)).toISOString(),
+ });
+ }
+ recordQuotaObservation({
+ platform: 'groq', keyId: 7, quotaPoolKey: 'groq::account', metric: 'tokens',
+ limit: 1000, remaining: 5, modelId: 'newest', source: 'header',
+ observedAt: new Date(Date.UTC(2026, 0, 2)).toISOString(),
+ });
+ recordQuotaObservation({
+ platform: 'groq', keyId: 7, quotaPoolKey: 'groq::account', metric: 'requests',
+ limit: 30, remaining: 1, modelId: 'other-metric', source: 'header',
+ observedAt: new Date(Date.UTC(2026, 0, 3)).toISOString(),
+ });
+ const rows = getQuotaStateForKeys().filter(r => r.platform === 'groq' && r.keyId === 7);
+ expect(rows).toHaveLength(2);
+ expect(rows.find(r => r.metric === 'tokens')?.modelId).toBe('newest');
+ expect(rows.find(r => r.metric === 'tokens')?.remaining).toBe(5);
+ expect(rows.find(r => r.metric === 'requests')?.modelId).toBe('other-metric');
+ });
+
+ it('prunes the observation log by age and count without touching state', () => {
+ const db = getDb();
+ const insert = db.prepare(`
+ INSERT INTO provider_quota_observations
+ (id, platform, key_id, quota_pool_key, metric, observed_at, created_at)
+ VALUES (?, 'groq', 7, 'groq::account', 'tokens', ?, ?)
+ `);
+ const now = Date.UTC(2026, 8, 1);
+ const stamp = (ms: number) => new Date(ms).toISOString().slice(0, 19).replace('T', ' ');
+ for (let i = 0; i < 10; i++) {
+ // 5 rows older than 30 days, 5 fresh ones.
+ const at = stamp(now - (i < 5 ? 40 : 1) * 86_400_000 - i * 1000);
+ insert.run(`obs-${i}`, at, at);
+ }
+ insertState({ platform: 'groq', keyId: 7, pool: 'groq::account', metric: 'tokens', limit: 1000, remaining: 10, resetAt: null });
+
+ expect(pruneQuotaObservations(db, now)).toEqual({ deleted: 5, done: true });
+ expect(db.prepare('SELECT COUNT(*) AS n FROM provider_quota_observations').get()).toEqual({ n: 5 });
+
+ process.env.QUOTA_OBSERVATIONS_MAX_ROWS = '2';
+ try {
+ expect(pruneQuotaObservations(db, now)).toEqual({ deleted: 3, done: true });
+ } finally {
+ delete process.env.QUOTA_OBSERVATIONS_MAX_ROWS;
+ }
+ const left = db.prepare('SELECT id FROM provider_quota_observations ORDER BY created_at DESC').all() as { id: string }[];
+ expect(left.map(r => r.id)).toEqual(['obs-5', 'obs-6']);
+ expect(readState('groq', 7, 'groq::account', 'tokens')?.remaining).toBe(10);
+ });
+
+ it('stops a large sweep at its time budget and reports it unfinished', () => {
+ const db = getDb();
+ const insert = db.prepare(`
+ INSERT INTO provider_quota_observations
+ (id, platform, key_id, quota_pool_key, metric, observed_at, created_at)
+ VALUES (?, 'groq', 7, 'groq::account', 'tokens', ?, ?)
+ `);
+ const now = Date.UTC(2026, 8, 1);
+ const old = new Date(now - 60 * 86_400_000).toISOString().slice(0, 19).replace('T', ' ');
+ const tx = db.transaction(() => { for (let i = 0; i < 45_000; i++) insert.run(`o-${i}`, old, old); });
+ tx();
+ // A zero budget allows exactly one chunk before the check trips.
+ const first = pruneQuotaObservations(db, now, 0);
+ expect(first.done).toBe(false);
+ expect(first.deleted).toBe(20_000);
+ const rest = pruneQuotaObservations(db, now, 60_000);
+ expect(rest).toEqual({ deleted: 25_000, done: true });
+ });
+
it('records an observation and surfaces it via getQuotaStateForKeys', () => {
const rec = recordQuotaObservation({
platform: 'groq',
diff --git a/server/src/db/migrate/defaults.ts b/server/src/db/migrate/defaults.ts
index 6c4edbf7..6ccdc19e 100644
--- a/server/src/db/migrate/defaults.ts
+++ b/server/src/db/migrate/defaults.ts
@@ -28,6 +28,7 @@ import * as backupsTable from '../migrations/20260823_000002_backups_table.js';
import * as attemptKeyLabel from '../migrations/20260823_000003_attempt_key_label.js';
import * as profileAutoInclude from '../migrations/20260823_000004_profile_auto_include.js';
import * as idempotencyClaims from '../migrations/20260901_000001_idempotency_claims.js';
+import * as quotaObservationLookup from '../migrations/20260901_000002_quota_observation_lookup.js';
export interface MigrationModule {
up(db: Db): void;
@@ -68,6 +69,7 @@ export const BACKUPS_TABLE_FILENAME = '20260823_000002_backups_table.ts';
export const ATTEMPT_KEY_LABEL_FILENAME = '20260823_000003_attempt_key_label.ts';
export const PROFILE_AUTO_INCLUDE_FILENAME = '20260823_000004_profile_auto_include.ts';
export const IDEMPOTENCY_CLAIMS_FILENAME = '20260901_000001_idempotency_claims.ts';
+export const QUOTA_OBSERVATION_LOOKUP_FILENAME = '20260901_000002_quota_observation_lookup.ts';
export const DEFAULT_MIGRATIONS: readonly DefaultMigration[] = [
{ filename: LEGACY_BASELINE_FILENAME, module: legacyBaseline },
@@ -99,4 +101,5 @@ export const DEFAULT_MIGRATIONS: readonly DefaultMigration[] = [
{ filename: ATTEMPT_KEY_LABEL_FILENAME, module: attemptKeyLabel },
{ filename: PROFILE_AUTO_INCLUDE_FILENAME, module: profileAutoInclude },
{ filename: IDEMPOTENCY_CLAIMS_FILENAME, module: idempotencyClaims },
+ { filename: QUOTA_OBSERVATION_LOOKUP_FILENAME, module: quotaObservationLookup },
];
diff --git a/server/src/db/migrations/20260901_000002_quota_observation_lookup.ts b/server/src/db/migrations/20260901_000002_quota_observation_lookup.ts
new file mode 100644
index 00000000..9c3683dd
--- /dev/null
+++ b/server/src/db/migrations/20260901_000002_quota_observation_lookup.ts
@@ -0,0 +1,40 @@
+// Migration: lookup indexes for the provider quota observation log
+// Created: 2026-09-01
+//
+// DOWN: reversible
+//
+// provider_quota_observations is an append-only log: every quota header, 429
+// body and probe result lands here, so a busy install accumulates hundreds of
+// thousands of rows within weeks. Two readers were paying for that on every
+// dashboard poll:
+//
+// - getQuotaStateForKeys() needs the NEWEST observation per
+// (platform, key_id, quota_pool_key, metric) — one row for each of the few
+// dozen provider_quota_state rows. The baseline index
+// (platform, key_id, observed_at) cannot answer that per pool+metric, so
+// the query window-functioned the whole log (raw_json included) and held
+// the event loop for seconds. GET /api/health and GET /api/free-tier call
+// it, and the Keys page polls the former every 30s.
+// - the retention sweep deletes by created_at, which had no index at all.
+//
+// The composite index turns the per-state lookup into a single seek; the
+// created_at index keeps the prune a range delete.
+
+import type { Db } from '../types.js';
+
+export function up(db: Db): void {
+ db.exec(`
+ CREATE INDEX IF NOT EXISTS idx_provider_quota_observations_latest
+ ON provider_quota_observations(platform, key_id, quota_pool_key, metric, observed_at DESC, created_at DESC);
+
+ CREATE INDEX IF NOT EXISTS idx_provider_quota_observations_created_at
+ ON provider_quota_observations(created_at);
+ `);
+}
+
+export function down(db: Db): void {
+ db.exec(`
+ DROP INDEX IF EXISTS idx_provider_quota_observations_created_at;
+ DROP INDEX IF EXISTS idx_provider_quota_observations_latest;
+ `);
+}
diff --git a/server/src/services/provider-quota.ts b/server/src/services/provider-quota.ts
index c6b34936..5835e48a 100644
--- a/server/src/services/provider-quota.ts
+++ b/server/src/services/provider-quota.ts
@@ -532,16 +532,13 @@ export function getQuotaStateForKeys(): QuotaObservationView[] {
return [];
}
normalizeExpiredQuotaState(db);
+ // One seek per state row for its newest observation. The log is append-only
+ // and grows into the hundreds of thousands of rows, so this must never scan
+ // it: the correlated subquery walks idx_provider_quota_observations_latest
+ // (platform, key_id, quota_pool_key, metric, observed_at DESC, created_at
+ // DESC) and stops at the first entry. The window-function form it replaces
+ // ranked the entire table, raw_json included, on every dashboard poll.
return db.prepare(`
- WITH latest AS (
- SELECT
- oq.*,
- ROW_NUMBER() OVER (
- PARTITION BY oq.platform, oq.key_id, oq.quota_pool_key, oq.metric
- ORDER BY oq.observed_at DESC, oq.created_at DESC
- ) AS rn
- FROM provider_quota_observations oq
- )
SELECT
pqs.platform,
pqs.key_id AS keyId,
@@ -568,12 +565,17 @@ export function getQuotaStateForKeys(): QuotaObservationView[] {
latest.created_at AS createdAt
FROM provider_quota_state pqs
LEFT JOIN api_keys k ON k.id = pqs.key_id
- LEFT JOIN latest
- ON latest.platform = pqs.platform
- AND latest.key_id = pqs.key_id
- AND latest.quota_pool_key = pqs.quota_pool_key
- AND latest.metric = pqs.metric
- AND latest.rn = 1
+ LEFT JOIN provider_quota_observations latest
+ ON latest.id = (
+ SELECT o.id
+ FROM provider_quota_observations o
+ WHERE o.platform = pqs.platform
+ AND o.key_id = pqs.key_id
+ AND o.quota_pool_key = pqs.quota_pool_key
+ AND o.metric = pqs.metric
+ ORDER BY o.observed_at DESC, o.created_at DESC
+ LIMIT 1
+ )
ORDER BY pqs.platform ASC, pqs.key_id ASC, pqs.quota_pool_key ASC, pqs.metric ASC
`).all() as QuotaObservationView[];
}
diff --git a/server/src/services/request-retention.ts b/server/src/services/request-retention.ts
index 114f3d51..2829fc10 100644
--- a/server/src/services/request-retention.ts
+++ b/server/src/services/request-retention.ts
@@ -14,6 +14,13 @@ const HOURLY_PRUNE_INTERVAL_MS = 24 * 60 * 60 * 1000;
// rows carry no aggregate that would silently change if they were pruned.
const DEFAULT_SERVER_LOGS_RETENTION_DAYS = 7;
const DEFAULT_SERVER_LOGS_MAX_ROWS = 50_000;
+// provider_quota_observations is the append-only audit trail behind
+// provider_quota_state (which always holds the current reading). Nothing reads
+// further back than the newest row per pool, so the history is diagnostics
+// only — yet a busy install wrote ~15k rows a day and reached 470k rows in a
+// month. Pruned on the daily gate: the delete is a range over created_at.
+const DEFAULT_QUOTA_OBSERVATIONS_RETENTION_DAYS = 30;
+const DEFAULT_QUOTA_OBSERVATIONS_MAX_ROWS = 200_000;
type RetentionDb = ReturnType<typeof getDb>;
@@ -24,6 +31,7 @@ export interface RequestAnalyticsRetentionConfig {
let nextPruneAtMs = 0;
let nextHourlyPruneAtMs = 0;
+let nextQuotaObservationsPruneAtMs = 0;
function readNonNegativeInt(name: string, defaultValue: number): number {
const raw = process.env[name];
@@ -59,6 +67,95 @@ export function getServerLogRetentionConfig(): ServerLogRetentionConfig {
};
}
+export interface QuotaObservationRetentionConfig {
+ retentionDays: number;
+ maxRows: number;
+}
+
+/** Same env convention as the pairs above; 0 on either knob disables that half. */
+export function getQuotaObservationRetentionConfig(): QuotaObservationRetentionConfig {
+ return {
+ retentionDays: readNonNegativeInt('QUOTA_OBSERVATIONS_RETENTION_DAYS', DEFAULT_QUOTA_OBSERVATIONS_RETENTION_DAYS),
+ maxRows: readNonNegativeInt('QUOTA_OBSERVATIONS_MAX_ROWS', DEFAULT_QUOTA_OBSERVATIONS_MAX_ROWS),
+ };
+}
+
+// The first sweep on an install that predates the prune may face hundreds of
+// thousands of rows; deleting them in one statement held the loop for ~4.5s on
+// a 470k-row log. Work is chunked under a wall-clock budget instead, and a
+// sweep that runs out of budget is resumed on the next 60s tick rather than
+// tomorrow.
+const QUOTA_OBSERVATIONS_PRUNE_CHUNK = 20_000;
+const QUOTA_OBSERVATIONS_PRUNE_BUDGET_MS = 250;
+
+/**
+ * Age- and count-bound the provider quota observation log.
+ *
+ * provider_quota_state is the source of truth for "how much is left"; this
+ * table only explains how it got there. Keeping the newest row per pool is
+ * guaranteed by the count bound being far above the number of live pools.
+ * created_at is SQLite datetime text, indexed since the 20260901_000002
+ * migration, so both deletes are range/ordered walks, not scans. Guarded
+ * against the table being absent for the same reason as the hourly prune.
+ *
+ * Returns how many rows went and whether the sweep finished; `done: false`
+ * means the budget ran out and the caller should come back soon.
+ */
+export function pruneQuotaObservations(
+ db: RetentionDb,
+ nowMs: number,
+ budgetMs = QUOTA_OBSERVATIONS_PRUNE_BUDGET_MS,
+): { deleted: number; done: boolean } {
+ const hasTable = !!db
+ .prepare("SELECT name FROM sqlite_master WHERE type='table' AND name='provider_quota_observations'")
+ .get();
+ if (!hasTable) return { deleted: 0, done: true };
+
+ const { retentionDays, maxRows } = getQuotaObservationRetentionConfig();
+ const started = Date.now();
+ const inBudget = () => Date.now() - started < budgetMs;
+ let deleted = 0;
+
+ if (retentionDays > 0) {
+ const cutoff = toSqliteTimestamp(new Date(nowMs - retentionDays * DAY_MS));
+ const byAge = db.prepare(`
+ DELETE FROM provider_quota_observations
+ WHERE rowid IN (
+ SELECT rowid FROM provider_quota_observations
+ WHERE created_at < ?
+ ORDER BY created_at ASC
+ LIMIT ?
+ )
+ `);
+ for (;;) {
+ const changes = byAge.run(cutoff, QUOTA_OBSERVATIONS_PRUNE_CHUNK).changes;
+ deleted += changes;
+ if (changes < QUOTA_OBSERVATIONS_PRUNE_CHUNK) break;
+ if (!inBudget()) return { deleted, done: false };
+ }
+ }
+
+ if (maxRows > 0) {
+ const byCount = db.prepare(`
+ DELETE FROM provider_quota_observations
+ WHERE rowid IN (
+ SELECT rowid
+ FROM provider_quota_observations
+ ORDER BY created_at DESC, rowid DESC
+ LIMIT ? OFFSET ?
+ )
+ `);
+ for (;;) {
+ const changes = byCount.run(QUOTA_OBSERVATIONS_PRUNE_CHUNK, maxRows).changes;
+ deleted += changes;
+ if (changes < QUOTA_OBSERVATIONS_PRUNE_CHUNK) break;
+ if (!inBudget()) return { deleted, done: false };
+ }
+ }
+
+ return { deleted, done: true };
+}
+
/**
* Age- and count-bound the persisted warn/error log rows.
*
@@ -144,6 +241,14 @@ export function pruneRequestAnalytics(options: {
// than the UI range, or the 30d count will silently drop again.
// Guarded against the table being absent (tests that init a DB before the
// migration runs would otherwise crash the prune loop).
+ // Quota observation log, once a day — or again next tick while a large
+ // backlog is still being worked off in budgeted chunks.
+ if (nowMs >= nextQuotaObservationsPruneAtMs) {
+ const sweep = pruneQuotaObservations(db, nowMs);
+ deleted += sweep.deleted;
+ nextQuotaObservationsPruneAtMs = nowMs + (sweep.done ? HOURLY_PRUNE_INTERVAL_MS : PRUNE_INTERVAL_MS);
+ }
+
if (nowMs >= nextHourlyPruneAtMs) {
nextHourlyPruneAtMs = nowMs + HOURLY_PRUNE_INTERVAL_MS;
const hasHourly = !!db