7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.
TypeScript
feat(quota): forecast scoped request rates and exhaustion (#1157)
suantea (asuan_suan)cf6047f08892
diff --git a/server/src/__tests__/services/quota-forecast.test.ts b/server/src/__tests__/services/quota-forecast.test.tsindex c78f8062..25a63f2d 100644--- a/server/src/__tests__/services/quota-forecast.test.ts+++ b/server/src/__tests__/services/quota-forecast.test.ts@@ -18,7 +18,18 @@ function insertState(row: {`).run(row.platform, row.keyId, row.pool, row.metric, row.limit, row.remaining, row.resetAt);}-const future = () => new Date(Date.now() + 12 * 3600 * 1000).toISOString(); // 12h ahead → within today+function insertRequest(platform: string, isoCreatedAt: string, keyId = 1, modelId = 'groq/llama-3.3') {+ getDb().prepare(+ `INSERT INTO requests (platform, model_id, key_id, status, input_tokens, output_tokens, latency_ms, error, request_type, created_at)+ VALUES (?, ?, ?, 'success', 10, 5, 100, NULL, 'chat', ?)`,+ ).run(platform, modelId, keyId, isoCreatedAt.slice(0, 19).replace('T', ' '));+}++function isoPlus(hours: number): string {+ return new Date(Date.now() + hours * 3600 * 1000).toISOString();+}++const future = () => isoPlus(12); // 12h ahead → within todaydescribe('quota-forecast: daily balance aggregation (#1104)', () => {beforeAll(() => {@@ -28,6 +39,7 @@ describe('quota-forecast: daily balance aggregation (#1104)', () => {beforeEach(() => {getDb().prepare('DELETE FROM provider_quota_state').run();+ getDb().prepare('DELETE FROM requests').run();});it('reports used/remaining/pct/reset for a request pool with a known limit', () => {@@ -116,4 +128,84 @@ describe('quota-forecast: daily balance aggregation (#1104)', () => {const forecast = getQuotaForecast();expect(forecast).toHaveLength(0);});++ // --- estimated-exhaustion fields (the #1104 extension) ---++ it('returns null rate_exhaustion when the observation window is empty', () => {+ insertState({ platform: 'groq', keyId: 1, pool: 'groq::account', metric: 'requests', limit: 100, remaining: 80, resetAt: future() });+ // No recent requests inserted — rate is too sparse to estimate.+ const e = getQuotaForecast()[0];+ expect(e.rate_per_min).toBeNull();+ expect(e.estimated_exhaustion_at).toBeNull();+ });++ it('estimates exhaustion time once enough recent requests exist', () => {+ insertState({ platform: 'groq', keyId: 1, pool: 'groq::account', metric: 'requests', limit: 100, remaining: 30, resetAt: future() });+ // Insert 8 requests spread across the last 10-minute window — above the+ // 3-request minimum that marks a rate observable.+ const now = Date.now();+ for (let i = 1; i <= 8; i++) insertRequest('groq', new Date(now - i * 60_000).toISOString());++ const e = getQuotaForecast()[0];+ expect(e.rate_per_min).toBeGreaterThan(0);+ expect(e.estimated_exhaustion_at).toBeTruthy();+ // The projection must land in the future (not past now).+ expect(new Date(e.estimated_exhaustion_at!).getTime()).toBeGreaterThan(Date.now());+ });++ it('returns no exhaustion when the window resets before the quota runs out', () => {+ insertState({ platform: 'groq', keyId: 1, pool: 'groq::account', metric: 'requests', limit: 10000, remaining: 9000, resetAt: isoPlus(1) });+ for (let i = 1; i <= 30; i++) insertRequest('groq', new Date(Date.now() - i * 1000).toISOString());+ const e = getQuotaForecast()[0];+ expect(e.rate_per_min).toBe(3);+ expect(e.estimated_exhaustion_at).toBeNull();+ });++ it('counts only recent traffic from keys observed in the pool', () => {+ insertState({ platform: 'groq', keyId: 1, pool: 'groq::account', metric: 'requests', limit: 100, remaining: 30, resetAt: future() });+ for (let i = 0; i < 8; i++) {+ insertRequest('groq', new Date(Date.now() - 60_000).toISOString(), 1);+ insertRequest('groq', new Date(Date.now() - 60_000).toISOString(), 2);+ insertRequest('groq', new Date(Date.now() - 20 * 60_000).toISOString(), 1);+ insertRequest('groq', new Date(Date.now() + 60_000).toISOString(), 1);+ }+ expect(getQuotaForecast()[0].rate_per_min).toBe(0.8);+ insertState({ platform: 'groq', keyId: 2, pool: 'groq::account', metric: 'requests', limit: 100, remaining: 30, resetAt: future() });+ expect(getQuotaForecast()[0].rate_per_min).toBe(1.6);+ });++ it('keeps free and paid model traffic in their own OpenRouter pools', () => {+ for (const pool of ['openrouter::free', 'openrouter::account']) {+ insertState({ platform: 'openrouter', keyId: 1, pool, metric: 'requests', limit: 100, remaining: 30, resetAt: future() });+ }+ for (let i = 0; i < 5; i++) insertRequest('openrouter', new Date(Date.now() - 60_000).toISOString(), 1, 'model:free');+ for (let i = 0; i < 9; i++) insertRequest('openrouter', new Date(Date.now() - 60_000).toISOString(), 1, 'model-paid');+ const forecast = getQuotaForecast();+ expect(forecast.find(e => e.pool === 'openrouter::free')?.rate_per_min).toBe(0.5);+ expect(forecast.find(e => e.pool === 'openrouter::account')?.rate_per_min).toBe(0.9);+ });++ it('ignores an invalid reset timestamp without throwing', () => {+ insertState({ platform: 'groq', keyId: 1, pool: 'groq::account', metric: 'requests', limit: 100, remaining: 30, resetAt: 'invalid' });+ for (let i = 0; i < 5; i++) insertRequest('groq', new Date(Date.now() - 60_000).toISOString());+ expect(getQuotaForecast()[0].estimated_exhaustion_at).toBeNull();+ });++ it('leaves rate null when remaining is unknown', () => {+ insertState({ platform: 'groq', keyId: 1, pool: 'groq::account', metric: 'requests', limit: 100, remaining: null, resetAt: future() });+ // The gate here is `remaining !== null`, not request volume.+ const e = getQuotaForecast()[0];+ expect(e.rate_per_min).toBeNull();+ expect(e.estimated_exhaustion_at).toBeNull();+ });++ it('excludes exhausted pools from estimation', () => {+ insertState({ platform: 'groq', keyId: 1, pool: 'groq::account', metric: 'requests', limit: 100, remaining: 0, resetAt: future() });+ // Even with many requests, remaining=0 → no projection.+ const now = Date.now();+ for (let i = 1; i <= 30; i++) insertRequest('groq', new Date(now - i * 1000).toISOString());+ const e = getQuotaForecast()[0];+ expect(e.remaining).toBe(0);+ expect(e.estimated_exhaustion_at).toBeNull();+ });});diff --git a/server/src/services/provider-quota.ts b/server/src/services/provider-quota.tsindex b76345a1..4ec8f5b2 100644--- a/server/src/services/provider-quota.ts+++ b/server/src/services/provider-quota.ts@@ -109,7 +109,7 @@ function pickBetterSource(existing: QuotaObservationSource | null | undefined, nreturn SOURCE_PRIORITY[next] >= SOURCE_PRIORITY[existing] ? next : existing;}-function inferPoolForPlatform(platform: Platform, modelId?: string | null): string {+export function inferPoolForPlatform(platform: Platform, modelId?: string | null): string {const normalizedModelId = modelId?.trim() ?? '';if (platform === 'openrouter') return normalizedModelId.endsWith(':free') ? 'openrouter::free' : 'openrouter::account';if (platform === 'google') return 'google::project';diff --git a/server/src/services/quota-forecast.ts b/server/src/services/quota-forecast.tsindex 582c76d1..dd5b1316 100644--- a/server/src/services/quota-forecast.ts+++ b/server/src/services/quota-forecast.ts@@ -1,5 +1,7 @@import type { QuotaObservationView } from './provider-quota.js';-import { getQuotaStateForKeys } from './provider-quota.js';+import { getQuotaStateForKeys, inferPoolForPlatform } from './provider-quota.js';+import { getDb } from '../db/index.js';+import type { Platform } from '@freellmapi/shared/types.js';// Daily free-tier balance forecast (#1104). Free tiers reset on a per-account// window (usually UTC midnight) and the only way to know how much headroom is@@ -21,6 +23,11 @@ export const LOW_BALANCE_ABSOLUTE = 20; // ...or fewer than 20 requests left// first request onwards and the flag would mean nothing.export const LOW_BALANCE_ABSOLUTE_MIN_LIMIT = 200;+// Rate observation window: look at requests in the last N minutes to estimate+// current consumption speed. Ten minutes covers bursty usage while ignoring+// sub-hour lulls; shorter windows would jitter, longer ones would lag.+export const RATE_OBSERVATION_WINDOW_MINUTES = 10;+export interface QuotaForecastEntry {/** Platform the pool belongs to, e.g. 'groq'. */platform: string;@@ -43,6 +50,18 @@ export interface QuotaForecastEntry {low_balance: boolean;/** Seconds until reset_at, or null when reset_at is unknown/expired. */seconds_until_reset: number | null;++ // --- estimated-exhaustion fields (the #1104 extension) ---++ /** Observed request rate in the last `RATE_OBSERVATION_WINDOW_MINUTES` minutes,+ * or null when there is not enough recent request volume to estimate honestly.+ * Non-null values are rounded to 2 decimal places. */+ rate_per_min: number | null;+ /** ISO timestamp of when the current window is expected to be exhausted at the+ * observed rate, or null when the rate is too low / unknown to predict. A pool+ * whose window has already expired or whose remaining is unknown is excluded+ * regardless. */+ estimated_exhaustion_at: string | null;}function secondsUntilReset(resetAt: string | null): number | null {@@ -52,7 +71,48 @@ function secondsUntilReset(resetAt: string | null): number | null {return Math.floor(ms / 1000);}-function entryFor(row: QuotaObservationView): QuotaForecastEntry | null {+/** Successful traffic grouped by the same key/pool identity used for observations. */+function recentRequestCounts(now: number): Map<string, number> {+ const sqliteUtc = (ms: number) => new Date(ms).toISOString().slice(0, 19).replace('T', ' ');+ const since = sqliteUtc(now - RATE_OBSERVATION_WINDOW_MINUTES * 60_000);+ const rows = getDb().prepare(`+ SELECT platform, key_id AS keyId, model_id AS modelId, COUNT(*) AS count+ FROM requests+ WHERE created_at >= ? AND datetime(created_at) >= ? AND datetime(created_at) <= ?+ AND status = 'success' AND key_id IS NOT NULL+ GROUP BY platform, key_id, model_id+ `).all(since, since, sqliteUtc(now)) as { platform: Platform; keyId: number; modelId: string; count: number }[];+ const counts = new Map<string, number>();+ for (const row of rows) {+ const identity = JSON.stringify([row.platform, inferPoolForPlatform(row.platform, row.modelId), row.keyId]);+ counts.set(identity, (counts.get(identity) ?? 0) + Number(row.count));+ }+ return counts;+}++function estimateExhaustionAt(+ remaining: number,+ ratePerMin: number | null,+ resetAt: string | null,+ now: number,+): string | null {+ // No observed rate → no honest projection. Rates too low to matter are+ // excluded at the caller level (rate_per_min itself is null).+ if (ratePerMin === null || ratePerMin <= 0) return null;+ // Remaining is known (caller checks) and limit is positive (enforced below).+ const minutesUntilEmpty = remaining / ratePerMin;+ const secondsFromNow = Math.ceil(minutesUntilEmpty * 60);+ const ms = now + secondsFromNow * 1000;+ // Never project past the natural window reset: if the window expires first,+ // we have headroom even if the request count would otherwise exhaust the+ // pool — the quota header tracks the remaining *before* reset, not absolute+ // spending.+ const resetMs = resetAt ? new Date(resetAt).getTime() : Infinity;+ if (!Number.isFinite(ms) || Number.isNaN(resetMs) || ms >= resetMs || ms <= now) return null;+ return new Date(ms).toISOString();+}++function entryFor(row: QuotaObservationView, ratePerMin: number | null, now: number): QuotaForecastEntry | null {// Only request-based windows are predictable from quota headers; token pools// reset semantics vary too much across providers to forecast honestly.if (row.metric !== 'requests') return null;@@ -74,6 +134,13 @@ function entryFor(row: QuotaObservationView): QuotaForecastEntry | null {|| remaining / limit < LOW_BALANCE_THRESHOLD;}+ // Rate projection is gated on remaining being known (unknown remaining makes+ // any rate look like a free lunch until the next quota refresh).+ if (remaining === null) ratePerMin = null;+ const estimatedExhaustionAt = (remaining !== null && ratePerMin !== null && remaining > 0)+ ? estimateExhaustionAt(remaining, ratePerMin, row.resetAt ?? null, now)+ : null;+return {platform: row.platform,pool: row.quotaPoolKey ?? `${row.platform}::default`,@@ -84,6 +151,8 @@ function entryFor(row: QuotaObservationView): QuotaForecastEntry | null {reset_at: row.resetAt ?? null,low_balance: lowBalance,seconds_until_reset: secondsUntilReset(row.resetAt ?? null),+ rate_per_min: ratePerMin,+ estimated_exhaustion_at: estimatedExhaustionAt,};}@@ -92,8 +161,23 @@ function entryFor(row: QuotaObservationView): QuotaForecastEntry | null {// matters for "can I keep calling" is the least headroom left.export function getQuotaForecast(): QuotaForecastEntry[] {const byKey = new Map<string, QuotaForecastEntry>();- for (const row of getQuotaStateForKeys()) {- const entry = entryFor(row);+ const rows = getQuotaStateForKeys();+ const now = Date.now();+ const counts = rows.length ? recentRequestCounts(now) : new Map<string, number>();+ const poolKeys = new Map<string, Set<string>>();+ for (const row of rows) {+ if (row.metric !== 'requests') continue;+ const pool = JSON.stringify([row.platform, row.quotaPoolKey]);+ const keys = poolKeys.get(pool) ?? new Set<string>();+ keys.add(JSON.stringify([row.platform, row.quotaPoolKey, row.keyId]));+ poolKeys.set(pool, keys);+ }+ for (const row of rows) {+ const keys = poolKeys.get(JSON.stringify([row.platform, row.quotaPoolKey])) ?? [];+ let count = 0;+ for (const key of keys) count += counts.get(key) ?? 0;+ const rate = count < 3 ? null : Math.round(count / RATE_OBSERVATION_WINDOW_MINUTES * 100) / 100;+ const entry = entryFor(row, rate, now);if (!entry) continue;// `pool` already carries its platform ("groq::account"), so it is the key.const key = entry.pool;