Skip to content

7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.

TypeScript

feat(quota): forecast scoped request rates and exhaustion (#1157)

suantea (asuan_suan)cf6047f08892
diff --git a/server/src/__tests__/services/quota-forecast.test.ts b/server/src/__tests__/services/quota-forecast.test.ts
index c78f8062..25a63f2d 100644
--- a/server/src/__tests__/services/quota-forecast.test.ts
+++ b/server/src/__tests__/services/quota-forecast.test.ts
@@ -18,7 +18,18 @@ function insertState(row: {
`).run(row.platform, row.keyId, row.pool, row.metric, row.limit, row.remaining, row.resetAt);
}
-const future = () => new Date(Date.now() + 12 * 3600 * 1000).toISOString(); // 12h ahead → within today
+function insertRequest(platform: string, isoCreatedAt: string, keyId = 1, modelId = 'groq/llama-3.3') {
+ getDb().prepare(
+ `INSERT INTO requests (platform, model_id, key_id, status, input_tokens, output_tokens, latency_ms, error, request_type, created_at)
+ VALUES (?, ?, ?, 'success', 10, 5, 100, NULL, 'chat', ?)`,
+ ).run(platform, modelId, keyId, isoCreatedAt.slice(0, 19).replace('T', ' '));
+}
+
+function isoPlus(hours: number): string {
+ return new Date(Date.now() + hours * 3600 * 1000).toISOString();
+}
+
+const future = () => isoPlus(12); // 12h ahead → within today
describe('quota-forecast: daily balance aggregation (#1104)', () => {
beforeAll(() => {
@@ -28,6 +39,7 @@ describe('quota-forecast: daily balance aggregation (#1104)', () => {
beforeEach(() => {
getDb().prepare('DELETE FROM provider_quota_state').run();
+ getDb().prepare('DELETE FROM requests').run();
});
it('reports used/remaining/pct/reset for a request pool with a known limit', () => {
@@ -116,4 +128,84 @@ describe('quota-forecast: daily balance aggregation (#1104)', () => {
const forecast = getQuotaForecast();
expect(forecast).toHaveLength(0);
});
+
+ // --- estimated-exhaustion fields (the #1104 extension) ---
+
+ it('returns null rate_exhaustion when the observation window is empty', () => {
+ insertState({ platform: 'groq', keyId: 1, pool: 'groq::account', metric: 'requests', limit: 100, remaining: 80, resetAt: future() });
+ // No recent requests inserted — rate is too sparse to estimate.
+ const e = getQuotaForecast()[0];
+ expect(e.rate_per_min).toBeNull();
+ expect(e.estimated_exhaustion_at).toBeNull();
+ });
+
+ it('estimates exhaustion time once enough recent requests exist', () => {
+ insertState({ platform: 'groq', keyId: 1, pool: 'groq::account', metric: 'requests', limit: 100, remaining: 30, resetAt: future() });
+ // Insert 8 requests spread across the last 10-minute window — above the
+ // 3-request minimum that marks a rate observable.
+ const now = Date.now();
+ for (let i = 1; i <= 8; i++) insertRequest('groq', new Date(now - i * 60_000).toISOString());
+
+ const e = getQuotaForecast()[0];
+ expect(e.rate_per_min).toBeGreaterThan(0);
+ expect(e.estimated_exhaustion_at).toBeTruthy();
+ // The projection must land in the future (not past now).
+ expect(new Date(e.estimated_exhaustion_at!).getTime()).toBeGreaterThan(Date.now());
+ });
+
+ it('returns no exhaustion when the window resets before the quota runs out', () => {
+ insertState({ platform: 'groq', keyId: 1, pool: 'groq::account', metric: 'requests', limit: 10000, remaining: 9000, resetAt: isoPlus(1) });
+ for (let i = 1; i <= 30; i++) insertRequest('groq', new Date(Date.now() - i * 1000).toISOString());
+ const e = getQuotaForecast()[0];
+ expect(e.rate_per_min).toBe(3);
+ expect(e.estimated_exhaustion_at).toBeNull();
+ });
+
+ it('counts only recent traffic from keys observed in the pool', () => {
+ insertState({ platform: 'groq', keyId: 1, pool: 'groq::account', metric: 'requests', limit: 100, remaining: 30, resetAt: future() });
+ for (let i = 0; i < 8; i++) {
+ insertRequest('groq', new Date(Date.now() - 60_000).toISOString(), 1);
+ insertRequest('groq', new Date(Date.now() - 60_000).toISOString(), 2);
+ insertRequest('groq', new Date(Date.now() - 20 * 60_000).toISOString(), 1);
+ insertRequest('groq', new Date(Date.now() + 60_000).toISOString(), 1);
+ }
+ expect(getQuotaForecast()[0].rate_per_min).toBe(0.8);
+ insertState({ platform: 'groq', keyId: 2, pool: 'groq::account', metric: 'requests', limit: 100, remaining: 30, resetAt: future() });
+ expect(getQuotaForecast()[0].rate_per_min).toBe(1.6);
+ });
+
+ it('keeps free and paid model traffic in their own OpenRouter pools', () => {
+ for (const pool of ['openrouter::free', 'openrouter::account']) {
+ insertState({ platform: 'openrouter', keyId: 1, pool, metric: 'requests', limit: 100, remaining: 30, resetAt: future() });
+ }
+ for (let i = 0; i < 5; i++) insertRequest('openrouter', new Date(Date.now() - 60_000).toISOString(), 1, 'model:free');
+ for (let i = 0; i < 9; i++) insertRequest('openrouter', new Date(Date.now() - 60_000).toISOString(), 1, 'model-paid');
+ const forecast = getQuotaForecast();
+ expect(forecast.find(e => e.pool === 'openrouter::free')?.rate_per_min).toBe(0.5);
+ expect(forecast.find(e => e.pool === 'openrouter::account')?.rate_per_min).toBe(0.9);
+ });
+
+ it('ignores an invalid reset timestamp without throwing', () => {
+ insertState({ platform: 'groq', keyId: 1, pool: 'groq::account', metric: 'requests', limit: 100, remaining: 30, resetAt: 'invalid' });
+ for (let i = 0; i < 5; i++) insertRequest('groq', new Date(Date.now() - 60_000).toISOString());
+ expect(getQuotaForecast()[0].estimated_exhaustion_at).toBeNull();
+ });
+
+ it('leaves rate null when remaining is unknown', () => {
+ insertState({ platform: 'groq', keyId: 1, pool: 'groq::account', metric: 'requests', limit: 100, remaining: null, resetAt: future() });
+ // The gate here is `remaining !== null`, not request volume.
+ const e = getQuotaForecast()[0];
+ expect(e.rate_per_min).toBeNull();
+ expect(e.estimated_exhaustion_at).toBeNull();
+ });
+
+ it('excludes exhausted pools from estimation', () => {
+ insertState({ platform: 'groq', keyId: 1, pool: 'groq::account', metric: 'requests', limit: 100, remaining: 0, resetAt: future() });
+ // Even with many requests, remaining=0 → no projection.
+ const now = Date.now();
+ for (let i = 1; i <= 30; i++) insertRequest('groq', new Date(now - i * 1000).toISOString());
+ const e = getQuotaForecast()[0];
+ expect(e.remaining).toBe(0);
+ expect(e.estimated_exhaustion_at).toBeNull();
+ });
});
diff --git a/server/src/services/provider-quota.ts b/server/src/services/provider-quota.ts
index b76345a1..4ec8f5b2 100644
--- a/server/src/services/provider-quota.ts
+++ b/server/src/services/provider-quota.ts
@@ -109,7 +109,7 @@ function pickBetterSource(existing: QuotaObservationSource | null | undefined, n
return SOURCE_PRIORITY[next] >= SOURCE_PRIORITY[existing] ? next : existing;
}
-function inferPoolForPlatform(platform: Platform, modelId?: string | null): string {
+export function inferPoolForPlatform(platform: Platform, modelId?: string | null): string {
const normalizedModelId = modelId?.trim() ?? '';
if (platform === 'openrouter') return normalizedModelId.endsWith(':free') ? 'openrouter::free' : 'openrouter::account';
if (platform === 'google') return 'google::project';
diff --git a/server/src/services/quota-forecast.ts b/server/src/services/quota-forecast.ts
index 582c76d1..dd5b1316 100644
--- a/server/src/services/quota-forecast.ts
+++ b/server/src/services/quota-forecast.ts
@@ -1,5 +1,7 @@
import type { QuotaObservationView } from './provider-quota.js';
-import { getQuotaStateForKeys } from './provider-quota.js';
+import { getQuotaStateForKeys, inferPoolForPlatform } from './provider-quota.js';
+import { getDb } from '../db/index.js';
+import type { Platform } from '@freellmapi/shared/types.js';
// Daily free-tier balance forecast (#1104). Free tiers reset on a per-account
// window (usually UTC midnight) and the only way to know how much headroom is
@@ -21,6 +23,11 @@ export const LOW_BALANCE_ABSOLUTE = 20; // ...or fewer than 20 requests left
// first request onwards and the flag would mean nothing.
export const LOW_BALANCE_ABSOLUTE_MIN_LIMIT = 200;
+// Rate observation window: look at requests in the last N minutes to estimate
+// current consumption speed. Ten minutes covers bursty usage while ignoring
+// sub-hour lulls; shorter windows would jitter, longer ones would lag.
+export const RATE_OBSERVATION_WINDOW_MINUTES = 10;
+
export interface QuotaForecastEntry {
/** Platform the pool belongs to, e.g. 'groq'. */
platform: string;
@@ -43,6 +50,18 @@ export interface QuotaForecastEntry {
low_balance: boolean;
/** Seconds until reset_at, or null when reset_at is unknown/expired. */
seconds_until_reset: number | null;
+
+ // --- estimated-exhaustion fields (the #1104 extension) ---
+
+ /** Observed request rate in the last `RATE_OBSERVATION_WINDOW_MINUTES` minutes,
+ * or null when there is not enough recent request volume to estimate honestly.
+ * Non-null values are rounded to 2 decimal places. */
+ rate_per_min: number | null;
+ /** ISO timestamp of when the current window is expected to be exhausted at the
+ * observed rate, or null when the rate is too low / unknown to predict. A pool
+ * whose window has already expired or whose remaining is unknown is excluded
+ * regardless. */
+ estimated_exhaustion_at: string | null;
}
function secondsUntilReset(resetAt: string | null): number | null {
@@ -52,7 +71,48 @@ function secondsUntilReset(resetAt: string | null): number | null {
return Math.floor(ms / 1000);
}
-function entryFor(row: QuotaObservationView): QuotaForecastEntry | null {
+/** Successful traffic grouped by the same key/pool identity used for observations. */
+function recentRequestCounts(now: number): Map<string, number> {
+ const sqliteUtc = (ms: number) => new Date(ms).toISOString().slice(0, 19).replace('T', ' ');
+ const since = sqliteUtc(now - RATE_OBSERVATION_WINDOW_MINUTES * 60_000);
+ const rows = getDb().prepare(`
+ SELECT platform, key_id AS keyId, model_id AS modelId, COUNT(*) AS count
+ FROM requests
+ WHERE created_at >= ? AND datetime(created_at) >= ? AND datetime(created_at) <= ?
+ AND status = 'success' AND key_id IS NOT NULL
+ GROUP BY platform, key_id, model_id
+ `).all(since, since, sqliteUtc(now)) as { platform: Platform; keyId: number; modelId: string; count: number }[];
+ const counts = new Map<string, number>();
+ for (const row of rows) {
+ const identity = JSON.stringify([row.platform, inferPoolForPlatform(row.platform, row.modelId), row.keyId]);
+ counts.set(identity, (counts.get(identity) ?? 0) + Number(row.count));
+ }
+ return counts;
+}
+
+function estimateExhaustionAt(
+ remaining: number,
+ ratePerMin: number | null,
+ resetAt: string | null,
+ now: number,
+): string | null {
+ // No observed rate → no honest projection. Rates too low to matter are
+ // excluded at the caller level (rate_per_min itself is null).
+ if (ratePerMin === null || ratePerMin <= 0) return null;
+ // Remaining is known (caller checks) and limit is positive (enforced below).
+ const minutesUntilEmpty = remaining / ratePerMin;
+ const secondsFromNow = Math.ceil(minutesUntilEmpty * 60);
+ const ms = now + secondsFromNow * 1000;
+ // Never project past the natural window reset: if the window expires first,
+ // we have headroom even if the request count would otherwise exhaust the
+ // pool — the quota header tracks the remaining *before* reset, not absolute
+ // spending.
+ const resetMs = resetAt ? new Date(resetAt).getTime() : Infinity;
+ if (!Number.isFinite(ms) || Number.isNaN(resetMs) || ms >= resetMs || ms <= now) return null;
+ return new Date(ms).toISOString();
+}
+
+function entryFor(row: QuotaObservationView, ratePerMin: number | null, now: number): QuotaForecastEntry | null {
// Only request-based windows are predictable from quota headers; token pools
// reset semantics vary too much across providers to forecast honestly.
if (row.metric !== 'requests') return null;
@@ -74,6 +134,13 @@ function entryFor(row: QuotaObservationView): QuotaForecastEntry | null {
|| remaining / limit < LOW_BALANCE_THRESHOLD;
}
+ // Rate projection is gated on remaining being known (unknown remaining makes
+ // any rate look like a free lunch until the next quota refresh).
+ if (remaining === null) ratePerMin = null;
+ const estimatedExhaustionAt = (remaining !== null && ratePerMin !== null && remaining > 0)
+ ? estimateExhaustionAt(remaining, ratePerMin, row.resetAt ?? null, now)
+ : null;
+
return {
platform: row.platform,
pool: row.quotaPoolKey ?? `${row.platform}::default`,
@@ -84,6 +151,8 @@ function entryFor(row: QuotaObservationView): QuotaForecastEntry | null {
reset_at: row.resetAt ?? null,
low_balance: lowBalance,
seconds_until_reset: secondsUntilReset(row.resetAt ?? null),
+ rate_per_min: ratePerMin,
+ estimated_exhaustion_at: estimatedExhaustionAt,
};
}
@@ -92,8 +161,23 @@ function entryFor(row: QuotaObservationView): QuotaForecastEntry | null {
// matters for "can I keep calling" is the least headroom left.
export function getQuotaForecast(): QuotaForecastEntry[] {
const byKey = new Map<string, QuotaForecastEntry>();
- for (const row of getQuotaStateForKeys()) {
- const entry = entryFor(row);
+ const rows = getQuotaStateForKeys();
+ const now = Date.now();
+ const counts = rows.length ? recentRequestCounts(now) : new Map<string, number>();
+ const poolKeys = new Map<string, Set<string>>();
+ for (const row of rows) {
+ if (row.metric !== 'requests') continue;
+ const pool = JSON.stringify([row.platform, row.quotaPoolKey]);
+ const keys = poolKeys.get(pool) ?? new Set<string>();
+ keys.add(JSON.stringify([row.platform, row.quotaPoolKey, row.keyId]));
+ poolKeys.set(pool, keys);
+ }
+ for (const row of rows) {
+ const keys = poolKeys.get(JSON.stringify([row.platform, row.quotaPoolKey])) ?? [];
+ let count = 0;
+ for (const key of keys) count += counts.get(key) ?? 0;
+ const rate = count < 3 ? null : Math.round(count / RATE_OBSERVATION_WINDOW_MINUTES * 100) / 100;
+ const entry = entryFor(row, rate, now);
if (!entry) continue;
// `pool` already carries its platform ("groq::account"), so it is the key.
const key = entry.pool;