7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.
TypeScript
fix(health): back off scheduled validation after provider 429s (#1377)
Chong Jia Zhen1a536efeedd6
diff --git a/server/src/__tests__/services/health-validation-ratelimit.test.ts b/server/src/__tests__/services/health-validation-ratelimit.test.tsnew file mode 100644index 00000000..a696152d--- /dev/null+++ b/server/src/__tests__/services/health-validation-ratelimit.test.ts@@ -0,0 +1,142 @@+import { describe, it, expect, beforeAll, beforeEach, vi } from 'vitest';++const validateKey = vi.hoisted(() => vi.fn());++vi.mock('../../providers/index.js', () => ({+ resolveProvider: () => ({+ name: 'LLMTR',+ validateKey,+ }),+}));++const { initDb, getDb } = await import('../../db/index.js');+const { encrypt } = await import('../../lib/crypto.js');+const {+ checkAllKeys,+ checkKeyHealth,+ recordValidationRateLimited,+ clearValidationRateLimitState,+} = await import('../../services/health.js');++// #1369: every validation of an LLMTR key is a POST to /chat/completions with+// a nonexistent model — exactly shaped like the bot traffic their abuse+// system hunts. The scheduled pass re-asked a 429ed key every ~5 minutes+// forever, so validation traffic itself became the ban engine. A key whose+// last validation was rate-limited must be skipped until the skip window+// passes; explicit forced passes still probe.++let nextId = 30000;++function seedKey(platform: string): number {+ const id = ++nextId;+ const enc = encrypt(`ratelimit-${id}`);+ getDb().prepare(`+ INSERT INTO api_keys (id, platform, label, encrypted_key, iv, auth_tag, enabled, status, last_checked_at)+ VALUES (?, ?, ?, ?, ?, ?, 1, 'healthy', null)+ `).run(id, platform, `ratelimit-${id}`, enc.encrypted, enc.iv, enc.authTag);+ return id;+}++function fakeClock() {+ const state = { t: 1_000_000 };+ return {+ now: () => state.t,+ sleep: async (ms: number) => { state.t += ms; },+ };+}++beforeAll(() => {+ process.env.ENCRYPTION_KEY = '0'.repeat(64);+ initDb(':memory:');+});++beforeEach(() => {+ getDb().prepare('DELETE FROM api_keys').run();+ clearValidationRateLimitState();+ validateKey.mockReset();+ vi.spyOn(console, 'log').mockImplementation(() => {});+ vi.spyOn(console, 'error').mockImplementation(() => {});+ vi.spyOn(console, 'warn').mockImplementation(() => {});+});++describe('validation rate-limit backoff (#1369)', () => {+ it('skips a key rate-limited within the window, checks the rest', async () => {+ const limited = seedKey('llmtr');+ const other = seedKey('groq');+ const clock = fakeClock();+ recordValidationRateLimited(limited, clock.now());++ const checked: number[] = [];+ const result = await checkAllKeys({+ now: clock.now,+ sleep: clock.sleep,+ concurrency: 1,+ check: async (keyId: number) => { checked.push(keyId); },+ });++ expect(checked).toEqual([other]);+ expect(result.skippedKeyIds).toContain(limited);+ });++ it('re-asks after the window passes, and always on a forced pass', async () => {+ const limited = seedKey('llmtr');+ const clock = fakeClock();+ recordValidationRateLimited(limited, clock.now());++ const hourMs = 60 * 60 * 1000;+ const checked: number[] = [];+ const check = async (keyId: number) => { checked.push(keyId); };++ // Still inside the window: skipped.+ await checkAllKeys({ now: clock.now, sleep: clock.sleep, concurrency: 1, check });+ expect(checked).toEqual([]);++ // Past the window: due again.+ await clock.sleep(hourMs + 1);+ await checkAllKeys({ now: clock.now, sleep: clock.sleep, concurrency: 1, check });+ expect(checked).toEqual([limited]);++ // Forced passes ignore the backoff: explicit operator intent.+ checked.length = 0;+ recordValidationRateLimited(limited, clock.now());+ await checkAllKeys({ now: clock.now, sleep: clock.sleep, concurrency: 1, check, force: true });+ expect(checked).toEqual([limited]);+ });++ it('remembers a 429 from validation and skips the key on the next pass', async () => {+ const id = seedKey('llmtr');+ const err = Object.assign(new Error('Too Many Requests'), { status: 429 });+ validateKey.mockRejectedValue(err);++ await checkKeyHealth(id);+ // checkKeyHealth stamps last_checked_at: backdate it so the recency skip+ // cannot mask the 429 skip under test.+ getDb().prepare("UPDATE api_keys SET last_checked_at = '2000-01-01' WHERE id = ?").run(id);++ // Real clocks on both sides: the 429 mark is recorded with Date.now, so+ // the pass must read it with the same domain. (A fake clock here would+ // skip for the wrong reason — negative time arithmetic, not the window.)+ const checked: number[] = [];+ const result = await checkAllKeys({+ concurrency: 1,+ check: async (keyId: number) => { checked.push(keyId); },+ });+ expect(checked).toEqual([]);+ expect(result.skippedKeyIds).toContain(id);+ });++ it('does not back off on non-429 transport errors', async () => {+ const id = seedKey('llmtr');+ validateKey.mockRejectedValue(new Error('Bearer [REDACTED] failed'));++ await checkKeyHealth(id);+ getDb().prepare("UPDATE api_keys SET last_checked_at = '2000-01-01' WHERE id = ?").run(id);++ const checked: number[] = [];+ await checkAllKeys({+ concurrency: 1,+ check: async (keyId: number) => { checked.push(keyId); },+ });+ expect(checked).toEqual([id]);+ });+});diff --git a/server/src/services/health.ts b/server/src/services/health.tsindex f65f6560..8678752c 100644--- a/server/src/services/health.ts+++ b/server/src/services/health.ts@@ -63,6 +63,36 @@ function positiveIntEnv(raw: string | undefined, fallback: number): number {// Track consecutive failures per keyconst failureCount = new Map<number, number>();+// A key whose last validation was rate-limited (429) is skipped by the+// scheduled pass until the skip window passes: re-asking a provider that just+// told us to slow down is how validation traffic becomes a ban. LLMTR's+// validator is a POST to /chat/completions with a nonexistent model —+// shaped exactly like the bot traffic their abuse system hunts — so an+// unpaced re-ask every 5 minutes per key kept accounts flagged (#1369).+// Forced passes (dashboard check-all, wake re-probe) still probe: explicit+// operator intent wins over pacing. A later successful validation clears the+// mark; validation that never ran leaves no mark.+const validationRateLimitedAt = new Map<number, number>();++const DEFAULT_VALIDATION_RATE_LIMIT_SKIP_MS = 60 * 60 * 1000;++/** How long a 429ed key stays out of the scheduled pass. Tunable like the+ * other pacing knobs; 0 or a bad value falls back to the default. */+function getValidationRateLimitSkipMs(): number {+ return positiveIntEnv(process.env.HEALTH_CHECK_RATE_LIMIT_SKIP_MS, DEFAULT_VALIDATION_RATE_LIMIT_SKIP_MS);+}++/** Record that validating this key was just rate-limited. Exported for tests;+ * production calls it from checkKeyHealth's 429 path. */+export function recordValidationRateLimited(keyId: number, atMs = Date.now()): void {+ validationRateLimitedAt.set(keyId, atMs);+}++/** Test seam: drop all validation-429 pacing state. */+export function clearValidationRateLimitState(): void {+ validationRateLimitedAt.clear();+}+function recordInvalidFailure(keyId: number, platform?: string): void {const count = (failureCount.get(keyId) ?? 0) + 1;failureCount.set(keyId, count);@@ -189,6 +219,9 @@ export async function checkKeyHealth(keyId: number): Promise<KeyStatus> {if (isValid) {failureCount.delete(keyId);+ // The provider answered again: whatever rate-limit told us to back off+ // is over, so the key rejoins the normal cadence.+ validationRateLimitedAt.delete(keyId);// #1348: a healthy key on a built-in platform the catalog does not carry// serves nothing until its models are discovered. Fire-and-forget; the// trigger is a no-op for every other platform, and throttled.@@ -218,6 +251,10 @@ export async function checkKeyHealth(keyId: number): Promise<KeyStatus> {// "[Health] Key N (" prefix is preserved so the 12-hourly crash watchdog// (cron bff5ae167d28) that scrapes /tmp/freellmapi.log for these lines// continues to match unchanged.+ // A 429 here means the provider rate-limited the validation itself:+ // remember it so the scheduled pass backs off instead of re-asking every+ // 5 minutes (#1369). Other transport errors carry no such instruction.+ if (err?.status === 429) recordValidationRateLimited(keyId);const lastError = sanitizeProviderErrorMessage(err?.message ?? err);console.error(`[Health] Key ${keyId} (${row.platform}, base=${row.base_url ?? 'default'}) ` +@@ -393,8 +430,17 @@ async function runHealthPass(opts: HealthPassOptions): Promise<HealthPassResult>`).all() as HealthKeyRow[];const skippedKeyIds: number[] = [];+ const skipMs = opts.force ? 0 : getValidationRateLimitSkipMs();const due = rows.filter(row => {if (opts.force) return true;+ // A key the provider just rate-limited stays out until the skip window+ // passes — even one parked at 'error'. A 429 is fresher, more specific+ // evidence than the eagerness to re-ask an errored key (#1369).+ const limitedAt = validationRateLimitedAt.get(row.id);+ if (limitedAt !== undefined && now() - limitedAt < skipMs) {+ skippedKeyIds.push(row.id);+ return false;+ }// A key parked at 'error' is out of rotation until a probe says otherwise// (the router writes that status, and last_checked_at with it), so it is// never skipped — it is the one key whose verdict is worth re-asking for.