Skip to content

7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.

TypeScript

fix(health): back off scheduled validation after provider 429s (#1377)

Chong Jia Zhen1a536efeedd6
diff --git a/server/src/__tests__/services/health-validation-ratelimit.test.ts b/server/src/__tests__/services/health-validation-ratelimit.test.ts
new file mode 100644
index 00000000..a696152d
--- /dev/null
+++ b/server/src/__tests__/services/health-validation-ratelimit.test.ts
@@ -0,0 +1,142 @@
+import { describe, it, expect, beforeAll, beforeEach, vi } from 'vitest';
+
+const validateKey = vi.hoisted(() => vi.fn());
+
+vi.mock('../../providers/index.js', () => ({
+ resolveProvider: () => ({
+ name: 'LLMTR',
+ validateKey,
+ }),
+}));
+
+const { initDb, getDb } = await import('../../db/index.js');
+const { encrypt } = await import('../../lib/crypto.js');
+const {
+ checkAllKeys,
+ checkKeyHealth,
+ recordValidationRateLimited,
+ clearValidationRateLimitState,
+} = await import('../../services/health.js');
+
+// #1369: every validation of an LLMTR key is a POST to /chat/completions with
+// a nonexistent model — exactly shaped like the bot traffic their abuse
+// system hunts. The scheduled pass re-asked a 429ed key every ~5 minutes
+// forever, so validation traffic itself became the ban engine. A key whose
+// last validation was rate-limited must be skipped until the skip window
+// passes; explicit forced passes still probe.
+
+let nextId = 30000;
+
+function seedKey(platform: string): number {
+ const id = ++nextId;
+ const enc = encrypt(`ratelimit-${id}`);
+ getDb().prepare(`
+ INSERT INTO api_keys (id, platform, label, encrypted_key, iv, auth_tag, enabled, status, last_checked_at)
+ VALUES (?, ?, ?, ?, ?, ?, 1, 'healthy', null)
+ `).run(id, platform, `ratelimit-${id}`, enc.encrypted, enc.iv, enc.authTag);
+ return id;
+}
+
+function fakeClock() {
+ const state = { t: 1_000_000 };
+ return {
+ now: () => state.t,
+ sleep: async (ms: number) => { state.t += ms; },
+ };
+}
+
+beforeAll(() => {
+ process.env.ENCRYPTION_KEY = '0'.repeat(64);
+ initDb(':memory:');
+});
+
+beforeEach(() => {
+ getDb().prepare('DELETE FROM api_keys').run();
+ clearValidationRateLimitState();
+ validateKey.mockReset();
+ vi.spyOn(console, 'log').mockImplementation(() => {});
+ vi.spyOn(console, 'error').mockImplementation(() => {});
+ vi.spyOn(console, 'warn').mockImplementation(() => {});
+});
+
+describe('validation rate-limit backoff (#1369)', () => {
+ it('skips a key rate-limited within the window, checks the rest', async () => {
+ const limited = seedKey('llmtr');
+ const other = seedKey('groq');
+ const clock = fakeClock();
+ recordValidationRateLimited(limited, clock.now());
+
+ const checked: number[] = [];
+ const result = await checkAllKeys({
+ now: clock.now,
+ sleep: clock.sleep,
+ concurrency: 1,
+ check: async (keyId: number) => { checked.push(keyId); },
+ });
+
+ expect(checked).toEqual([other]);
+ expect(result.skippedKeyIds).toContain(limited);
+ });
+
+ it('re-asks after the window passes, and always on a forced pass', async () => {
+ const limited = seedKey('llmtr');
+ const clock = fakeClock();
+ recordValidationRateLimited(limited, clock.now());
+
+ const hourMs = 60 * 60 * 1000;
+ const checked: number[] = [];
+ const check = async (keyId: number) => { checked.push(keyId); };
+
+ // Still inside the window: skipped.
+ await checkAllKeys({ now: clock.now, sleep: clock.sleep, concurrency: 1, check });
+ expect(checked).toEqual([]);
+
+ // Past the window: due again.
+ await clock.sleep(hourMs + 1);
+ await checkAllKeys({ now: clock.now, sleep: clock.sleep, concurrency: 1, check });
+ expect(checked).toEqual([limited]);
+
+ // Forced passes ignore the backoff: explicit operator intent.
+ checked.length = 0;
+ recordValidationRateLimited(limited, clock.now());
+ await checkAllKeys({ now: clock.now, sleep: clock.sleep, concurrency: 1, check, force: true });
+ expect(checked).toEqual([limited]);
+ });
+
+ it('remembers a 429 from validation and skips the key on the next pass', async () => {
+ const id = seedKey('llmtr');
+ const err = Object.assign(new Error('Too Many Requests'), { status: 429 });
+ validateKey.mockRejectedValue(err);
+
+ await checkKeyHealth(id);
+ // checkKeyHealth stamps last_checked_at: backdate it so the recency skip
+ // cannot mask the 429 skip under test.
+ getDb().prepare("UPDATE api_keys SET last_checked_at = '2000-01-01' WHERE id = ?").run(id);
+
+ // Real clocks on both sides: the 429 mark is recorded with Date.now, so
+ // the pass must read it with the same domain. (A fake clock here would
+ // skip for the wrong reason — negative time arithmetic, not the window.)
+ const checked: number[] = [];
+ const result = await checkAllKeys({
+ concurrency: 1,
+ check: async (keyId: number) => { checked.push(keyId); },
+ });
+ expect(checked).toEqual([]);
+ expect(result.skippedKeyIds).toContain(id);
+ });
+
+ it('does not back off on non-429 transport errors', async () => {
+ const id = seedKey('llmtr');
+ validateKey.mockRejectedValue(new Error('Bearer [REDACTED] failed'));
+
+ await checkKeyHealth(id);
+ getDb().prepare("UPDATE api_keys SET last_checked_at = '2000-01-01' WHERE id = ?").run(id);
+
+ const checked: number[] = [];
+ await checkAllKeys({
+ concurrency: 1,
+ check: async (keyId: number) => { checked.push(keyId); },
+ });
+ expect(checked).toEqual([id]);
+ });
+});
diff --git a/server/src/services/health.ts b/server/src/services/health.ts
index f65f6560..8678752c 100644
--- a/server/src/services/health.ts
+++ b/server/src/services/health.ts
@@ -63,6 +63,36 @@ function positiveIntEnv(raw: string | undefined, fallback: number): number {
// Track consecutive failures per key
const failureCount = new Map<number, number>();
+// A key whose last validation was rate-limited (429) is skipped by the
+// scheduled pass until the skip window passes: re-asking a provider that just
+// told us to slow down is how validation traffic becomes a ban. LLMTR's
+// validator is a POST to /chat/completions with a nonexistent model —
+// shaped exactly like the bot traffic their abuse system hunts — so an
+// unpaced re-ask every 5 minutes per key kept accounts flagged (#1369).
+// Forced passes (dashboard check-all, wake re-probe) still probe: explicit
+// operator intent wins over pacing. A later successful validation clears the
+// mark; validation that never ran leaves no mark.
+const validationRateLimitedAt = new Map<number, number>();
+
+const DEFAULT_VALIDATION_RATE_LIMIT_SKIP_MS = 60 * 60 * 1000;
+
+/** How long a 429ed key stays out of the scheduled pass. Tunable like the
+ * other pacing knobs; 0 or a bad value falls back to the default. */
+function getValidationRateLimitSkipMs(): number {
+ return positiveIntEnv(process.env.HEALTH_CHECK_RATE_LIMIT_SKIP_MS, DEFAULT_VALIDATION_RATE_LIMIT_SKIP_MS);
+}
+
+/** Record that validating this key was just rate-limited. Exported for tests;
+ * production calls it from checkKeyHealth's 429 path. */
+export function recordValidationRateLimited(keyId: number, atMs = Date.now()): void {
+ validationRateLimitedAt.set(keyId, atMs);
+}
+
+/** Test seam: drop all validation-429 pacing state. */
+export function clearValidationRateLimitState(): void {
+ validationRateLimitedAt.clear();
+}
+
function recordInvalidFailure(keyId: number, platform?: string): void {
const count = (failureCount.get(keyId) ?? 0) + 1;
failureCount.set(keyId, count);
@@ -189,6 +219,9 @@ export async function checkKeyHealth(keyId: number): Promise<KeyStatus> {
if (isValid) {
failureCount.delete(keyId);
+ // The provider answered again: whatever rate-limit told us to back off
+ // is over, so the key rejoins the normal cadence.
+ validationRateLimitedAt.delete(keyId);
// #1348: a healthy key on a built-in platform the catalog does not carry
// serves nothing until its models are discovered. Fire-and-forget; the
// trigger is a no-op for every other platform, and throttled.
@@ -218,6 +251,10 @@ export async function checkKeyHealth(keyId: number): Promise<KeyStatus> {
// "[Health] Key N (" prefix is preserved so the 12-hourly crash watchdog
// (cron bff5ae167d28) that scrapes /tmp/freellmapi.log for these lines
// continues to match unchanged.
+ // A 429 here means the provider rate-limited the validation itself:
+ // remember it so the scheduled pass backs off instead of re-asking every
+ // 5 minutes (#1369). Other transport errors carry no such instruction.
+ if (err?.status === 429) recordValidationRateLimited(keyId);
const lastError = sanitizeProviderErrorMessage(err?.message ?? err);
console.error(
`[Health] Key ${keyId} (${row.platform}, base=${row.base_url ?? 'default'}) ` +
@@ -393,8 +430,17 @@ async function runHealthPass(opts: HealthPassOptions): Promise<HealthPassResult>
`).all() as HealthKeyRow[];
const skippedKeyIds: number[] = [];
+ const skipMs = opts.force ? 0 : getValidationRateLimitSkipMs();
const due = rows.filter(row => {
if (opts.force) return true;
+ // A key the provider just rate-limited stays out until the skip window
+ // passes — even one parked at 'error'. A 429 is fresher, more specific
+ // evidence than the eagerness to re-ask an errored key (#1369).
+ const limitedAt = validationRateLimitedAt.get(row.id);
+ if (limitedAt !== undefined && now() - limitedAt < skipMs) {
+ skippedKeyIds.push(row.id);
+ return false;
+ }
// A key parked at 'error' is out of rotation until a probe says otherwise
// (the router writes that status, and last_checked_at with it), so it is
// never skipped — it is the one key whose verdict is worth re-asking for.