7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.
TypeScript
fix(llmtr): stop probing LLMTR keys; promote them on first served request (#1459) (#1471)
Tashfeenee8d8bd4cddb
diff --git a/server/src/__tests__/providers/llmtr.test.ts b/server/src/__tests__/providers/llmtr.test.tsindex d2c2c75f..d1673714 100644--- a/server/src/__tests__/providers/llmtr.test.ts+++ b/server/src/__tests__/providers/llmtr.test.ts@@ -81,6 +81,11 @@ describe('LLMTR provider', () => {await expect(unpaced().validateKey('test-key')).rejects.toMatchObject({ status, retryAfterMs: 17_000 });});+ it('#1459: opts out of automated key validation; other providers keep it', () => {+ expect(getProvider('llmtr')!.supportsKeyValidation).toBe(false);+ expect(getProvider('groq')!.supportsKeyValidation).toBe(true);+ });+it('#1390: a 403 refusing automated validation probes is inconclusive, never invalid', async () => {// Live LLMTR response to our probe on 2026-10-03, reported on a working key.vi.spyOn(global, 'fetch').mockResolvedValue(json({ error: { type: 'forbidden', message: 'Automated API key validation tools are not supported.' } }, 403));diff --git a/server/src/__tests__/services/health-transport-preserve.test.ts b/server/src/__tests__/services/health-transport-preserve.test.tsindex 61018c2e..34a05e2c 100644--- a/server/src/__tests__/services/health-transport-preserve.test.ts+++ b/server/src/__tests__/services/health-transport-preserve.test.ts@@ -117,6 +117,12 @@ describe('markKeyHealthyFromRequest', () => {expect(row(id)).toEqual({ status: 'healthy', last_health_error: null });});+ it("promotes a never-probed 'unknown' key to healthy (#1459)", () => {+ const id = seedKey('unknown');+ markKeyHealthyFromRequest(id);+ expect(row(id)).toEqual({ status: 'healthy', last_health_error: null });+ });+it("never resurrects an 'invalid' key — only a real probe clears that", () => {const id = seedKey('invalid');markKeyHealthyFromRequest(id);diff --git a/server/src/__tests__/services/health-unverifiable-provider.test.ts b/server/src/__tests__/services/health-unverifiable-provider.test.tsnew file mode 100644index 00000000..269db777--- /dev/null+++ b/server/src/__tests__/services/health-unverifiable-provider.test.ts@@ -0,0 +1,79 @@+import { describe, it, expect, beforeAll, beforeEach, vi } from 'vitest';++// #1459: LLMTR refuses automated key checks and suspends accounts that keep+// sending them, so a provider with supportsKeyValidation === false must never+// be probed by the health pass or the cooldown probe.+const validateKey = vi.fn();+vi.mock('../../providers/index.js', async (importOriginal) => {+ const actual = await importOriginal() as any;+ return {+ ...actual,+ resolveProvider: () => ({ name: 'no-probe', supportsKeyValidation: false, validateKey }),+ };+});++const { initDb, getDb } = await import('../../db/index.js');+const { encrypt } = await import('../../lib/crypto.js');+const { checkKeyHealth, checkAllKeys, probeKeyValidity, markKeyHealthyFromRequest } = await import('../../services/health.js');++describe('a provider that forbids key validation (#1459)', () => {+ let nextId = 9100;++ beforeAll(() => {+ process.env.ENCRYPTION_KEY = '0'.repeat(64);+ initDb(':memory:');+ });++ beforeEach(() => {+ validateKey.mockReset();+ vi.spyOn(console, 'error').mockImplementation(() => {});+ vi.spyOn(console, 'warn').mockImplementation(() => {});+ vi.spyOn(console, 'log').mockImplementation(() => {});+ });++ function seedKey(status: string, lastError: string | null = null): number {+ const id = ++nextId;+ const enc = encrypt(`no-probe-${id}`);+ getDb().prepare(`+ INSERT INTO api_keys (id, platform, label, encrypted_key, iv, auth_tag, enabled, status, last_health_error)+ VALUES (?, 'llmtr', ?, ?, ?, ?, 1, ?, ?)+ `).run(id, `no-probe-${id}`, enc.encrypted, enc.iv, enc.authTag, status, lastError);+ return id;+ }++ function row(id: number): { status: string; last_health_error: string | null } {+ return getDb().prepare('SELECT status, last_health_error FROM api_keys WHERE id = ?').get(id) as any;+ }++ it('checkKeyHealth sends no probe, keeps the status and clears the stored probe error', async () => {+ const id = seedKey('unknown', 'LLMTR rejected the key-validation probe itself (automated validation not supported); the key was not checked');++ expect(await checkKeyHealth(id)).toBe('unknown');+ expect(validateKey).not.toHaveBeenCalled();+ expect(row(id)).toEqual({ status: 'unknown', last_health_error: null });+ });++ it('a healthy key stays healthy through the scheduled pass with no probe sent', async () => {+ const id = seedKey('healthy');++ await checkAllKeys({ force: true });++ expect(validateKey).not.toHaveBeenCalled();+ expect(row(id).status).toBe('healthy');+ });++ it('the cooldown probe treats the key as inconclusive without probing', async () => {+ const id = seedKey('healthy');++ expect(await probeKeyValidity(id)).toBe('error');+ expect(validateKey).not.toHaveBeenCalled();+ });++ it('the first served request promotes the key to healthy', () => {+ const id = seedKey('unknown');++ markKeyHealthyFromRequest(id);++ expect(row(id)).toEqual({ status: 'healthy', last_health_error: null });+ });+});diff --git a/server/src/providers/base.ts b/server/src/providers/base.tsindex d4efd92d..27151f9a 100644--- a/server/src/providers/base.ts+++ b/server/src/providers/base.ts@@ -268,6 +268,17 @@ export abstract class BaseProvider {return false;}+ /** Whether the health checker may probe this provider's keys at all (#1459).+ * A provider that forbids automated key checks (LLMTR answers any probe with+ * 403 "automated API key validation tools are not supported", and its+ * security system suspends accounts that keep sending them) returns false:+ * the health pass and the cooldown probe then never call validateKey for its+ * keys, and a key is promoted to healthy by the first request it serves+ * (markKeyHealthyFromRequest) instead of by a probe. */+ get supportsKeyValidation(): boolean {+ return true;+ }+/*** Turn a conventional 401/403 validation response into a diagnostic result.* Providers still return a simple boolean when no useful error body exists,diff --git a/server/src/providers/llmtr.ts b/server/src/providers/llmtr.tsindex 0ae09167..6750fe12 100644--- a/server/src/providers/llmtr.ts+++ b/server/src/providers/llmtr.ts@@ -76,6 +76,13 @@ export class LlmtrProvider extends OpenAICompatProvider {* overlap easily) share one request instead of sending two identical ones. */private readonly inFlightValidations = new Map<string, Promise<KeyValidationResult>>();+ /** #1459: LLMTR refuses automated key checks outright, so the health pass+ * and the cooldown probe never send one. Keys turn healthy on their first+ * served request instead. validateKey below stays for explicit callers. */+ override get supportsKeyValidation(): boolean {+ return false;+ }+constructor(opts: LlmtrProviderOptions = {}) {super({ platform: 'llmtr', name: 'LLMTR', baseUrl: LLMTR_BASE_URL });this.validationMinGapMs = opts.validationMinGapMs ?? validationMinGapMs();diff --git a/server/src/services/health.ts b/server/src/services/health.tsindex 8678752c..35a0c425 100644--- a/server/src/services/health.ts+++ b/server/src/services/health.ts@@ -190,6 +190,16 @@ export async function checkKeyHealth(keyId: number): Promise<KeyStatus> {const provider = resolveProvider(row.platform as Platform, row.base_url);if (!provider) return 'error';+ // #1459: a provider that forbids automated key checks gets no probe. Its+ // status is left to real traffic (markKeyHealthyFromRequest), and any reason+ // an older probe stored is cleared so the dashboard stops showing it.+ if (provider.supportsKeyValidation === false) {+ if (row.last_health_error != null) {+ db.prepare('UPDATE api_keys SET last_health_error = NULL WHERE id = ?').run(keyId);+ }+ return row.status as KeyStatus;+ }+try {const apiKey = decrypt(row.encrypted_key, row.iv, row.auth_tag);// #590: probe the key from the same exit its traffic uses. A key that is@@ -293,6 +303,9 @@ export async function probeKeyValidity(keyId: number): Promise<KeyProbeOutcome>const provider = resolveProvider(row.platform as Platform, row.base_url);if (!provider) return 'error';+ // #1459: no probe for a provider that forbids them; the cooldown simply+ // runs out, as it would after an inconclusive probe.+ if (provider.supportsKeyValidation === false) return 'error';const apiKey = decrypt(row.encrypted_key, row.iv, row.auth_tag);// Same reasoning as checkKeyHealth: probe through the key's own proxy (#590).@@ -313,17 +326,20 @@ export async function probeKeyValidity(keyId: number): Promise<KeyProbeOutcome>}/**- * Promote a key out of 'error' after it successfully served a live request.+ * Promote a key out of 'error' or 'unknown' after it successfully served a live+ * request.** Serving traffic is stronger evidence than any probe, so a key stuck at 'error'* from an earlier transport blip should not have to wait for the next health pass- * to become routable again. Deliberately narrow: 'invalid' means a provider- * confirmed the credential is bad, and only a real validateKey pass clears that.+ * to become routable again, and a key never probed yet (or one whose provider+ * forbids probes, #1459) has just proved itself. Deliberately narrow: 'invalid'+ * means a provider confirmed the credential is bad, and only a real validateKey+ * pass clears that.*/export function markKeyHealthyFromRequest(keyId: number): void {try {getDb()- .prepare("UPDATE api_keys SET status = 'healthy', last_health_error = NULL WHERE id = ? AND status = 'error'")+ .prepare("UPDATE api_keys SET status = 'healthy', last_health_error = NULL WHERE id = ? AND status IN ('error', 'unknown')").run(keyId);failureCount.delete(keyId);} catch {