Skip to content

7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.

TypeScript

fix(llmtr): stop probing LLMTR keys; promote them on first served request (#1459) (#1471)

Tashfeenee8d8bd4cddb
diff --git a/server/src/__tests__/providers/llmtr.test.ts b/server/src/__tests__/providers/llmtr.test.ts
index d2c2c75f..d1673714 100644
--- a/server/src/__tests__/providers/llmtr.test.ts
+++ b/server/src/__tests__/providers/llmtr.test.ts
@@ -81,6 +81,11 @@ describe('LLMTR provider', () => {
await expect(unpaced().validateKey('test-key')).rejects.toMatchObject({ status, retryAfterMs: 17_000 });
});
+ it('#1459: opts out of automated key validation; other providers keep it', () => {
+ expect(getProvider('llmtr')!.supportsKeyValidation).toBe(false);
+ expect(getProvider('groq')!.supportsKeyValidation).toBe(true);
+ });
+
it('#1390: a 403 refusing automated validation probes is inconclusive, never invalid', async () => {
// Live LLMTR response to our probe on 2026-10-03, reported on a working key.
vi.spyOn(global, 'fetch').mockResolvedValue(json({ error: { type: 'forbidden', message: 'Automated API key validation tools are not supported.' } }, 403));
diff --git a/server/src/__tests__/services/health-transport-preserve.test.ts b/server/src/__tests__/services/health-transport-preserve.test.ts
index 61018c2e..34a05e2c 100644
--- a/server/src/__tests__/services/health-transport-preserve.test.ts
+++ b/server/src/__tests__/services/health-transport-preserve.test.ts
@@ -117,6 +117,12 @@ describe('markKeyHealthyFromRequest', () => {
expect(row(id)).toEqual({ status: 'healthy', last_health_error: null });
});
+ it("promotes a never-probed 'unknown' key to healthy (#1459)", () => {
+ const id = seedKey('unknown');
+ markKeyHealthyFromRequest(id);
+ expect(row(id)).toEqual({ status: 'healthy', last_health_error: null });
+ });
+
it("never resurrects an 'invalid' key — only a real probe clears that", () => {
const id = seedKey('invalid');
markKeyHealthyFromRequest(id);
diff --git a/server/src/__tests__/services/health-unverifiable-provider.test.ts b/server/src/__tests__/services/health-unverifiable-provider.test.ts
new file mode 100644
index 00000000..269db777
--- /dev/null
+++ b/server/src/__tests__/services/health-unverifiable-provider.test.ts
@@ -0,0 +1,79 @@
+import { describe, it, expect, beforeAll, beforeEach, vi } from 'vitest';
+
+// #1459: LLMTR refuses automated key checks and suspends accounts that keep
+// sending them, so a provider with supportsKeyValidation === false must never
+// be probed by the health pass or the cooldown probe.
+const validateKey = vi.fn();
+vi.mock('../../providers/index.js', async (importOriginal) => {
+ const actual = await importOriginal() as any;
+ return {
+ ...actual,
+ resolveProvider: () => ({ name: 'no-probe', supportsKeyValidation: false, validateKey }),
+ };
+});
+
+const { initDb, getDb } = await import('../../db/index.js');
+const { encrypt } = await import('../../lib/crypto.js');
+const { checkKeyHealth, checkAllKeys, probeKeyValidity, markKeyHealthyFromRequest } = await import('../../services/health.js');
+
+describe('a provider that forbids key validation (#1459)', () => {
+ let nextId = 9100;
+
+ beforeAll(() => {
+ process.env.ENCRYPTION_KEY = '0'.repeat(64);
+ initDb(':memory:');
+ });
+
+ beforeEach(() => {
+ validateKey.mockReset();
+ vi.spyOn(console, 'error').mockImplementation(() => {});
+ vi.spyOn(console, 'warn').mockImplementation(() => {});
+ vi.spyOn(console, 'log').mockImplementation(() => {});
+ });
+
+ function seedKey(status: string, lastError: string | null = null): number {
+ const id = ++nextId;
+ const enc = encrypt(`no-probe-${id}`);
+ getDb().prepare(`
+ INSERT INTO api_keys (id, platform, label, encrypted_key, iv, auth_tag, enabled, status, last_health_error)
+ VALUES (?, 'llmtr', ?, ?, ?, ?, 1, ?, ?)
+ `).run(id, `no-probe-${id}`, enc.encrypted, enc.iv, enc.authTag, status, lastError);
+ return id;
+ }
+
+ function row(id: number): { status: string; last_health_error: string | null } {
+ return getDb().prepare('SELECT status, last_health_error FROM api_keys WHERE id = ?').get(id) as any;
+ }
+
+ it('checkKeyHealth sends no probe, keeps the status and clears the stored probe error', async () => {
+ const id = seedKey('unknown', 'LLMTR rejected the key-validation probe itself (automated validation not supported); the key was not checked');
+
+ expect(await checkKeyHealth(id)).toBe('unknown');
+ expect(validateKey).not.toHaveBeenCalled();
+ expect(row(id)).toEqual({ status: 'unknown', last_health_error: null });
+ });
+
+ it('a healthy key stays healthy through the scheduled pass with no probe sent', async () => {
+ const id = seedKey('healthy');
+
+ await checkAllKeys({ force: true });
+
+ expect(validateKey).not.toHaveBeenCalled();
+ expect(row(id).status).toBe('healthy');
+ });
+
+ it('the cooldown probe treats the key as inconclusive without probing', async () => {
+ const id = seedKey('healthy');
+
+ expect(await probeKeyValidity(id)).toBe('error');
+ expect(validateKey).not.toHaveBeenCalled();
+ });
+
+ it('the first served request promotes the key to healthy', () => {
+ const id = seedKey('unknown');
+
+ markKeyHealthyFromRequest(id);
+
+ expect(row(id)).toEqual({ status: 'healthy', last_health_error: null });
+ });
+});
diff --git a/server/src/providers/base.ts b/server/src/providers/base.ts
index d4efd92d..27151f9a 100644
--- a/server/src/providers/base.ts
+++ b/server/src/providers/base.ts
@@ -268,6 +268,17 @@ export abstract class BaseProvider {
return false;
}
+ /** Whether the health checker may probe this provider's keys at all (#1459).
+ * A provider that forbids automated key checks (LLMTR answers any probe with
+ * 403 "automated API key validation tools are not supported", and its
+ * security system suspends accounts that keep sending them) returns false:
+ * the health pass and the cooldown probe then never call validateKey for its
+ * keys, and a key is promoted to healthy by the first request it serves
+ * (markKeyHealthyFromRequest) instead of by a probe. */
+ get supportsKeyValidation(): boolean {
+ return true;
+ }
+
/**
* Turn a conventional 401/403 validation response into a diagnostic result.
* Providers still return a simple boolean when no useful error body exists,
diff --git a/server/src/providers/llmtr.ts b/server/src/providers/llmtr.ts
index 0ae09167..6750fe12 100644
--- a/server/src/providers/llmtr.ts
+++ b/server/src/providers/llmtr.ts
@@ -76,6 +76,13 @@ export class LlmtrProvider extends OpenAICompatProvider {
* overlap easily) share one request instead of sending two identical ones. */
private readonly inFlightValidations = new Map<string, Promise<KeyValidationResult>>();
+ /** #1459: LLMTR refuses automated key checks outright, so the health pass
+ * and the cooldown probe never send one. Keys turn healthy on their first
+ * served request instead. validateKey below stays for explicit callers. */
+ override get supportsKeyValidation(): boolean {
+ return false;
+ }
+
constructor(opts: LlmtrProviderOptions = {}) {
super({ platform: 'llmtr', name: 'LLMTR', baseUrl: LLMTR_BASE_URL });
this.validationMinGapMs = opts.validationMinGapMs ?? validationMinGapMs();
diff --git a/server/src/services/health.ts b/server/src/services/health.ts
index 8678752c..35a0c425 100644
--- a/server/src/services/health.ts
+++ b/server/src/services/health.ts
@@ -190,6 +190,16 @@ export async function checkKeyHealth(keyId: number): Promise<KeyStatus> {
const provider = resolveProvider(row.platform as Platform, row.base_url);
if (!provider) return 'error';
+ // #1459: a provider that forbids automated key checks gets no probe. Its
+ // status is left to real traffic (markKeyHealthyFromRequest), and any reason
+ // an older probe stored is cleared so the dashboard stops showing it.
+ if (provider.supportsKeyValidation === false) {
+ if (row.last_health_error != null) {
+ db.prepare('UPDATE api_keys SET last_health_error = NULL WHERE id = ?').run(keyId);
+ }
+ return row.status as KeyStatus;
+ }
+
try {
const apiKey = decrypt(row.encrypted_key, row.iv, row.auth_tag);
// #590: probe the key from the same exit its traffic uses. A key that is
@@ -293,6 +303,9 @@ export async function probeKeyValidity(keyId: number): Promise<KeyProbeOutcome>
const provider = resolveProvider(row.platform as Platform, row.base_url);
if (!provider) return 'error';
+ // #1459: no probe for a provider that forbids them; the cooldown simply
+ // runs out, as it would after an inconclusive probe.
+ if (provider.supportsKeyValidation === false) return 'error';
const apiKey = decrypt(row.encrypted_key, row.iv, row.auth_tag);
// Same reasoning as checkKeyHealth: probe through the key's own proxy (#590).
@@ -313,17 +326,20 @@ export async function probeKeyValidity(keyId: number): Promise<KeyProbeOutcome>
}
/**
- * Promote a key out of 'error' after it successfully served a live request.
+ * Promote a key out of 'error' or 'unknown' after it successfully served a live
+ * request.
*
* Serving traffic is stronger evidence than any probe, so a key stuck at 'error'
* from an earlier transport blip should not have to wait for the next health pass
- * to become routable again. Deliberately narrow: 'invalid' means a provider
- * confirmed the credential is bad, and only a real validateKey pass clears that.
+ * to become routable again, and a key never probed yet (or one whose provider
+ * forbids probes, #1459) has just proved itself. Deliberately narrow: 'invalid'
+ * means a provider confirmed the credential is bad, and only a real validateKey
+ * pass clears that.
*/
export function markKeyHealthyFromRequest(keyId: number): void {
try {
getDb()
- .prepare("UPDATE api_keys SET status = 'healthy', last_health_error = NULL WHERE id = ? AND status = 'error'")
+ .prepare("UPDATE api_keys SET status = 'healthy', last_health_error = NULL WHERE id = ? AND status IN ('error', 'unknown')")
.run(keyId);
failureCount.delete(keyId);
} catch {