Skip to content

7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.

TypeScript

fix(llmtr): a 403 refusing automated validation is inconclusive, not invalid (#1390) (#1416)

Tashfeen021bcfbe876c
diff --git a/server/src/__tests__/providers/llmtr.test.ts b/server/src/__tests__/providers/llmtr.test.ts
index 8657649f..9b829c46 100644
--- a/server/src/__tests__/providers/llmtr.test.ts
+++ b/server/src/__tests__/providers/llmtr.test.ts
@@ -76,6 +76,17 @@ describe('LLMTR provider', () => {
await expect(getProvider('llmtr')!.validateKey('test-key')).rejects.toMatchObject({ status, retryAfterMs: 17_000 });
});
+ it('#1390: a 403 refusing automated validation probes is inconclusive, never invalid', async () => {
+ // Live LLMTR response to our probe on 2026-10-03, reported on a working key.
+ vi.spyOn(global, 'fetch').mockResolvedValue(json({ error: { type: 'forbidden', message: 'Automated API key validation tools are not supported.' } }, 403));
+ await expect(getProvider('llmtr')!.validateKey('good-key')).rejects.toThrow(/not supported[\s\S]*the key was not checked|the key was not checked/);
+ });
+
+ it('#1390: a plain 403 with an ordinary auth message still reports invalid', async () => {
+ vi.spyOn(global, 'fetch').mockResolvedValue(json({ error: { type: 'auth_error', message: 'Invalid API key' } }, 403));
+ await expect(getProvider('llmtr')!.validateKey('bad-key')).resolves.toMatchObject({ valid: false });
+ });
+
it('does not accept an HTML 404', async () => {
vi.spyOn(global, 'fetch').mockResolvedValue(new Response('not JSON', { status: 404 }));
await expect(getProvider('llmtr')!.validateKey('test-key')).rejects.toMatchObject({ status: 404 });
diff --git a/server/src/providers/llmtr.ts b/server/src/providers/llmtr.ts
index 5f975de5..c0720f34 100644
--- a/server/src/providers/llmtr.ts
+++ b/server/src/providers/llmtr.ts
@@ -13,6 +13,15 @@ function checkModel(requested: string, returned: string): void {
}
}
+/** True when a 403 body says LLMTR refuses automated key-validation probes
+ * (#1390) rather than rejecting the key. Reads a clone so the caller's
+ * validationResult can still consume the body. */
+async function blocksAutomatedValidation(res: Response): Promise<boolean> {
+ const body = await res.clone().json().catch(() => null) as { error?: { message?: string }; message?: string } | null;
+ const text = [body?.error?.message, body?.message].filter((v): v is string => typeof v === 'string').join(' ').toLowerCase();
+ return text.includes('automated api key validation') || (text.includes('automated') && text.includes('validation'));
+}
+
/** Selected zero-priced routes have renewable daily/rolling quotas, not a
* monthly cash grant. Only the signed catalog supplies model rows: the public
* roster also contains paid models, expiring promotions and BYOK-only rows. */
@@ -32,7 +41,19 @@ export class LlmtrProvider extends OpenAICompatProvider {
body: JSON.stringify({ model: VALIDATION_MODEL, messages: [{ role: 'user', content: 'key validation' }], max_tokens: 1, stream: false }),
}, providerTimeoutMs(this.platform, 30_000), { timeoutBounds: 'request' });
recordQuotaObservationsFromResponse(res, { ...quotaContext, platform: this.platform, endpoint: 'key-validation' });
- if ([401, 403].includes(res.status)) return this.validationResult(res);
+ if ([401, 403].includes(res.status)) {
+ // #1390: LLMTR answers the probe itself with 403 "Automated API key
+ // validation tools are not supported." That is a verdict about the
+ // request shape, not the key. Returning valid:false here would let the
+ // health service's three-strike counter auto-disable a perfectly good
+ // key purely because it was probed. Same shape as the Cloudflare
+ // challenge guard in base.validationResult (#1298): inconclusive, never
+ // a disable.
+ if (res.status === 403 && await blocksAutomatedValidation(res)) {
+ throw providerHttpError(res, 'LLMTR rejected the key-validation probe itself (automated validation not supported); the key was not checked');
+ }
+ return this.validationResult(res);
+ }
if (res.status === 404) {
const body = await res.clone().json().catch(() => null) as { error?: { type?: string } } | null;
if (body?.error?.type === 'model_not_found') return true;