Skip to content

7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.

TypeScript

fix: distinguish Groq OTPM and ITPM limits in rate-limit learning (#1546)

emirhantasli43c190eadf2b
diff --git a/server/src/__tests__/lib/fallback-loop-groq-directional-tpm.test.ts b/server/src/__tests__/lib/fallback-loop-groq-directional-tpm.test.ts
new file mode 100644
index 00000000..de6fa074
--- /dev/null
+++ b/server/src/__tests__/lib/fallback-loop-groq-directional-tpm.test.ts
@@ -0,0 +1,130 @@
+import { describe, it, expect, beforeAll, beforeEach, vi } from 'vitest';
+
+// Groq orgs with split per-minute limits reject on ITPM (prompt tokens) or
+// OTPM (expected completion tokens) separately from the combined TPM. Live
+// failure: one OTPM 1000 rejection was learned as tpm_limit = 1000 on a
+// 7000-ITPM model, and from then on the router sized out every request over
+// 1000 tokens on that model — permanently, since learned limits never rise.
+
+vi.mock('../../services/health.js', () => ({
+ checkKeyHealth: vi.fn(),
+ markKeyHealthyFromRequest: vi.fn(),
+}));
+
+import { initDb, getDb } from '../../db/index.js';
+import { encrypt } from '../../lib/crypto.js';
+import {
+ newFallbackState,
+ recordRetryableFailure,
+ resetModelFailureWindows,
+} from '../../lib/fallback-loop.js';
+import { isOnCooldown, resetKeyLocalityCache } from '../../services/ratelimit.js';
+import type { RouteResult } from '../../services/router.js';
+
+const PLATFORM = 'groq';
+const OTPM_BODY = 'Groq API error 429: Request too large for model `qwen/qwen3.8-27b` in organization `org_x` service tier `on_demand` on output tokens per minute (OTPM): Limit 1000, Requested 1838. The request\'s expected output tokens exceed the limit.';
+const ITPM_BODY = 'Groq API error 413: Request too large for model `qwen/qwen3.8-27b` in organization `org_x` service tier `on_demand` on input tokens per minute (ITPM): Limit 7000, Requested 16136, please reduce your message size and try again.';
+// A full per-minute window (not an oversized request) uses Groq's "Rate limit
+// reached … Used N" shape; these follow its TPM wording with the directional axis.
+const OTPM_WINDOW_BODY = 'Groq API error 429: Rate limit reached for model `qwen/qwen3.8-27b` in organization `org_x` service tier `on_demand` on output tokens per minute (OTPM): Limit 1000, Used 900, Requested 300. Please try again in 12s.';
+const ITPM_WINDOW_BODY = 'Groq API error 429: Rate limit reached for model `qwen/qwen3.8-27b` in organization `org_x` service tier `on_demand` on input tokens per minute (ITPM): Limit 7000, Used 6500, Requested 1500. Please try again in 8s.';
+
+let keyId = 0;
+let model: { id: number; model_id: string };
+
+function route(): RouteResult {
+ return {
+ provider: {} as any,
+ modelId: model.model_id,
+ modelDbId: model.id,
+ apiKey: 'k',
+ keyId,
+ platform: PLATFORM,
+ displayName: model.model_id,
+ rpdLimit: null,
+ tpdLimit: null,
+ };
+}
+
+const tpmLimit = () =>
+ (getDb().prepare('SELECT tpm_limit FROM models WHERE id = ?').get(model.id) as { tpm_limit: number | null }).tpm_limit;
+
+beforeAll(() => {
+ process.env.ENCRYPTION_KEY = '0'.repeat(64);
+ initDb(':memory:');
+ const db = getDb();
+ db.prepare('DELETE FROM api_keys').run();
+ const { encrypted, iv, authTag } = encrypt('test-key');
+ keyId = Number(db.prepare(`
+ INSERT INTO api_keys (platform, label, encrypted_key, iv, auth_tag, status, enabled)
+ VALUES (?, 'key', ?, ?, ?, 'healthy', 1)
+ `).run(PLATFORM, encrypted, iv, authTag).lastInsertRowid);
+ resetKeyLocalityCache();
+ model = db.prepare('SELECT id, model_id FROM models WHERE platform = ? LIMIT 1').get(PLATFORM) as typeof model;
+});
+
+beforeEach(() => {
+ getDb().prepare('DELETE FROM rate_limit_cooldowns').run();
+ getDb().prepare('UPDATE models SET tpm_limit = 7000 WHERE id = ?').run(model.id);
+ resetModelFailureWindows();
+});
+
+describe('Groq OTPM rejection', () => {
+ it('keeps tpm_limit at the input ceiling', () => {
+ recordRetryableFailure(route(), Object.assign(new Error(OTPM_BODY), { status: 429 }), newFallbackState());
+ expect(tpmLimit()).toBe(7000);
+ });
+
+ it('stays unchanged across repeated rejections', () => {
+ for (let i = 0; i < 5; i++) {
+ recordRetryableFailure(route(), Object.assign(new Error(OTPM_BODY), { status: 429 }), newFallbackState());
+ }
+ expect(tpmLimit()).toBe(7000);
+ });
+
+ it('skips the model for this request only — no cooldown, so the next request can use it', () => {
+ const state = newFallbackState();
+ recordRetryableFailure(route(), Object.assign(new Error(OTPM_BODY), { status: 429 }), state);
+ expect(state.skipModels.has(model.id)).toBe(true);
+ expect(isOnCooldown(PLATFORM, model.model_id, keyId)).toBe(false);
+ });
+
+ it('does not latch the expected-output count as the request size', () => {
+ const state = newFallbackState();
+ recordRetryableFailure(route(), Object.assign(new Error(OTPM_BODY), { status: 429 }), state);
+ expect(state.observedTotalTokens).toBeUndefined();
+ expect(state.observedInputTokens).toBeUndefined();
+ });
+});
+
+describe('Groq OTPM window exhausted (rate limit reached)', () => {
+ it('cools the route down for the window but keeps tpm_limit at the input ceiling', () => {
+ recordRetryableFailure(route(), Object.assign(new Error(OTPM_WINDOW_BODY), { status: 429 }), newFallbackState());
+ expect(isOnCooldown(PLATFORM, model.model_id, keyId)).toBe(true);
+ expect(tpmLimit()).toBe(7000);
+ });
+});
+
+describe('Groq ITPM window exhausted (rate limit reached)', () => {
+ it('cools the route down and lowers a too-high tpm_limit to the input ceiling', () => {
+ getDb().prepare('UPDATE models SET tpm_limit = 70000 WHERE id = ?').run(model.id);
+ recordRetryableFailure(route(), Object.assign(new Error(ITPM_WINDOW_BODY), { status: 429 }), newFallbackState());
+ expect(isOnCooldown(PLATFORM, model.model_id, keyId)).toBe(true);
+ expect(tpmLimit()).toBe(7000);
+ });
+});
+
+describe('Groq ITPM rejection', () => {
+ it('lowers a too-high tpm_limit to the input ceiling', () => {
+ getDb().prepare('UPDATE models SET tpm_limit = 70000 WHERE id = ?').run(model.id);
+ recordRetryableFailure(route(), Object.assign(new Error(ITPM_BODY), { status: 413 }), newFallbackState());
+ expect(tpmLimit()).toBe(7000);
+ });
+
+ it('latches the prompt size as an input-only reading', () => {
+ const state = newFallbackState();
+ recordRetryableFailure(route(), Object.assign(new Error(ITPM_BODY), { status: 413 }), state);
+ expect(state.observedInputTokens).toBe(16136);
+ expect(state.observedTotalTokens).toBeUndefined();
+ });
+});
diff --git a/server/src/__tests__/lib/provider-size-parser.test.ts b/server/src/__tests__/lib/provider-size-parser.test.ts
index c27a8a22..58fa78b9 100644
--- a/server/src/__tests__/lib/provider-size-parser.test.ts
+++ b/server/src/__tests__/lib/provider-size-parser.test.ts
@@ -7,6 +7,16 @@ import { parseProviderReportedSize } from '../../lib/provider-size-parser.js';
// forces a parser update.
describe('parseProviderReportedSize', () => {
describe('groq', () => {
+ it('does not read an OTPM "Requested N" (expected output) as the request size', () => {
+ const msg = 'Groq API error 429: Request too large for model `qwen/qwen3.8-27b` in organization `org_x` service tier `on_demand` on output tokens per minute (OTPM): Limit 1000, Requested 1838. The request\'s expected output tokens exceed the limit.';
+ expect(parseProviderReportedSize('groq', msg)).toBeNull();
+ });
+
+ it('reads an ITPM "Requested N" as an input-only size', () => {
+ const msg = 'Groq API error 413: Request too large for model `qwen/qwen3.8-27b` in organization `org_x` service tier `on_demand` on input tokens per minute (ITPM): Limit 7000, Requested 16136, please reduce your message size and try again.';
+ expect(parseProviderReportedSize('groq', msg)).toEqual({ tokens: 16136, kind: 'input' });
+ });
+
it('extracts Requested N from a TPM 413', () => {
const msg = 'Groq API error 413: Request too large for model `openai/gpt-oss-120b` in organization `org_01kptjck7bejta1btzc11cccy2` service tier `on_demand` on tokens per minute (TPM): Limit 8000, Requested 36532, please reduce your message size and try again.';
expect(parseProviderReportedSize('groq', msg)).toEqual({ tokens: 36532, kind: 'total' });
diff --git a/server/src/__tests__/services/learn-limits.test.ts b/server/src/__tests__/services/learn-limits.test.ts
index 7c25c574..54c4ee7f 100644
--- a/server/src/__tests__/services/learn-limits.test.ts
+++ b/server/src/__tests__/services/learn-limits.test.ts
@@ -22,6 +22,25 @@ describe('parseProviderLimit', () => {
expect(parseProviderLimit('requests per minute and tokens per minute (TPM): Limit 8000')).toEqual({ kind: 'tpm', limit: 8000 });
});
+ // Groq orgs with split per-minute limits name the direction: ITPM counts
+ // prompt tokens only, OTPM counts expected completion tokens only. Neither
+ // is the combined TPM the router gates on (live bodies, org redacted).
+ const GROQ_OTPM = 'Groq API error 429: Request too large for model `qwen/qwen3.8-27b` in organization `org_x` service tier `on_demand` on output tokens per minute (OTPM): Limit 1000, Requested 1838. The request\'s expected output tokens exceed the limit.';
+ const GROQ_ITPM = 'Groq API error 413: Request too large for model `qwen/qwen3.8-27b` in organization `org_x` service tier `on_demand` on input tokens per minute (ITPM): Limit 7000, Requested 16136, please reduce your message size and try again.';
+
+ it('reads a Groq OTPM body as the output axis, not TPM', () => {
+ expect(parseProviderLimit(GROQ_OTPM)).toEqual({ kind: 'otpm', limit: 1000 });
+ });
+
+ it('reads a Groq ITPM body as the input axis, not TPM', () => {
+ expect(parseProviderLimit(GROQ_ITPM)).toEqual({ kind: 'itpm', limit: 7000 });
+ });
+
+ it('reads bare OTPM / ITPM abbreviations', () => {
+ expect(parseProviderLimit('OTPM: Limit 1000, Requested 1838')).toEqual({ kind: 'otpm', limit: 1000 });
+ expect(parseProviderLimit('ITPM: Limit 7000, Requested 9000')).toEqual({ kind: 'itpm', limit: 7000 });
+ });
+
it('returns null without a confident axis even when a Limit is present', () => {
expect(parseProviderLimit('Some limit: 9999 was exceeded')).toBeNull();
});
@@ -86,4 +105,24 @@ describe('learnLimitFromError (persistence)', () => {
const rpm = (getDb().prepare('SELECT rpm_limit FROM models WHERE id = ?').get(id) as { rpm_limit: number | null }).rpm_limit;
expect(rpm).toBe(20);
});
+
+ it('never writes an OTPM (output-only) ceiling into tpm_limit', () => {
+ const id = insertModel({ tpm: 7000 });
+ const r = learnLimitFromError(id, { message: 'on output tokens per minute (OTPM): Limit 1000, Requested 1838' });
+ expect(r).toBeNull();
+ expect(tpmCol(id)).toBe(7000);
+ });
+
+ it('leaves a NULL tpm_limit alone on an OTPM rejection', () => {
+ const id = insertModel({ tpm: null });
+ learnLimitFromError(id, { message: 'on output tokens per minute (OTPM): Limit 1000, Requested 1838' });
+ expect(tpmCol(id)).toBeNull();
+ });
+
+ it('lowers tpm_limit to an ITPM ceiling (input is part of every request, so it is a safe bound)', () => {
+ const id = insertModel({ tpm: 70000 });
+ const r = learnLimitFromError(id, { message: 'on input tokens per minute (ITPM): Limit 7000, Requested 16136' });
+ expect(r).toEqual({ kind: 'itpm', limit: 7000 });
+ expect(tpmCol(id)).toBe(7000);
+ });
});
diff --git a/server/src/lib/provider-size-parser.ts b/server/src/lib/provider-size-parser.ts
index 3ce35bf9..f3074d34 100644
--- a/server/src/lib/provider-size-parser.ts
+++ b/server/src/lib/provider-size-parser.ts
@@ -66,8 +66,13 @@ export function parseProviderReportedSize(platform: string, message: string | nu
if (typeof message !== 'string' || message === '') return null;
const re = PATTERNS[platform];
if (!re) return null;
+ // A Groq OTPM rejection's "Requested N" is the EXPECTED OUTPUT, not the
+ // request's size; latching it would size out models on prompt length for a
+ // completion-side limit. An ITPM rejection's N is the prompt alone.
+ if (platform === 'groq' && /output\s+tokens?\s*per\s*min|\botpm\b/i.test(message)) return null;
const tokens = pullFirstInt(message, re);
if (tokens == null) return null;
- const inputOnly = platform === 'cloudflare' && /contains\s+at\s+least\s+[\d,]+\s+input\s+tokens/i.test(message);
+ const inputOnly = (platform === 'cloudflare' && /contains\s+at\s+least\s+[\d,]+\s+input\s+tokens/i.test(message))
+ || (platform === 'groq' && /input\s+tokens?\s*per\s*min|\bitpm\b/i.test(message));
return { tokens, kind: inputOnly ? 'input' : 'total' };
}
diff --git a/server/src/services/ratelimit.ts b/server/src/services/ratelimit.ts
index bb8555bc..bb5994da 100644
--- a/server/src/services/ratelimit.ts
+++ b/server/src/services/ratelimit.ts
@@ -1463,14 +1463,22 @@ export function getRateLimitStatus(
// stop us BEFORE the next 413 instead of re-discovering it on every request.
// (Fork-validated: andersmmg's updateLimitsFromError parses the same Groq shape.)
-export type LearnedLimitKind = 'tpm' | 'tpd' | 'rpm' | 'rpd';
+// 'itpm' / 'otpm' are the directional per-minute limits some Groq orgs carry
+// on top of the combined TPM: "input tokens per minute (ITPM)" counts prompt
+// tokens only, "output tokens per minute (OTPM)" counts expected completion
+// tokens only.
+export type LearnedLimitKind = 'tpm' | 'itpm' | 'otpm' | 'tpd' | 'rpm' | 'rpd';
export interface LearnedLimit { kind: LearnedLimitKind; limit: number; }
// Order matters: check the per-DAY axes before per-MINUTE so "tokens per day"
-// isn't shadowed by the "tpm" word-boundary alternative, and tokens before
-// requests so a body mentioning both lands on the more specific token ceiling.
+// isn't shadowed by the "tpm" word-boundary alternative, the directional
+// ITPM/OTPM axes before the combined TPM so "output tokens per minute" isn't
+// read as TPM, and tokens before requests so a body mentioning both lands on
+// the more specific token ceiling.
const LIMIT_AXIS_PATTERNS: Array<{ kind: LearnedLimitKind; re: RegExp }> = [
{ kind: 'tpd', re: /tokens?\s*per\s*day|\btpd\b/i },
+ { kind: 'otpm', re: /output\s+tokens?\s*per\s*min(?:ute)?|\botpm\b/i },
+ { kind: 'itpm', re: /input\s+tokens?\s*per\s*min(?:ute)?|\bitpm\b/i },
{ kind: 'tpm', re: /tokens?\s*per\s*min(?:ute)?|\btpm\b/i },
{ kind: 'rpd', re: /requests?\s*per\s*day|\brpd\b/i },
{ kind: 'rpm', re: /requests?\s*per\s*min(?:ute)?|\brpm\b/i },
@@ -1496,8 +1504,15 @@ export function parseProviderLimit(message: string | undefined | null): LearnedL
// Whitelisted column names — the only values ever interpolated into the UPDATE
// below, so there is no injection surface despite the template literal.
-const LIMIT_COLUMN: Record<LearnedLimitKind, string> = {
- tpm: 'tpm_limit', tpd: 'tpd_limit', rpm: 'rpm_limit', rpd: 'rpd_limit',
+// The router's tpm_limit gate scores input estimate + output reserve, so an
+// ITPM ceiling is a safe (slightly conservative) bound for it. An OTPM ceiling
+// has no column: it bounds the completion alone, and writing it into tpm_limit
+// would size out every request whose PROMPT exceeds it — one OTPM 1000
+// rejection permanently capped a 7000-ITPM Groq model at 1000 tokens. The
+// rejected request is already skipped for this model (context-too-large path),
+// which is all an OTPM rejection calls for.
+const LIMIT_COLUMN: Record<LearnedLimitKind, string | null> = {
+ tpm: 'tpm_limit', itpm: 'tpm_limit', otpm: null, tpd: 'tpd_limit', rpm: 'rpm_limit', rpd: 'rpd_limit',
};
/**
@@ -1512,6 +1527,7 @@ export function learnLimitFromError(modelDbId: number, err: { message?: string }
const parsed = parseProviderLimit(err?.message);
if (!parsed) return null;
const col = LIMIT_COLUMN[parsed.kind];
+ if (col == null) return null;
const result = withDb(db =>
db.prepare(
`UPDATE models SET ${col} = ? WHERE id = ? AND (${col} IS NULL OR ${col} > ?)`,