7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.
TypeScript
fix: distinguish Groq OTPM and ITPM limits in rate-limit learning (#1546)
emirhantasli43c190eadf2b
diff --git a/server/src/__tests__/lib/fallback-loop-groq-directional-tpm.test.ts b/server/src/__tests__/lib/fallback-loop-groq-directional-tpm.test.tsnew file mode 100644index 00000000..de6fa074--- /dev/null+++ b/server/src/__tests__/lib/fallback-loop-groq-directional-tpm.test.ts@@ -0,0 +1,130 @@+import { describe, it, expect, beforeAll, beforeEach, vi } from 'vitest';++// Groq orgs with split per-minute limits reject on ITPM (prompt tokens) or+// OTPM (expected completion tokens) separately from the combined TPM. Live+// failure: one OTPM 1000 rejection was learned as tpm_limit = 1000 on a+// 7000-ITPM model, and from then on the router sized out every request over+// 1000 tokens on that model — permanently, since learned limits never rise.++vi.mock('../../services/health.js', () => ({+ checkKeyHealth: vi.fn(),+ markKeyHealthyFromRequest: vi.fn(),+}));++import { initDb, getDb } from '../../db/index.js';+import { encrypt } from '../../lib/crypto.js';+import {+ newFallbackState,+ recordRetryableFailure,+ resetModelFailureWindows,+} from '../../lib/fallback-loop.js';+import { isOnCooldown, resetKeyLocalityCache } from '../../services/ratelimit.js';+import type { RouteResult } from '../../services/router.js';++const PLATFORM = 'groq';+const OTPM_BODY = 'Groq API error 429: Request too large for model `qwen/qwen3.8-27b` in organization `org_x` service tier `on_demand` on output tokens per minute (OTPM): Limit 1000, Requested 1838. The request\'s expected output tokens exceed the limit.';+const ITPM_BODY = 'Groq API error 413: Request too large for model `qwen/qwen3.8-27b` in organization `org_x` service tier `on_demand` on input tokens per minute (ITPM): Limit 7000, Requested 16136, please reduce your message size and try again.';+// A full per-minute window (not an oversized request) uses Groq's "Rate limit+// reached … Used N" shape; these follow its TPM wording with the directional axis.+const OTPM_WINDOW_BODY = 'Groq API error 429: Rate limit reached for model `qwen/qwen3.8-27b` in organization `org_x` service tier `on_demand` on output tokens per minute (OTPM): Limit 1000, Used 900, Requested 300. Please try again in 12s.';+const ITPM_WINDOW_BODY = 'Groq API error 429: Rate limit reached for model `qwen/qwen3.8-27b` in organization `org_x` service tier `on_demand` on input tokens per minute (ITPM): Limit 7000, Used 6500, Requested 1500. Please try again in 8s.';++let keyId = 0;+let model: { id: number; model_id: string };++function route(): RouteResult {+ return {+ provider: {} as any,+ modelId: model.model_id,+ modelDbId: model.id,+ apiKey: 'k',+ keyId,+ platform: PLATFORM,+ displayName: model.model_id,+ rpdLimit: null,+ tpdLimit: null,+ };+}++const tpmLimit = () =>+ (getDb().prepare('SELECT tpm_limit FROM models WHERE id = ?').get(model.id) as { tpm_limit: number | null }).tpm_limit;++beforeAll(() => {+ process.env.ENCRYPTION_KEY = '0'.repeat(64);+ initDb(':memory:');+ const db = getDb();+ db.prepare('DELETE FROM api_keys').run();+ const { encrypted, iv, authTag } = encrypt('test-key');+ keyId = Number(db.prepare(`+ INSERT INTO api_keys (platform, label, encrypted_key, iv, auth_tag, status, enabled)+ VALUES (?, 'key', ?, ?, ?, 'healthy', 1)+ `).run(PLATFORM, encrypted, iv, authTag).lastInsertRowid);+ resetKeyLocalityCache();+ model = db.prepare('SELECT id, model_id FROM models WHERE platform = ? LIMIT 1').get(PLATFORM) as typeof model;+});++beforeEach(() => {+ getDb().prepare('DELETE FROM rate_limit_cooldowns').run();+ getDb().prepare('UPDATE models SET tpm_limit = 7000 WHERE id = ?').run(model.id);+ resetModelFailureWindows();+});++describe('Groq OTPM rejection', () => {+ it('keeps tpm_limit at the input ceiling', () => {+ recordRetryableFailure(route(), Object.assign(new Error(OTPM_BODY), { status: 429 }), newFallbackState());+ expect(tpmLimit()).toBe(7000);+ });++ it('stays unchanged across repeated rejections', () => {+ for (let i = 0; i < 5; i++) {+ recordRetryableFailure(route(), Object.assign(new Error(OTPM_BODY), { status: 429 }), newFallbackState());+ }+ expect(tpmLimit()).toBe(7000);+ });++ it('skips the model for this request only — no cooldown, so the next request can use it', () => {+ const state = newFallbackState();+ recordRetryableFailure(route(), Object.assign(new Error(OTPM_BODY), { status: 429 }), state);+ expect(state.skipModels.has(model.id)).toBe(true);+ expect(isOnCooldown(PLATFORM, model.model_id, keyId)).toBe(false);+ });++ it('does not latch the expected-output count as the request size', () => {+ const state = newFallbackState();+ recordRetryableFailure(route(), Object.assign(new Error(OTPM_BODY), { status: 429 }), state);+ expect(state.observedTotalTokens).toBeUndefined();+ expect(state.observedInputTokens).toBeUndefined();+ });+});++describe('Groq OTPM window exhausted (rate limit reached)', () => {+ it('cools the route down for the window but keeps tpm_limit at the input ceiling', () => {+ recordRetryableFailure(route(), Object.assign(new Error(OTPM_WINDOW_BODY), { status: 429 }), newFallbackState());+ expect(isOnCooldown(PLATFORM, model.model_id, keyId)).toBe(true);+ expect(tpmLimit()).toBe(7000);+ });+});++describe('Groq ITPM window exhausted (rate limit reached)', () => {+ it('cools the route down and lowers a too-high tpm_limit to the input ceiling', () => {+ getDb().prepare('UPDATE models SET tpm_limit = 70000 WHERE id = ?').run(model.id);+ recordRetryableFailure(route(), Object.assign(new Error(ITPM_WINDOW_BODY), { status: 429 }), newFallbackState());+ expect(isOnCooldown(PLATFORM, model.model_id, keyId)).toBe(true);+ expect(tpmLimit()).toBe(7000);+ });+});++describe('Groq ITPM rejection', () => {+ it('lowers a too-high tpm_limit to the input ceiling', () => {+ getDb().prepare('UPDATE models SET tpm_limit = 70000 WHERE id = ?').run(model.id);+ recordRetryableFailure(route(), Object.assign(new Error(ITPM_BODY), { status: 413 }), newFallbackState());+ expect(tpmLimit()).toBe(7000);+ });++ it('latches the prompt size as an input-only reading', () => {+ const state = newFallbackState();+ recordRetryableFailure(route(), Object.assign(new Error(ITPM_BODY), { status: 413 }), state);+ expect(state.observedInputTokens).toBe(16136);+ expect(state.observedTotalTokens).toBeUndefined();+ });+});diff --git a/server/src/__tests__/lib/provider-size-parser.test.ts b/server/src/__tests__/lib/provider-size-parser.test.tsindex c27a8a22..58fa78b9 100644--- a/server/src/__tests__/lib/provider-size-parser.test.ts+++ b/server/src/__tests__/lib/provider-size-parser.test.ts@@ -7,6 +7,16 @@ import { parseProviderReportedSize } from '../../lib/provider-size-parser.js';// forces a parser update.describe('parseProviderReportedSize', () => {describe('groq', () => {+ it('does not read an OTPM "Requested N" (expected output) as the request size', () => {+ const msg = 'Groq API error 429: Request too large for model `qwen/qwen3.8-27b` in organization `org_x` service tier `on_demand` on output tokens per minute (OTPM): Limit 1000, Requested 1838. The request\'s expected output tokens exceed the limit.';+ expect(parseProviderReportedSize('groq', msg)).toBeNull();+ });++ it('reads an ITPM "Requested N" as an input-only size', () => {+ const msg = 'Groq API error 413: Request too large for model `qwen/qwen3.8-27b` in organization `org_x` service tier `on_demand` on input tokens per minute (ITPM): Limit 7000, Requested 16136, please reduce your message size and try again.';+ expect(parseProviderReportedSize('groq', msg)).toEqual({ tokens: 16136, kind: 'input' });+ });+it('extracts Requested N from a TPM 413', () => {const msg = 'Groq API error 413: Request too large for model `openai/gpt-oss-120b` in organization `org_01kptjck7bejta1btzc11cccy2` service tier `on_demand` on tokens per minute (TPM): Limit 8000, Requested 36532, please reduce your message size and try again.';expect(parseProviderReportedSize('groq', msg)).toEqual({ tokens: 36532, kind: 'total' });diff --git a/server/src/__tests__/services/learn-limits.test.ts b/server/src/__tests__/services/learn-limits.test.tsindex 7c25c574..54c4ee7f 100644--- a/server/src/__tests__/services/learn-limits.test.ts+++ b/server/src/__tests__/services/learn-limits.test.ts@@ -22,6 +22,25 @@ describe('parseProviderLimit', () => {expect(parseProviderLimit('requests per minute and tokens per minute (TPM): Limit 8000')).toEqual({ kind: 'tpm', limit: 8000 });});+ // Groq orgs with split per-minute limits name the direction: ITPM counts+ // prompt tokens only, OTPM counts expected completion tokens only. Neither+ // is the combined TPM the router gates on (live bodies, org redacted).+ const GROQ_OTPM = 'Groq API error 429: Request too large for model `qwen/qwen3.8-27b` in organization `org_x` service tier `on_demand` on output tokens per minute (OTPM): Limit 1000, Requested 1838. The request\'s expected output tokens exceed the limit.';+ const GROQ_ITPM = 'Groq API error 413: Request too large for model `qwen/qwen3.8-27b` in organization `org_x` service tier `on_demand` on input tokens per minute (ITPM): Limit 7000, Requested 16136, please reduce your message size and try again.';++ it('reads a Groq OTPM body as the output axis, not TPM', () => {+ expect(parseProviderLimit(GROQ_OTPM)).toEqual({ kind: 'otpm', limit: 1000 });+ });++ it('reads a Groq ITPM body as the input axis, not TPM', () => {+ expect(parseProviderLimit(GROQ_ITPM)).toEqual({ kind: 'itpm', limit: 7000 });+ });++ it('reads bare OTPM / ITPM abbreviations', () => {+ expect(parseProviderLimit('OTPM: Limit 1000, Requested 1838')).toEqual({ kind: 'otpm', limit: 1000 });+ expect(parseProviderLimit('ITPM: Limit 7000, Requested 9000')).toEqual({ kind: 'itpm', limit: 7000 });+ });+it('returns null without a confident axis even when a Limit is present', () => {expect(parseProviderLimit('Some limit: 9999 was exceeded')).toBeNull();});@@ -86,4 +105,24 @@ describe('learnLimitFromError (persistence)', () => {const rpm = (getDb().prepare('SELECT rpm_limit FROM models WHERE id = ?').get(id) as { rpm_limit: number | null }).rpm_limit;expect(rpm).toBe(20);});++ it('never writes an OTPM (output-only) ceiling into tpm_limit', () => {+ const id = insertModel({ tpm: 7000 });+ const r = learnLimitFromError(id, { message: 'on output tokens per minute (OTPM): Limit 1000, Requested 1838' });+ expect(r).toBeNull();+ expect(tpmCol(id)).toBe(7000);+ });++ it('leaves a NULL tpm_limit alone on an OTPM rejection', () => {+ const id = insertModel({ tpm: null });+ learnLimitFromError(id, { message: 'on output tokens per minute (OTPM): Limit 1000, Requested 1838' });+ expect(tpmCol(id)).toBeNull();+ });++ it('lowers tpm_limit to an ITPM ceiling (input is part of every request, so it is a safe bound)', () => {+ const id = insertModel({ tpm: 70000 });+ const r = learnLimitFromError(id, { message: 'on input tokens per minute (ITPM): Limit 7000, Requested 16136' });+ expect(r).toEqual({ kind: 'itpm', limit: 7000 });+ expect(tpmCol(id)).toBe(7000);+ });});diff --git a/server/src/lib/provider-size-parser.ts b/server/src/lib/provider-size-parser.tsindex 3ce35bf9..f3074d34 100644--- a/server/src/lib/provider-size-parser.ts+++ b/server/src/lib/provider-size-parser.ts@@ -66,8 +66,13 @@ export function parseProviderReportedSize(platform: string, message: string | nuif (typeof message !== 'string' || message === '') return null;const re = PATTERNS[platform];if (!re) return null;+ // A Groq OTPM rejection's "Requested N" is the EXPECTED OUTPUT, not the+ // request's size; latching it would size out models on prompt length for a+ // completion-side limit. An ITPM rejection's N is the prompt alone.+ if (platform === 'groq' && /output\s+tokens?\s*per\s*min|\botpm\b/i.test(message)) return null;const tokens = pullFirstInt(message, re);if (tokens == null) return null;- const inputOnly = platform === 'cloudflare' && /contains\s+at\s+least\s+[\d,]+\s+input\s+tokens/i.test(message);+ const inputOnly = (platform === 'cloudflare' && /contains\s+at\s+least\s+[\d,]+\s+input\s+tokens/i.test(message))+ || (platform === 'groq' && /input\s+tokens?\s*per\s*min|\bitpm\b/i.test(message));return { tokens, kind: inputOnly ? 'input' : 'total' };}diff --git a/server/src/services/ratelimit.ts b/server/src/services/ratelimit.tsindex bb8555bc..bb5994da 100644--- a/server/src/services/ratelimit.ts+++ b/server/src/services/ratelimit.ts@@ -1463,14 +1463,22 @@ export function getRateLimitStatus(// stop us BEFORE the next 413 instead of re-discovering it on every request.// (Fork-validated: andersmmg's updateLimitsFromError parses the same Groq shape.)-export type LearnedLimitKind = 'tpm' | 'tpd' | 'rpm' | 'rpd';+// 'itpm' / 'otpm' are the directional per-minute limits some Groq orgs carry+// on top of the combined TPM: "input tokens per minute (ITPM)" counts prompt+// tokens only, "output tokens per minute (OTPM)" counts expected completion+// tokens only.+export type LearnedLimitKind = 'tpm' | 'itpm' | 'otpm' | 'tpd' | 'rpm' | 'rpd';export interface LearnedLimit { kind: LearnedLimitKind; limit: number; }// Order matters: check the per-DAY axes before per-MINUTE so "tokens per day"-// isn't shadowed by the "tpm" word-boundary alternative, and tokens before-// requests so a body mentioning both lands on the more specific token ceiling.+// isn't shadowed by the "tpm" word-boundary alternative, the directional+// ITPM/OTPM axes before the combined TPM so "output tokens per minute" isn't+// read as TPM, and tokens before requests so a body mentioning both lands on+// the more specific token ceiling.const LIMIT_AXIS_PATTERNS: Array<{ kind: LearnedLimitKind; re: RegExp }> = [{ kind: 'tpd', re: /tokens?\s*per\s*day|\btpd\b/i },+ { kind: 'otpm', re: /output\s+tokens?\s*per\s*min(?:ute)?|\botpm\b/i },+ { kind: 'itpm', re: /input\s+tokens?\s*per\s*min(?:ute)?|\bitpm\b/i },{ kind: 'tpm', re: /tokens?\s*per\s*min(?:ute)?|\btpm\b/i },{ kind: 'rpd', re: /requests?\s*per\s*day|\brpd\b/i },{ kind: 'rpm', re: /requests?\s*per\s*min(?:ute)?|\brpm\b/i },@@ -1496,8 +1504,15 @@ export function parseProviderLimit(message: string | undefined | null): LearnedL// Whitelisted column names — the only values ever interpolated into the UPDATE// below, so there is no injection surface despite the template literal.-const LIMIT_COLUMN: Record<LearnedLimitKind, string> = {- tpm: 'tpm_limit', tpd: 'tpd_limit', rpm: 'rpm_limit', rpd: 'rpd_limit',+// The router's tpm_limit gate scores input estimate + output reserve, so an+// ITPM ceiling is a safe (slightly conservative) bound for it. An OTPM ceiling+// has no column: it bounds the completion alone, and writing it into tpm_limit+// would size out every request whose PROMPT exceeds it — one OTPM 1000+// rejection permanently capped a 7000-ITPM Groq model at 1000 tokens. The+// rejected request is already skipped for this model (context-too-large path),+// which is all an OTPM rejection calls for.+const LIMIT_COLUMN: Record<LearnedLimitKind, string | null> = {+ tpm: 'tpm_limit', itpm: 'tpm_limit', otpm: null, tpd: 'tpd_limit', rpm: 'rpm_limit', rpd: 'rpd_limit',};/**@@ -1512,6 +1527,7 @@ export function learnLimitFromError(modelDbId: number, err: { message?: string }const parsed = parseProviderLimit(err?.message);if (!parsed) return null;const col = LIMIT_COLUMN[parsed.kind];+ if (col == null) return null;const result = withDb(db =>db.prepare(`UPDATE models SET ${col} = ? WHERE id = ? AND (${col} IS NULL OR ${col} > ?)`,