Skip to content

7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.

TypeScript

Add router penalty inspector

Tashfeenbc07927985bd
diff --git a/client/src/components/penalty-inspector.tsx b/client/src/components/penalty-inspector.tsx
new file mode 100644
index 00000000..28aad46c
--- /dev/null
+++ b/client/src/components/penalty-inspector.tsx
@@ -0,0 +1,170 @@
+import { useQuery } from '@tanstack/react-query'
+import { AlertTriangle, Clock3, Gauge } from 'lucide-react'
+import { useI18n } from '@/i18n'
+import { apiFetch } from '@/lib/api'
+
+type InspectorReason = 'penalty' | 'cooldown' | 'recent_errors'
+
+interface PenaltyInspectorRow {
+ modelDbId: number | null
+ platform: string
+ modelId: string
+ displayName: string
+ enabled: boolean
+ fallbackEnabled: boolean
+ priority: number | null
+ penalty: {
+ hits: number
+ value: number
+ rateLimitFactor: number
+ }
+ cooldowns: Array<{
+ keyId: number
+ keyLabel: string | null
+ keyStatus: string | null
+ expiresAtMs: number
+ expiresInMs: number
+ }>
+ recentErrors: Array<{
+ id: number
+ keyId: number | null
+ keyLabel: string | null
+ error: string
+ latencyMs: number
+ createdAt: string
+ }>
+ recentErrorCount: number
+ reasons: InspectorReason[]
+}
+
+interface PenaltyInspectorData {
+ generatedAtMs: number
+ lookbackMinutes: number
+ rows: PenaltyInspectorRow[]
+}
+
+function formatDuration(ms: number): string {
+ const seconds = Math.max(0, Math.ceil(ms / 1000))
+ if (seconds < 60) return `${seconds}s`
+ const minutes = Math.ceil(seconds / 60)
+ if (minutes < 60) return `${minutes}m`
+ const hours = Math.ceil(minutes / 60)
+ return `${hours}h`
+}
+
+function formatTime(value: string): string {
+ const date = new Date(`${value.replace(' ', 'T')}Z`)
+ if (Number.isNaN(date.getTime())) return value
+ return date.toLocaleTimeString([], { hour: '2-digit', minute: '2-digit' })
+}
+
+function penaltyClass(value: number): string {
+ if (value >= 8) return 'bg-red-600/15 text-red-700 dark:text-red-400'
+ if (value >= 5) return 'bg-orange-600/15 text-orange-700 dark:text-orange-400'
+ if (value >= 3) return 'bg-amber-600/15 text-amber-700 dark:text-amber-400'
+ if (value > 0) return 'bg-emerald-600/15 text-emerald-700 dark:text-emerald-400'
+ return 'bg-muted text-muted-foreground'
+}
+
+export function PenaltyInspector() {
+ const { t } = useI18n()
+ const { data } = useQuery<PenaltyInspectorData>({
+ queryKey: ['fallback', 'penalty-inspector'],
+ queryFn: () => apiFetch('/api/fallback/penalty-inspector'),
+ refetchInterval: 5_000,
+ })
+
+ const rows = data?.rows ?? []
+ if (rows.length === 0) return null
+
+ return (
+ <section className="rounded-lg border bg-card">
+ <div className="flex flex-wrap items-center justify-between gap-2 border-b px-4 py-3">
+ <div className="flex items-center gap-2">
+ <AlertTriangle className="size-4 text-amber-600 dark:text-amber-400" />
+ <div>
+ <h2 className="text-sm font-medium">{t('penaltyInspector.title')}</h2>
+ <p className="text-xs text-muted-foreground">
+ {t('penaltyInspector.subtitle', { minutes: data?.lookbackMinutes ?? 30 })}
+ </p>
+ </div>
+ </div>
+ <span className="rounded-full bg-muted px-2 py-0.5 text-[11px] text-muted-foreground tabular-nums">
+ {t('penaltyInspector.rowCount', { count: rows.length })}
+ </span>
+ </div>
+
+ <div className="divide-y">
+ {rows.map(row => (
+ <div key={`${row.platform}:${row.modelId}:${row.modelDbId ?? 'missing'}`} className="grid gap-3 px-4 py-3 lg:grid-cols-[minmax(14rem,1.2fr)_minmax(10rem,0.8fr)_minmax(14rem,1.3fr)]">
+ <div className="min-w-0">
+ <div className="flex flex-wrap items-center gap-1.5">
+ <span className="truncate text-sm font-medium">{row.displayName}</span>
+ <span className="text-xs text-muted-foreground">{row.platform}</span>
+ {!row.fallbackEnabled && (
+ <span className="rounded-full bg-muted px-1.5 py-0.5 text-[10px] text-muted-foreground">
+ {t('penaltyInspector.offChain')}
+ </span>
+ )}
+ </div>
+ <code className="mt-0.5 block truncate font-mono text-[11px] text-muted-foreground">{row.modelId}</code>
+ <div className="mt-2 flex flex-wrap gap-1">
+ {row.reasons.map(reason => (
+ <span key={reason} className="rounded-full bg-muted px-1.5 py-0.5 text-[10px] text-muted-foreground">
+ {t(`penaltyInspector.reasons.${reason}`)}
+ </span>
+ ))}
+ </div>
+ </div>
+
+ <div className="flex flex-col gap-2 text-xs">
+ <div className="flex flex-wrap items-center gap-1.5">
+ <Gauge className="size-3.5 text-muted-foreground" />
+ <span className={`rounded-full px-2 py-0.5 tabular-nums ${penaltyClass(row.penalty.value)}`}>
+ {t('penaltyInspector.penaltyValue', { value: row.penalty.value })}
+ </span>
+ {row.penalty.value > 0 && (
+ <span className="text-muted-foreground tabular-nums">
+ {t('penaltyInspector.factor', { factor: row.penalty.rateLimitFactor.toFixed(2) })}
+ </span>
+ )}
+ </div>
+ <div className="flex flex-wrap items-center gap-1.5">
+ <Clock3 className="size-3.5 text-muted-foreground" />
+ {row.cooldowns.length === 0 ? (
+ <span className="text-muted-foreground">{t('penaltyInspector.noCooldown')}</span>
+ ) : row.cooldowns.map(cooldown => (
+ <span key={`${cooldown.keyId}:${cooldown.expiresAtMs}`} className="rounded-full bg-sky-600/15 px-2 py-0.5 text-sky-700 dark:text-sky-400">
+ {t('penaltyInspector.cooldownChip', {
+ key: cooldown.keyLabel || `#${cooldown.keyId}`,
+ time: formatDuration(cooldown.expiresInMs),
+ })}
+ </span>
+ ))}
+ </div>
+ </div>
+
+ <div className="min-w-0">
+ <div className="mb-1 text-xs text-muted-foreground">
+ {t('penaltyInspector.errorsHeading', { count: row.recentErrorCount })}
+ </div>
+ {row.recentErrors.length === 0 ? (
+ <p className="text-xs text-muted-foreground">{t('penaltyInspector.noRecentErrors')}</p>
+ ) : (
+ <div className="space-y-1">
+ {row.recentErrors.map(error => (
+ <div key={error.id} className="min-w-0 text-xs">
+ <span className="mr-2 text-muted-foreground tabular-nums">{formatTime(error.createdAt)}</span>
+ <span className="mr-2 text-muted-foreground">{error.keyLabel || (error.keyId ? `#${error.keyId}` : t('penaltyInspector.noKey'))}</span>
+ <span title={error.error} className="break-words">{error.error}</span>
+ </div>
+ ))}
+ </div>
+ )}
+ </div>
+ </div>
+ ))}
+ </div>
+ </section>
+ )
+}
diff --git a/client/src/i18n/locales/en.json b/client/src/i18n/locales/en.json
index 4d8960c9..b1aae20e 100644
--- a/client/src/i18n/locales/en.json
+++ b/client/src/i18n/locales/en.json
@@ -166,6 +166,24 @@
"weightRequired": "At least one weight must be above zero.",
"scoreColumn": "Score"
},
+ "penaltyInspector": {
+ "title": "Router pressure",
+ "subtitle": "Penalties, cooldowns, and failures from the last {minutes} min",
+ "rowCount": "{count} active",
+ "offChain": "off chain",
+ "penaltyValue": "Penalty {value}",
+ "factor": "score x{factor}",
+ "noCooldown": "No cooldown",
+ "cooldownChip": "{key} - {time}",
+ "errorsHeading": "{count} recent errors",
+ "noRecentErrors": "No recent errors",
+ "noKey": "no key",
+ "reasons": {
+ "penalty": "penalized",
+ "cooldown": "cooldown",
+ "recent_errors": "failing"
+ }
+ },
"keys": {
"pageTitle": "Keys",
"pageDescription": "Provider credentials and the unified API key your apps connect with.",
diff --git a/client/src/pages/FallbackPage.tsx b/client/src/pages/FallbackPage.tsx
index 16875532..5ba80c73 100644
--- a/client/src/pages/FallbackPage.tsx
+++ b/client/src/pages/FallbackPage.tsx
@@ -29,6 +29,7 @@ import { PageHeader } from '@/components/page-header'
import { FloatingBar } from '@/components/floating-bar'
import { ModelsTabs } from '@/components/models-tabs'
import { Tooltip } from '@/components/tooltip'
+import { PenaltyInspector } from '@/components/penalty-inspector'
export interface FallbackEntry {
modelDbId: number
@@ -854,6 +855,8 @@ export default function FallbackPage() {
</p>
</section>
+ <PenaltyInspector />
+
{/* Unified routing / fallback table */}
{isLoading ? (
<p className="text-sm text-muted-foreground">{t('common.loading')}</p>
diff --git a/server/src/__tests__/routes/fallback.test.ts b/server/src/__tests__/routes/fallback.test.ts
index 17631f61..4d479570 100644
--- a/server/src/__tests__/routes/fallback.test.ts
+++ b/server/src/__tests__/routes/fallback.test.ts
@@ -1,7 +1,10 @@
import { describe, it, expect, beforeAll } from 'vitest';
import type { Express } from 'express';
import { createApp } from '../../app.js';
-import { initDb } from '../../db/index.js';
+import { getDb, initDb } from '../../db/index.js';
+import { encrypt } from '../../lib/crypto.js';
+import { recordRateLimitHit } from '../../services/router.js';
+import { setCooldown } from '../../services/ratelimit.js';
import { mintDashboardToken, isGatedApiPath } from '../helpers/auth.js';
let dashToken = '';
@@ -73,6 +76,81 @@ describe('Fallback API', () => {
}
});
+ it('GET /api/fallback/penalty-inspector is empty when no model has active pressure', async () => {
+ const { status, body } = await request(app, 'GET', '/api/fallback/penalty-inspector');
+ expect(status).toBe(200);
+ expect(body.lookbackMinutes).toBe(30);
+ expect(body.rows).toEqual([]);
+ });
+
+ it('GET /api/fallback/penalty-inspector combines penalties, cooldowns and recent errors', async () => {
+ const db = getDb();
+ const target = db.prepare(`
+ SELECT id, platform, model_id, display_name
+ FROM models
+ WHERE platform = 'groq' AND key_id IS NULL
+ ORDER BY id
+ LIMIT 1
+ `).get() as { id: number; platform: string; model_id: string; display_name: string };
+ const secret = encrypt('inspector-test-key');
+ const keyId = Number(db.prepare(`
+ INSERT INTO api_keys (platform, label, encrypted_key, iv, auth_tag, status, enabled)
+ VALUES (?, 'inspector', ?, ?, ?, 'healthy', 1)
+ `).run(target.platform, secret.encrypted, secret.iv, secret.authTag).lastInsertRowid);
+
+ recordRateLimitHit(target.id);
+ recordRateLimitHit(target.id);
+ setCooldown(target.platform, target.model_id, keyId, 60_000);
+ db.prepare(`
+ INSERT INTO requests (platform, model_id, key_id, status, latency_ms, error)
+ VALUES (?, ?, ?, 'error', 321, 'Groq API error 429: rate limit')
+ `).run(target.platform, target.model_id, keyId);
+
+ const { status, body } = await request(app, 'GET', '/api/fallback/penalty-inspector');
+ expect(status).toBe(200);
+ const row = body.rows.find((r: any) => r.modelDbId === target.id);
+ expect(row).toBeDefined();
+ expect(row.displayName).toBe(target.display_name);
+ expect(row.reasons.sort()).toEqual(['cooldown', 'penalty', 'recent_errors']);
+ expect(row.penalty.value).toBeGreaterThan(0);
+ expect(row.penalty.hits).toBeGreaterThanOrEqual(2);
+ expect(row.penalty.rateLimitFactor).toBeLessThan(1);
+ expect(row.cooldowns).toHaveLength(1);
+ expect(row.cooldowns[0]).toMatchObject({ keyId, keyLabel: 'inspector', keyStatus: 'healthy' });
+ expect(row.cooldowns[0].expiresInMs).toBeGreaterThan(0);
+ expect(row.recentErrorCount).toBe(1);
+ expect(row.recentErrors[0]).toMatchObject({
+ keyId,
+ keyLabel: 'inspector',
+ latencyMs: 321,
+ error: 'Groq API error 429: rate limit',
+ });
+ });
+
+ it('GET /api/fallback/penalty-inspector includes recent failures even without cooldowns', async () => {
+ const db = getDb();
+ const target = db.prepare(`
+ SELECT id, platform, model_id
+ FROM models
+ WHERE platform != 'groq'
+ ORDER BY id
+ LIMIT 1
+ `).get() as { id: number; platform: string; model_id: string };
+ db.prepare(`
+ INSERT INTO requests (platform, model_id, key_id, status, latency_ms, error)
+ VALUES (?, ?, NULL, 'error', 111, 'upstream 503 unavailable')
+ `).run(target.platform, target.model_id);
+
+ const { status, body } = await request(app, 'GET', '/api/fallback/penalty-inspector');
+ expect(status).toBe(200);
+ const row = body.rows.find((r: any) => r.modelDbId === target.id);
+ expect(row).toBeDefined();
+ expect(row.reasons).toEqual(['recent_errors']);
+ expect(row.penalty.value).toBe(0);
+ expect(row.cooldowns).toEqual([]);
+ expect(row.recentErrorCount).toBe(1);
+ });
+
it('PUT /api/fallback/routing accepts the custom strategy with weights and persists them', async () => {
const put = await request(app, 'PUT', '/api/fallback/routing', {
strategy: 'custom',
diff --git a/server/src/routes/fallback.ts b/server/src/routes/fallback.ts
index c6065894..21808001 100644
--- a/server/src/routes/fallback.ts
+++ b/server/src/routes/fallback.ts
@@ -11,6 +11,7 @@ import { getAllPenalties, getRoutingScores, getRoutingStrategy, setRoutingStrate
import { BANDIT_PRESETS, type RoutingStrategy } from '../services/scoring.js';
import { parseBudget } from '../lib/budget.js';
import { getModelGroups } from '../services/model-groups.js';
+import { getPenaltyInspector } from '../services/penalty-inspector.js';
export const fallbackRouter = Router();
@@ -21,6 +22,10 @@ fallbackRouter.get('/routing', (_req: Request, res: Response) => {
res.json(getRoutingScores());
});
+fallbackRouter.get('/penalty-inspector', (_req: Request, res: Response) => {
+ res.json(getPenaltyInspector());
+});
+
const routingSchema = z.object({
strategy: z.enum(['priority', 'balanced', 'smartest', 'fastest', 'reliable', 'custom']),
// Only meaningful with strategy 'custom': the user's weight vector. Any
diff --git a/server/src/services/penalty-inspector.ts b/server/src/services/penalty-inspector.ts
new file mode 100644
index 00000000..fc300c15
--- /dev/null
+++ b/server/src/services/penalty-inspector.ts
@@ -0,0 +1,245 @@
+import { getDb } from '../db/index.js';
+import { getAllPenalties } from './router.js';
+import { rateLimitFactor } from './scoring.js';
+
+const INSPECTOR_LOOKBACK_MINUTES = 30;
+const MAX_ERRORS_PER_MODEL = 5;
+
+export type InspectorReason = 'penalty' | 'cooldown' | 'recent_errors';
+
+export interface InspectorRow {
+ modelDbId: number | null;
+ platform: string;
+ modelId: string;
+ displayName: string;
+ enabled: boolean;
+ fallbackEnabled: boolean;
+ priority: number | null;
+ penalty: {
+ hits: number;
+ value: number;
+ rateLimitFactor: number;
+ };
+ cooldowns: Array<{
+ keyId: number;
+ keyLabel: string | null;
+ keyStatus: string | null;
+ expiresAtMs: number;
+ expiresInMs: number;
+ }>;
+ recentErrors: Array<{
+ id: number;
+ keyId: number | null;
+ keyLabel: string | null;
+ error: string;
+ latencyMs: number;
+ createdAt: string;
+ }>;
+ recentErrorCount: number;
+ reasons: InspectorReason[];
+}
+
+export interface PenaltyInspectorSnapshot {
+ generatedAtMs: number;
+ lookbackMinutes: number;
+ rows: InspectorRow[];
+}
+
+function toSqliteDateTime(timestamp: number): string {
+ return new Date(timestamp).toISOString().slice(0, 19).replace('T', ' ');
+}
+
+function rowKey(modelDbId: number | null, platform: string, modelId: string): string {
+ return modelDbId != null ? `id:${modelDbId}` : `model:${platform}:${modelId}`;
+}
+
+function ensureInspectorRow(
+ rows: Map<string, InspectorRow>,
+ input: {
+ modelDbId: number | null;
+ platform: string;
+ modelId: string;
+ displayName?: string | null;
+ modelEnabled?: number | null;
+ fallbackEnabled?: number | null;
+ priority?: number | null;
+ },
+): InspectorRow {
+ const key = rowKey(input.modelDbId, input.platform, input.modelId);
+ const existing = rows.get(key);
+ if (existing) return existing;
+
+ const row: InspectorRow = {
+ modelDbId: input.modelDbId,
+ platform: input.platform,
+ modelId: input.modelId,
+ displayName: input.displayName ?? input.modelId,
+ enabled: input.modelEnabled !== 0,
+ fallbackEnabled: input.fallbackEnabled !== 0,
+ priority: input.priority ?? null,
+ penalty: { hits: 0, value: 0, rateLimitFactor: 1 },
+ cooldowns: [],
+ recentErrors: [],
+ recentErrorCount: 0,
+ reasons: [],
+ };
+ rows.set(key, row);
+ return row;
+}
+
+function addReason(row: InspectorRow, reason: InspectorReason): void {
+ if (!row.reasons.includes(reason)) row.reasons.push(reason);
+}
+
+export function getPenaltyInspector(): PenaltyInspectorSnapshot {
+ const db = getDb();
+ const now = Date.now();
+ const rows = new Map<string, InspectorRow>();
+
+ const penalties = getAllPenalties();
+ if (penalties.length > 0) {
+ const placeholders = penalties.map(() => '?').join(',');
+ const models = db.prepare(`
+ SELECT m.id AS model_db_id, m.platform, m.model_id, m.display_name,
+ m.enabled AS model_enabled, fc.enabled AS fallback_enabled, fc.priority
+ FROM models m
+ LEFT JOIN fallback_config fc ON fc.model_db_id = m.id
+ WHERE m.id IN (${placeholders})
+ `).all(...penalties.map(p => p.modelDbId)) as Array<{
+ model_db_id: number;
+ platform: string;
+ model_id: string;
+ display_name: string;
+ model_enabled: number;
+ fallback_enabled: number | null;
+ priority: number | null;
+ }>;
+ const modelById = new Map(models.map(m => [m.model_db_id, m]));
+ for (const p of penalties) {
+ const model = modelById.get(p.modelDbId);
+ if (!model) continue;
+ const row = ensureInspectorRow(rows, {
+ modelDbId: model.model_db_id,
+ platform: model.platform,
+ modelId: model.model_id,
+ displayName: model.display_name,
+ modelEnabled: model.model_enabled,
+ fallbackEnabled: model.fallback_enabled,
+ priority: model.priority,
+ });
+ row.penalty = { hits: p.count, value: p.penalty, rateLimitFactor: rateLimitFactor(p.penalty) };
+ addReason(row, 'penalty');
+ }
+ }
+
+ const cooldownRows = db.prepare(`
+ SELECT c.platform, c.model_id, c.key_id, c.expires_at_ms,
+ m.id AS model_db_id, m.display_name, m.enabled AS model_enabled,
+ fc.enabled AS fallback_enabled, fc.priority,
+ ak.label AS key_label, ak.status AS key_status
+ FROM rate_limit_cooldowns c
+ LEFT JOIN models m ON m.platform = c.platform AND m.model_id = c.model_id
+ LEFT JOIN fallback_config fc ON fc.model_db_id = m.id
+ LEFT JOIN api_keys ak ON ak.id = c.key_id
+ WHERE c.expires_at_ms > ?
+ ORDER BY c.expires_at_ms ASC
+ `).all(now) as Array<{
+ platform: string;
+ model_id: string;
+ key_id: number;
+ expires_at_ms: number;
+ model_db_id: number | null;
+ display_name: string | null;
+ model_enabled: number | null;
+ fallback_enabled: number | null;
+ priority: number | null;
+ key_label: string | null;
+ key_status: string | null;
+ }>;
+ for (const c of cooldownRows) {
+ const row = ensureInspectorRow(rows, {
+ modelDbId: c.model_db_id,
+ platform: c.platform,
+ modelId: c.model_id,
+ displayName: c.display_name,
+ modelEnabled: c.model_enabled,
+ fallbackEnabled: c.fallback_enabled,
+ priority: c.priority,
+ });
+ row.cooldowns.push({
+ keyId: c.key_id,
+ keyLabel: c.key_label,
+ keyStatus: c.key_status,
+ expiresAtMs: c.expires_at_ms,
+ expiresInMs: Math.max(0, c.expires_at_ms - now),
+ });
+ addReason(row, 'cooldown');
+ }
+
+ const since = toSqliteDateTime(now - INSPECTOR_LOOKBACK_MINUTES * 60 * 1000);
+ const errorRows = db.prepare(`
+ SELECT r.id, r.platform, r.model_id, r.key_id, r.error, r.latency_ms, r.created_at,
+ m.id AS model_db_id, m.display_name, m.enabled AS model_enabled,
+ fc.enabled AS fallback_enabled, fc.priority,
+ ak.label AS key_label
+ FROM requests r
+ LEFT JOIN models m ON m.platform = r.platform AND m.model_id = r.model_id
+ LEFT JOIN fallback_config fc ON fc.model_db_id = m.id
+ LEFT JOIN api_keys ak ON ak.id = r.key_id
+ WHERE r.status = 'error'
+ AND r.created_at >= ?
+ ORDER BY r.created_at DESC, r.id DESC
+ LIMIT 250
+ `).all(since) as Array<{
+ id: number;
+ platform: string;
+ model_id: string;
+ key_id: number | null;
+ error: string | null;
+ latency_ms: number;
+ created_at: string;
+ model_db_id: number | null;
+ display_name: string | null;
+ model_enabled: number | null;
+ fallback_enabled: number | null;
+ priority: number | null;
+ key_label: string | null;
+ }>;
+ for (const e of errorRows) {
+ const row = ensureInspectorRow(rows, {
+ modelDbId: e.model_db_id,
+ platform: e.platform,
+ modelId: e.model_id,
+ displayName: e.display_name,
+ modelEnabled: e.model_enabled,
+ fallbackEnabled: e.fallback_enabled,
+ priority: e.priority,
+ });
+ row.recentErrorCount++;
+ if (row.recentErrors.length < MAX_ERRORS_PER_MODEL) {
+ row.recentErrors.push({
+ id: e.id,
+ keyId: e.key_id,
+ keyLabel: e.key_label,
+ error: e.error ?? 'Unknown upstream error',
+ latencyMs: e.latency_ms,
+ createdAt: e.created_at,
+ });
+ }
+ addReason(row, 'recent_errors');
+ }
+
+ const ordered = [...rows.values()].sort((a, b) =>
+ b.penalty.value - a.penalty.value ||
+ b.cooldowns.length - a.cooldowns.length ||
+ b.recentErrorCount - a.recentErrorCount ||
+ (a.priority ?? Number.MAX_SAFE_INTEGER) - (b.priority ?? Number.MAX_SAFE_INTEGER) ||
+ a.displayName.localeCompare(b.displayName),
+ );
+
+ return {
+ generatedAtMs: now,
+ lookbackMinutes: INSPECTOR_LOOKBACK_MINUTES,
+ rows: ordered,
+ };
+}