7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.
TypeScript
Add router penalty inspector
Tashfeenbc07927985bd
diff --git a/client/src/components/penalty-inspector.tsx b/client/src/components/penalty-inspector.tsxnew file mode 100644index 00000000..28aad46c--- /dev/null+++ b/client/src/components/penalty-inspector.tsx@@ -0,0 +1,170 @@+import { useQuery } from '@tanstack/react-query'+import { AlertTriangle, Clock3, Gauge } from 'lucide-react'+import { useI18n } from '@/i18n'+import { apiFetch } from '@/lib/api'++type InspectorReason = 'penalty' | 'cooldown' | 'recent_errors'++interface PenaltyInspectorRow {+ modelDbId: number | null+ platform: string+ modelId: string+ displayName: string+ enabled: boolean+ fallbackEnabled: boolean+ priority: number | null+ penalty: {+ hits: number+ value: number+ rateLimitFactor: number+ }+ cooldowns: Array<{+ keyId: number+ keyLabel: string | null+ keyStatus: string | null+ expiresAtMs: number+ expiresInMs: number+ }>+ recentErrors: Array<{+ id: number+ keyId: number | null+ keyLabel: string | null+ error: string+ latencyMs: number+ createdAt: string+ }>+ recentErrorCount: number+ reasons: InspectorReason[]+}++interface PenaltyInspectorData {+ generatedAtMs: number+ lookbackMinutes: number+ rows: PenaltyInspectorRow[]+}++function formatDuration(ms: number): string {+ const seconds = Math.max(0, Math.ceil(ms / 1000))+ if (seconds < 60) return `${seconds}s`+ const minutes = Math.ceil(seconds / 60)+ if (minutes < 60) return `${minutes}m`+ const hours = Math.ceil(minutes / 60)+ return `${hours}h`+}++function formatTime(value: string): string {+ const date = new Date(`${value.replace(' ', 'T')}Z`)+ if (Number.isNaN(date.getTime())) return value+ return date.toLocaleTimeString([], { hour: '2-digit', minute: '2-digit' })+}++function penaltyClass(value: number): string {+ if (value >= 8) return 'bg-red-600/15 text-red-700 dark:text-red-400'+ if (value >= 5) return 'bg-orange-600/15 text-orange-700 dark:text-orange-400'+ if (value >= 3) return 'bg-amber-600/15 text-amber-700 dark:text-amber-400'+ if (value > 0) return 'bg-emerald-600/15 text-emerald-700 dark:text-emerald-400'+ return 'bg-muted text-muted-foreground'+}++export function PenaltyInspector() {+ const { t } = useI18n()+ const { data } = useQuery<PenaltyInspectorData>({+ queryKey: ['fallback', 'penalty-inspector'],+ queryFn: () => apiFetch('/api/fallback/penalty-inspector'),+ refetchInterval: 5_000,+ })++ const rows = data?.rows ?? []+ if (rows.length === 0) return null++ return (+ <section className="rounded-lg border bg-card">+ <div className="flex flex-wrap items-center justify-between gap-2 border-b px-4 py-3">+ <div className="flex items-center gap-2">+ <AlertTriangle className="size-4 text-amber-600 dark:text-amber-400" />+ <div>+ <h2 className="text-sm font-medium">{t('penaltyInspector.title')}</h2>+ <p className="text-xs text-muted-foreground">+ {t('penaltyInspector.subtitle', { minutes: data?.lookbackMinutes ?? 30 })}+ </p>+ </div>+ </div>+ <span className="rounded-full bg-muted px-2 py-0.5 text-[11px] text-muted-foreground tabular-nums">+ {t('penaltyInspector.rowCount', { count: rows.length })}+ </span>+ </div>++ <div className="divide-y">+ {rows.map(row => (+ <div key={`${row.platform}:${row.modelId}:${row.modelDbId ?? 'missing'}`} className="grid gap-3 px-4 py-3 lg:grid-cols-[minmax(14rem,1.2fr)_minmax(10rem,0.8fr)_minmax(14rem,1.3fr)]">+ <div className="min-w-0">+ <div className="flex flex-wrap items-center gap-1.5">+ <span className="truncate text-sm font-medium">{row.displayName}</span>+ <span className="text-xs text-muted-foreground">{row.platform}</span>+ {!row.fallbackEnabled && (+ <span className="rounded-full bg-muted px-1.5 py-0.5 text-[10px] text-muted-foreground">+ {t('penaltyInspector.offChain')}+ </span>+ )}+ </div>+ <code className="mt-0.5 block truncate font-mono text-[11px] text-muted-foreground">{row.modelId}</code>+ <div className="mt-2 flex flex-wrap gap-1">+ {row.reasons.map(reason => (+ <span key={reason} className="rounded-full bg-muted px-1.5 py-0.5 text-[10px] text-muted-foreground">+ {t(`penaltyInspector.reasons.${reason}`)}+ </span>+ ))}+ </div>+ </div>++ <div className="flex flex-col gap-2 text-xs">+ <div className="flex flex-wrap items-center gap-1.5">+ <Gauge className="size-3.5 text-muted-foreground" />+ <span className={`rounded-full px-2 py-0.5 tabular-nums ${penaltyClass(row.penalty.value)}`}>+ {t('penaltyInspector.penaltyValue', { value: row.penalty.value })}+ </span>+ {row.penalty.value > 0 && (+ <span className="text-muted-foreground tabular-nums">+ {t('penaltyInspector.factor', { factor: row.penalty.rateLimitFactor.toFixed(2) })}+ </span>+ )}+ </div>+ <div className="flex flex-wrap items-center gap-1.5">+ <Clock3 className="size-3.5 text-muted-foreground" />+ {row.cooldowns.length === 0 ? (+ <span className="text-muted-foreground">{t('penaltyInspector.noCooldown')}</span>+ ) : row.cooldowns.map(cooldown => (+ <span key={`${cooldown.keyId}:${cooldown.expiresAtMs}`} className="rounded-full bg-sky-600/15 px-2 py-0.5 text-sky-700 dark:text-sky-400">+ {t('penaltyInspector.cooldownChip', {+ key: cooldown.keyLabel || `#${cooldown.keyId}`,+ time: formatDuration(cooldown.expiresInMs),+ })}+ </span>+ ))}+ </div>+ </div>++ <div className="min-w-0">+ <div className="mb-1 text-xs text-muted-foreground">+ {t('penaltyInspector.errorsHeading', { count: row.recentErrorCount })}+ </div>+ {row.recentErrors.length === 0 ? (+ <p className="text-xs text-muted-foreground">{t('penaltyInspector.noRecentErrors')}</p>+ ) : (+ <div className="space-y-1">+ {row.recentErrors.map(error => (+ <div key={error.id} className="min-w-0 text-xs">+ <span className="mr-2 text-muted-foreground tabular-nums">{formatTime(error.createdAt)}</span>+ <span className="mr-2 text-muted-foreground">{error.keyLabel || (error.keyId ? `#${error.keyId}` : t('penaltyInspector.noKey'))}</span>+ <span title={error.error} className="break-words">{error.error}</span>+ </div>+ ))}+ </div>+ )}+ </div>+ </div>+ ))}+ </div>+ </section>+ )+}diff --git a/client/src/i18n/locales/en.json b/client/src/i18n/locales/en.jsonindex 4d8960c9..b1aae20e 100644--- a/client/src/i18n/locales/en.json+++ b/client/src/i18n/locales/en.json@@ -166,6 +166,24 @@"weightRequired": "At least one weight must be above zero.","scoreColumn": "Score"},+ "penaltyInspector": {+ "title": "Router pressure",+ "subtitle": "Penalties, cooldowns, and failures from the last {minutes} min",+ "rowCount": "{count} active",+ "offChain": "off chain",+ "penaltyValue": "Penalty {value}",+ "factor": "score x{factor}",+ "noCooldown": "No cooldown",+ "cooldownChip": "{key} - {time}",+ "errorsHeading": "{count} recent errors",+ "noRecentErrors": "No recent errors",+ "noKey": "no key",+ "reasons": {+ "penalty": "penalized",+ "cooldown": "cooldown",+ "recent_errors": "failing"+ }+ },"keys": {"pageTitle": "Keys","pageDescription": "Provider credentials and the unified API key your apps connect with.",diff --git a/client/src/pages/FallbackPage.tsx b/client/src/pages/FallbackPage.tsxindex 16875532..5ba80c73 100644--- a/client/src/pages/FallbackPage.tsx+++ b/client/src/pages/FallbackPage.tsx@@ -29,6 +29,7 @@ import { PageHeader } from '@/components/page-header'import { FloatingBar } from '@/components/floating-bar'import { ModelsTabs } from '@/components/models-tabs'import { Tooltip } from '@/components/tooltip'+import { PenaltyInspector } from '@/components/penalty-inspector'export interface FallbackEntry {modelDbId: number@@ -854,6 +855,8 @@ export default function FallbackPage() {</p></section>+ <PenaltyInspector />+{/* Unified routing / fallback table */}{isLoading ? (<p className="text-sm text-muted-foreground">{t('common.loading')}</p>diff --git a/server/src/__tests__/routes/fallback.test.ts b/server/src/__tests__/routes/fallback.test.tsindex 17631f61..4d479570 100644--- a/server/src/__tests__/routes/fallback.test.ts+++ b/server/src/__tests__/routes/fallback.test.ts@@ -1,7 +1,10 @@import { describe, it, expect, beforeAll } from 'vitest';import type { Express } from 'express';import { createApp } from '../../app.js';-import { initDb } from '../../db/index.js';+import { getDb, initDb } from '../../db/index.js';+import { encrypt } from '../../lib/crypto.js';+import { recordRateLimitHit } from '../../services/router.js';+import { setCooldown } from '../../services/ratelimit.js';import { mintDashboardToken, isGatedApiPath } from '../helpers/auth.js';let dashToken = '';@@ -73,6 +76,81 @@ describe('Fallback API', () => {}});+ it('GET /api/fallback/penalty-inspector is empty when no model has active pressure', async () => {+ const { status, body } = await request(app, 'GET', '/api/fallback/penalty-inspector');+ expect(status).toBe(200);+ expect(body.lookbackMinutes).toBe(30);+ expect(body.rows).toEqual([]);+ });++ it('GET /api/fallback/penalty-inspector combines penalties, cooldowns and recent errors', async () => {+ const db = getDb();+ const target = db.prepare(`+ SELECT id, platform, model_id, display_name+ FROM models+ WHERE platform = 'groq' AND key_id IS NULL+ ORDER BY id+ LIMIT 1+ `).get() as { id: number; platform: string; model_id: string; display_name: string };+ const secret = encrypt('inspector-test-key');+ const keyId = Number(db.prepare(`+ INSERT INTO api_keys (platform, label, encrypted_key, iv, auth_tag, status, enabled)+ VALUES (?, 'inspector', ?, ?, ?, 'healthy', 1)+ `).run(target.platform, secret.encrypted, secret.iv, secret.authTag).lastInsertRowid);++ recordRateLimitHit(target.id);+ recordRateLimitHit(target.id);+ setCooldown(target.platform, target.model_id, keyId, 60_000);+ db.prepare(`+ INSERT INTO requests (platform, model_id, key_id, status, latency_ms, error)+ VALUES (?, ?, ?, 'error', 321, 'Groq API error 429: rate limit')+ `).run(target.platform, target.model_id, keyId);++ const { status, body } = await request(app, 'GET', '/api/fallback/penalty-inspector');+ expect(status).toBe(200);+ const row = body.rows.find((r: any) => r.modelDbId === target.id);+ expect(row).toBeDefined();+ expect(row.displayName).toBe(target.display_name);+ expect(row.reasons.sort()).toEqual(['cooldown', 'penalty', 'recent_errors']);+ expect(row.penalty.value).toBeGreaterThan(0);+ expect(row.penalty.hits).toBeGreaterThanOrEqual(2);+ expect(row.penalty.rateLimitFactor).toBeLessThan(1);+ expect(row.cooldowns).toHaveLength(1);+ expect(row.cooldowns[0]).toMatchObject({ keyId, keyLabel: 'inspector', keyStatus: 'healthy' });+ expect(row.cooldowns[0].expiresInMs).toBeGreaterThan(0);+ expect(row.recentErrorCount).toBe(1);+ expect(row.recentErrors[0]).toMatchObject({+ keyId,+ keyLabel: 'inspector',+ latencyMs: 321,+ error: 'Groq API error 429: rate limit',+ });+ });++ it('GET /api/fallback/penalty-inspector includes recent failures even without cooldowns', async () => {+ const db = getDb();+ const target = db.prepare(`+ SELECT id, platform, model_id+ FROM models+ WHERE platform != 'groq'+ ORDER BY id+ LIMIT 1+ `).get() as { id: number; platform: string; model_id: string };+ db.prepare(`+ INSERT INTO requests (platform, model_id, key_id, status, latency_ms, error)+ VALUES (?, ?, NULL, 'error', 111, 'upstream 503 unavailable')+ `).run(target.platform, target.model_id);++ const { status, body } = await request(app, 'GET', '/api/fallback/penalty-inspector');+ expect(status).toBe(200);+ const row = body.rows.find((r: any) => r.modelDbId === target.id);+ expect(row).toBeDefined();+ expect(row.reasons).toEqual(['recent_errors']);+ expect(row.penalty.value).toBe(0);+ expect(row.cooldowns).toEqual([]);+ expect(row.recentErrorCount).toBe(1);+ });+it('PUT /api/fallback/routing accepts the custom strategy with weights and persists them', async () => {const put = await request(app, 'PUT', '/api/fallback/routing', {strategy: 'custom',diff --git a/server/src/routes/fallback.ts b/server/src/routes/fallback.tsindex c6065894..21808001 100644--- a/server/src/routes/fallback.ts+++ b/server/src/routes/fallback.ts@@ -11,6 +11,7 @@ import { getAllPenalties, getRoutingScores, getRoutingStrategy, setRoutingStrateimport { BANDIT_PRESETS, type RoutingStrategy } from '../services/scoring.js';import { parseBudget } from '../lib/budget.js';import { getModelGroups } from '../services/model-groups.js';+import { getPenaltyInspector } from '../services/penalty-inspector.js';export const fallbackRouter = Router();@@ -21,6 +22,10 @@ fallbackRouter.get('/routing', (_req: Request, res: Response) => {res.json(getRoutingScores());});+fallbackRouter.get('/penalty-inspector', (_req: Request, res: Response) => {+ res.json(getPenaltyInspector());+});+const routingSchema = z.object({strategy: z.enum(['priority', 'balanced', 'smartest', 'fastest', 'reliable', 'custom']),// Only meaningful with strategy 'custom': the user's weight vector. Anydiff --git a/server/src/services/penalty-inspector.ts b/server/src/services/penalty-inspector.tsnew file mode 100644index 00000000..fc300c15--- /dev/null+++ b/server/src/services/penalty-inspector.ts@@ -0,0 +1,245 @@+import { getDb } from '../db/index.js';+import { getAllPenalties } from './router.js';+import { rateLimitFactor } from './scoring.js';++const INSPECTOR_LOOKBACK_MINUTES = 30;+const MAX_ERRORS_PER_MODEL = 5;++export type InspectorReason = 'penalty' | 'cooldown' | 'recent_errors';++export interface InspectorRow {+ modelDbId: number | null;+ platform: string;+ modelId: string;+ displayName: string;+ enabled: boolean;+ fallbackEnabled: boolean;+ priority: number | null;+ penalty: {+ hits: number;+ value: number;+ rateLimitFactor: number;+ };+ cooldowns: Array<{+ keyId: number;+ keyLabel: string | null;+ keyStatus: string | null;+ expiresAtMs: number;+ expiresInMs: number;+ }>;+ recentErrors: Array<{+ id: number;+ keyId: number | null;+ keyLabel: string | null;+ error: string;+ latencyMs: number;+ createdAt: string;+ }>;+ recentErrorCount: number;+ reasons: InspectorReason[];+}++export interface PenaltyInspectorSnapshot {+ generatedAtMs: number;+ lookbackMinutes: number;+ rows: InspectorRow[];+}++function toSqliteDateTime(timestamp: number): string {+ return new Date(timestamp).toISOString().slice(0, 19).replace('T', ' ');+}++function rowKey(modelDbId: number | null, platform: string, modelId: string): string {+ return modelDbId != null ? `id:${modelDbId}` : `model:${platform}:${modelId}`;+}++function ensureInspectorRow(+ rows: Map<string, InspectorRow>,+ input: {+ modelDbId: number | null;+ platform: string;+ modelId: string;+ displayName?: string | null;+ modelEnabled?: number | null;+ fallbackEnabled?: number | null;+ priority?: number | null;+ },+): InspectorRow {+ const key = rowKey(input.modelDbId, input.platform, input.modelId);+ const existing = rows.get(key);+ if (existing) return existing;++ const row: InspectorRow = {+ modelDbId: input.modelDbId,+ platform: input.platform,+ modelId: input.modelId,+ displayName: input.displayName ?? input.modelId,+ enabled: input.modelEnabled !== 0,+ fallbackEnabled: input.fallbackEnabled !== 0,+ priority: input.priority ?? null,+ penalty: { hits: 0, value: 0, rateLimitFactor: 1 },+ cooldowns: [],+ recentErrors: [],+ recentErrorCount: 0,+ reasons: [],+ };+ rows.set(key, row);+ return row;+}++function addReason(row: InspectorRow, reason: InspectorReason): void {+ if (!row.reasons.includes(reason)) row.reasons.push(reason);+}++export function getPenaltyInspector(): PenaltyInspectorSnapshot {+ const db = getDb();+ const now = Date.now();+ const rows = new Map<string, InspectorRow>();++ const penalties = getAllPenalties();+ if (penalties.length > 0) {+ const placeholders = penalties.map(() => '?').join(',');+ const models = db.prepare(`+ SELECT m.id AS model_db_id, m.platform, m.model_id, m.display_name,+ m.enabled AS model_enabled, fc.enabled AS fallback_enabled, fc.priority+ FROM models m+ LEFT JOIN fallback_config fc ON fc.model_db_id = m.id+ WHERE m.id IN (${placeholders})+ `).all(...penalties.map(p => p.modelDbId)) as Array<{+ model_db_id: number;+ platform: string;+ model_id: string;+ display_name: string;+ model_enabled: number;+ fallback_enabled: number | null;+ priority: number | null;+ }>;+ const modelById = new Map(models.map(m => [m.model_db_id, m]));+ for (const p of penalties) {+ const model = modelById.get(p.modelDbId);+ if (!model) continue;+ const row = ensureInspectorRow(rows, {+ modelDbId: model.model_db_id,+ platform: model.platform,+ modelId: model.model_id,+ displayName: model.display_name,+ modelEnabled: model.model_enabled,+ fallbackEnabled: model.fallback_enabled,+ priority: model.priority,+ });+ row.penalty = { hits: p.count, value: p.penalty, rateLimitFactor: rateLimitFactor(p.penalty) };+ addReason(row, 'penalty');+ }+ }++ const cooldownRows = db.prepare(`+ SELECT c.platform, c.model_id, c.key_id, c.expires_at_ms,+ m.id AS model_db_id, m.display_name, m.enabled AS model_enabled,+ fc.enabled AS fallback_enabled, fc.priority,+ ak.label AS key_label, ak.status AS key_status+ FROM rate_limit_cooldowns c+ LEFT JOIN models m ON m.platform = c.platform AND m.model_id = c.model_id+ LEFT JOIN fallback_config fc ON fc.model_db_id = m.id+ LEFT JOIN api_keys ak ON ak.id = c.key_id+ WHERE c.expires_at_ms > ?+ ORDER BY c.expires_at_ms ASC+ `).all(now) as Array<{+ platform: string;+ model_id: string;+ key_id: number;+ expires_at_ms: number;+ model_db_id: number | null;+ display_name: string | null;+ model_enabled: number | null;+ fallback_enabled: number | null;+ priority: number | null;+ key_label: string | null;+ key_status: string | null;+ }>;+ for (const c of cooldownRows) {+ const row = ensureInspectorRow(rows, {+ modelDbId: c.model_db_id,+ platform: c.platform,+ modelId: c.model_id,+ displayName: c.display_name,+ modelEnabled: c.model_enabled,+ fallbackEnabled: c.fallback_enabled,+ priority: c.priority,+ });+ row.cooldowns.push({+ keyId: c.key_id,+ keyLabel: c.key_label,+ keyStatus: c.key_status,+ expiresAtMs: c.expires_at_ms,+ expiresInMs: Math.max(0, c.expires_at_ms - now),+ });+ addReason(row, 'cooldown');+ }++ const since = toSqliteDateTime(now - INSPECTOR_LOOKBACK_MINUTES * 60 * 1000);+ const errorRows = db.prepare(`+ SELECT r.id, r.platform, r.model_id, r.key_id, r.error, r.latency_ms, r.created_at,+ m.id AS model_db_id, m.display_name, m.enabled AS model_enabled,+ fc.enabled AS fallback_enabled, fc.priority,+ ak.label AS key_label+ FROM requests r+ LEFT JOIN models m ON m.platform = r.platform AND m.model_id = r.model_id+ LEFT JOIN fallback_config fc ON fc.model_db_id = m.id+ LEFT JOIN api_keys ak ON ak.id = r.key_id+ WHERE r.status = 'error'+ AND r.created_at >= ?+ ORDER BY r.created_at DESC, r.id DESC+ LIMIT 250+ `).all(since) as Array<{+ id: number;+ platform: string;+ model_id: string;+ key_id: number | null;+ error: string | null;+ latency_ms: number;+ created_at: string;+ model_db_id: number | null;+ display_name: string | null;+ model_enabled: number | null;+ fallback_enabled: number | null;+ priority: number | null;+ key_label: string | null;+ }>;+ for (const e of errorRows) {+ const row = ensureInspectorRow(rows, {+ modelDbId: e.model_db_id,+ platform: e.platform,+ modelId: e.model_id,+ displayName: e.display_name,+ modelEnabled: e.model_enabled,+ fallbackEnabled: e.fallback_enabled,+ priority: e.priority,+ });+ row.recentErrorCount++;+ if (row.recentErrors.length < MAX_ERRORS_PER_MODEL) {+ row.recentErrors.push({+ id: e.id,+ keyId: e.key_id,+ keyLabel: e.key_label,+ error: e.error ?? 'Unknown upstream error',+ latencyMs: e.latency_ms,+ createdAt: e.created_at,+ });+ }+ addReason(row, 'recent_errors');+ }++ const ordered = [...rows.values()].sort((a, b) =>+ b.penalty.value - a.penalty.value ||+ b.cooldowns.length - a.cooldowns.length ||+ b.recentErrorCount - a.recentErrorCount ||+ (a.priority ?? Number.MAX_SAFE_INTEGER) - (b.priority ?? Number.MAX_SAFE_INTEGER) ||+ a.displayName.localeCompare(b.displayName),+ );++ return {+ generatedAtMs: now,+ lookbackMinutes: INSPECTOR_LOOKBACK_MINUTES,+ rows: ordered,+ };+}