7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.
TypeScript
chore(catalog): intelligence tier audit vs Artificial Analysis Index v4.0 (2026-06)
Tashfeend948cb7393ef
diff --git a/server/src/__tests__/db/intelligence-tiers.test.ts b/server/src/__tests__/db/intelligence-tiers.test.tsnew file mode 100644index 00000000..75574080--- /dev/null+++ b/server/src/__tests__/db/intelligence-tiers.test.ts@@ -0,0 +1,85 @@+import { describe, it, expect, beforeAll } from 'vitest';+import { initDb, getDb } from '../../db/index.js';++// Verifies the V17 intelligence-tier audit (2026-06): size_label is normalized+// to Artificial Analysis Intelligence Index v4.0 bands, and the same model+// family lands in ONE tier regardless of provider.+describe('intelligence tier audit (migrateModelsV17)', () => {+ beforeAll(() => {+ process.env.ENCRYPTION_KEY = '0'.repeat(64);+ initDb(':memory:');+ });++ function tier(platform: string, modelId: string): string | undefined {+ const row = getDb()+ .prepare('SELECT size_label FROM models WHERE platform = ? AND model_id = ?')+ .get(platform, modelId) as { size_label: string } | undefined;+ return row?.size_label;+ }++ function tiersForFamily(like: string): string[] {+ const rows = getDb()+ .prepare("SELECT DISTINCT size_label FROM models WHERE LOWER(model_id) LIKE ?")+ .all(like.toLowerCase()) as { size_label: string }[];+ return rows.map(r => r.size_label);+ }++ it('promotes frontier-class Gemini Flash models (AA 46–55) to Frontier', () => {+ expect(tier('google', 'gemini-3.5-flash')).toBe('Frontier');+ expect(tier('google', 'gemini-3-flash-preview')).toBe('Frontier');+ });++ it('demotes over-tiered models down from Frontier', () => {+ // Qwen3-Coder 480B (AA 25, non-reasoning coder) → Medium+ expect(tier('nvidia', 'qwen/qwen3-coder-480b-a35b-instruct')).toBe('Medium');+ expect(tier('openrouter', 'qwen/qwen3-coder:free')).toBe('Medium');+ // Mistral Large 3 (AA 23) → Medium+ expect(tier('ollama', 'mistral-large-3:675b')).toBe('Medium');+ // Gemini 2.5 Pro (AA 35) → Large+ expect(tier('google', 'gemini-2.5-pro')).toBe('Large');+ // Nemotron 3 Super 120B (AA 36) → Large+ expect(tier('nvidia', 'nvidia/nemotron-3-super-120b-a12b')).toBe('Large');+ });++ it('demotes lapped models down from Large', () => {+ expect(tier('google', 'gemini-2.5-flash')).toBe('Medium'); // AA 21+ expect(tier('github', 'gpt-4o')).toBe('Medium'); // AA 17+ expect(tier('cohere', 'command-a-03-2025')).toBe('Medium'); // AA 13+ expect(tier('ollama', 'devstral-2:123b')).toBe('Medium'); // AA 22+ });++ it('promotes under-tiered Gemma 4 and Gemini Flash-Lite up to Large', () => {+ expect(tier('nvidia', 'google/gemma-4-31b-it')).toBe('Large'); // AA 39+ expect(tier('google', 'gemini-3.1-flash-lite-preview')).toBe('Large'); // AA 34+ });++ it('demotes weak models to Small', () => {+ expect(tier('sambanova', 'gemma-3-12b-it')).toBe('Small'); // AA 9+ expect(tier('mistral', 'codestral-latest')).toBe('Small'); // AA 8+ expect(tier('cohere', 'command-r-08-2024')).toBe('Small'); // legacy+ });++ it('keeps the qwen3-coder vs qwen3-coder-next split correct', () => {+ expect(tier('ollama', 'qwen3-coder:480b')).toBe('Medium');+ expect(tier('ollama', 'qwen3-coder-next')).toBe('Large');+ });++ it('assigns a single consistent tier per model family across providers', () => {+ // The bug the audit also fixes: same model, different tier per provider.+ expect(tiersForFamily('%llama-4-scout%')).toEqual(['Medium']);+ expect(tiersForFamily('%llama-3.3-70b%')).toEqual(['Medium']);+ expect(tiersForFamily('%gpt-oss-120b%')).toEqual(['Large']);+ });++ it('leaves models with no published AA Index at their seeded tier', () => {+ expect(tier('ollama', 'cogito-2.1:671b')).toBe('Frontier');+ expect(tier('openrouter', 'nousresearch/hermes-3-llama-3.1-405b:free')).toBe('Large');+ expect(tier('openrouter', 'poolside/laguna-m.1:free')).toBe('Large');+ });++ it('keeps genuine frontier models at Frontier', () => {+ expect(tier('google', 'gemini-3.1-pro-preview')).toBe('Frontier');+ expect(tier('cloudflare', '@cf/moonshotai/kimi-k2.6')).toBe('Frontier');+ expect(tier('nvidia', 'z-ai/glm-5.1')).toBe('Frontier');+ });+});diff --git a/server/src/db/index.ts b/server/src/db/index.tsindex 3129873f..b4f17029 100644--- a/server/src/db/index.ts+++ b/server/src/db/index.ts@@ -51,6 +51,7 @@ export function initDb(dbPath?: string): Database.Database {migrateModelsV14(db);migrateModelsV15(db);migrateModelsV16Vision(db);+ migrateModelsV17IntelligenceTiers(db);ensureUnifiedKey(db);console.log(`Database initialized at ${resolvedPath}`);@@ -1364,6 +1365,110 @@ function migrateModelsV16Vision(db: Database.Database) {apply();}+// ── V17: intelligence tier audit (2026-06) ──+// `size_label` is the cross-provider capability tier that DOMINATES the router's+// intelligence axis (intelligenceComposite = tier*1000 - intelligence_rank in+// services/router.ts), so it must track real benchmarks rather than the+// release-day guesses several rows were seeded with. This normalizes every model+// family that has a published Artificial Analysis Intelligence Index v4.0 score+// (served/default mode, June 2026) to the correct tier — and as a side effect+// fixes cross-provider inconsistencies where the same model family had landed in+// different tiers per provider (e.g. Llama 4 Scout, Llama 3.3 70B).+//+// Tier bands by AA Index v4.0: Frontier ≥45 · Large 26–44 · Medium 13–25 · Small ≤12.+//+// Rules are keyed by model_id LIKE patterns so one rule covers a family across+// all providers. Models with NO published AA Index are intentionally left at+// their seeded tier (Cogito 2.1, Owl Alpha, Poolside Laguna, Hermes 3 405B,+// Baidu CoBuddy, Groq Compound, GLM-4.6V Flash, GLM-4.5 Flash, Mistral Small 4,+// Devstral). intelligence_rank (the within-tier tiebreak, low impact) is left+// untouched. Idempotent — every statement is an absolute SET, safe to re-run.+function migrateModelsV17IntelligenceTiers(db: Database.Database) {+ const apply = db.transaction(() => {+ // Frontier (AA ≥ 45): genuine frontier-class. Promotes Gemini 3.5 Flash (55)+ // and Gemini 3 Flash Preview (46) up from Large.+ db.prepare(`+ UPDATE models SET size_label = 'Frontier' WHERE+ LOWER(model_id) LIKE '%gemini-3.1-pro%'+ OR LOWER(model_id) LIKE '%gemini-3.5-flash%'+ OR LOWER(model_id) LIKE '%gemini-3-flash%'+ OR LOWER(model_id) LIKE '%kimi-k2.6%'+ OR LOWER(model_id) LIKE '%kimi-k2-thinking%'+ OR LOWER(model_id) LIKE '%deepseek-v4-pro%'+ OR LOWER(model_id) LIKE '%deepseek-v4-flash%'+ OR LOWER(model_id) LIKE '%glm-5.1%'+ OR LOWER(model_id) LIKE '%minimax-m2.7%'+ `).run();++ // Large (AA 26–44). Demotes Gemini 2.5 Pro (35), Nemotron 3 Super/120B (36),+ // GLM-4.7 (42), DeepSeek V3.1/V3.2 (28/32), Trinity (32) down from Frontier;+ // promotes Gemma 4 31B (39) / 26B (31) and Gemini 3.1 Flash-Lite (34) up.+ db.prepare(`+ UPDATE models SET size_label = 'Large' WHERE+ LOWER(model_id) LIKE '%minimax-m2.5%'+ OR LOWER(model_id) LIKE '%qwen3-next%'+ OR LOWER(model_id) LIKE '%qwen3-coder-next%'+ OR LOWER(model_id) LIKE '%gpt-oss-120b%' OR LOWER(model_id) LIKE '%gpt-oss:120b%'+ OR LOWER(model_id) LIKE '%glm-4.7%'+ OR LOWER(model_id) LIKE '%nemotron-3-super%' OR LOWER(model_id) LIKE '%nemotron-3-120b%'+ OR LOWER(model_id) LIKE '%gemini-2.5-pro%'+ OR LOWER(model_id) LIKE '%deepseek-v3.2%'+ OR LOWER(model_id) LIKE '%deepseek-v3.1%'+ OR LOWER(model_id) LIKE '%trinity-large%'+ OR LOWER(model_id) LIKE '%mistral-medium%'+ OR LOWER(model_id) LIKE '%magistral-medium%'+ OR LOWER(model_id) LIKE '%gpt-4.1%'+ OR LOWER(model_id) LIKE '%gemma-4-31b%' OR LOWER(model_id) LIKE '%gemma4:31b%'+ OR LOWER(model_id) LIKE '%gemma-4-26b%'+ OR LOWER(model_id) LIKE '%gemini-3.1-flash-lite%'+ `).run();++ // Medium (AA 13–25). Demotes Qwen3-Coder 480B (25) and Mistral Large 3 (23)+ // down from Frontier; Llama 4 Maverick (18), GPT-4o (17), Gemini 2.5 Flash+ // (21), GLM-4.5 Air (23), DeepSeek R1 Distill (17), Command A/R+ down from+ // Large; unifies Llama 4 Scout (14) and Llama 3.3 70B (14) across providers.+ db.prepare(`+ UPDATE models SET size_label = 'Medium' WHERE+ (LOWER(model_id) LIKE '%qwen3-coder%' AND LOWER(model_id) NOT LIKE '%qwen3-coder-next%')+ OR LOWER(model_id) LIKE '%qwen-3-235b%' OR LOWER(model_id) LIKE '%qwen3-235b%'+ OR LOWER(model_id) LIKE '%mistral-large%'+ OR LOWER(model_id) LIKE '%gpt-oss-20b%' OR LOWER(model_id) LIKE '%gpt-oss:20b%'+ OR LOWER(model_id) LIKE '%gpt-oss-safeguard-20b%' OR model_id = 'openai-fast'+ OR LOWER(model_id) LIKE '%glm-4.5-air%'+ OR LOWER(model_id) LIKE '%devstral-2%'+ OR LOWER(model_id) LIKE '%deepseek-r1-distill%'+ OR LOWER(model_id) LIKE '%qwen3-30b%'+ OR LOWER(model_id) LIKE '%qwen3-32b%'+ OR LOWER(model_id) LIKE '%llama-4-maverick%'+ OR LOWER(model_id) LIKE '%llama-4-scout%'+ OR LOWER(model_id) LIKE '%llama-3.3-70b%'+ OR LOWER(model_id) LIKE '%llama-3.1-70b%'+ OR (LOWER(model_id) LIKE '%gemini-2.5-flash%' AND LOWER(model_id) NOT LIKE '%flash-lite%')+ OR LOWER(model_id) LIKE '%gemini-2.5-flash-lite%'+ OR LOWER(model_id) LIKE '%gpt-4o%'+ OR LOWER(model_id) LIKE '%command-a-03-2025%'+ OR LOWER(model_id) LIKE '%command-r-plus%'+ OR LOWER(model_id) LIKE '%nemotron-3-nano%'+ OR LOWER(model_id) LIKE '%nemotron-nano-9b%'+ `).run();++ // Small (AA ≤ 12). Demotes Gemma 3 12B (9), Command R 08-2024 (legacy ~7),+ // and Codestral (8) down from Medium.+ db.prepare(`+ UPDATE models SET size_label = 'Small' WHERE+ LOWER(model_id) LIKE '%gemma-3-12b%'+ OR LOWER(model_id) LIKE '%command-r-08-2024%'+ OR LOWER(model_id) LIKE '%codestral%'+ OR LOWER(model_id) LIKE '%llama-3.1-8b%' OR LOWER(model_id) LIKE '%llama3.1-8b%'+ OR LOWER(model_id) LIKE '%meta-llama-3.1-8b%'+ OR LOWER(model_id) LIKE '%ministral-8b%'+ OR LOWER(model_id) LIKE '%granite-4.0-h-micro%'+ OR LOWER(model_id) LIKE '%lfm-2.5-1.2b%'+ `).run();+ });+ apply();+}+function ensureUnifiedKey(db: Database.Database) {const existing = db.prepare("SELECT value FROM settings WHERE key = 'unified_api_key'").get() as { value: string } | undefined;if (!existing) {