Skip to content

7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.

TypeScript

chore(catalog): intelligence tier audit vs Artificial Analysis Index v4.0 (2026-06)

Tashfeend948cb7393ef
diff --git a/server/src/__tests__/db/intelligence-tiers.test.ts b/server/src/__tests__/db/intelligence-tiers.test.ts
new file mode 100644
index 00000000..75574080
--- /dev/null
+++ b/server/src/__tests__/db/intelligence-tiers.test.ts
@@ -0,0 +1,85 @@
+import { describe, it, expect, beforeAll } from 'vitest';
+import { initDb, getDb } from '../../db/index.js';
+
+// Verifies the V17 intelligence-tier audit (2026-06): size_label is normalized
+// to Artificial Analysis Intelligence Index v4.0 bands, and the same model
+// family lands in ONE tier regardless of provider.
+describe('intelligence tier audit (migrateModelsV17)', () => {
+ beforeAll(() => {
+ process.env.ENCRYPTION_KEY = '0'.repeat(64);
+ initDb(':memory:');
+ });
+
+ function tier(platform: string, modelId: string): string | undefined {
+ const row = getDb()
+ .prepare('SELECT size_label FROM models WHERE platform = ? AND model_id = ?')
+ .get(platform, modelId) as { size_label: string } | undefined;
+ return row?.size_label;
+ }
+
+ function tiersForFamily(like: string): string[] {
+ const rows = getDb()
+ .prepare("SELECT DISTINCT size_label FROM models WHERE LOWER(model_id) LIKE ?")
+ .all(like.toLowerCase()) as { size_label: string }[];
+ return rows.map(r => r.size_label);
+ }
+
+ it('promotes frontier-class Gemini Flash models (AA 46–55) to Frontier', () => {
+ expect(tier('google', 'gemini-3.5-flash')).toBe('Frontier');
+ expect(tier('google', 'gemini-3-flash-preview')).toBe('Frontier');
+ });
+
+ it('demotes over-tiered models down from Frontier', () => {
+ // Qwen3-Coder 480B (AA 25, non-reasoning coder) → Medium
+ expect(tier('nvidia', 'qwen/qwen3-coder-480b-a35b-instruct')).toBe('Medium');
+ expect(tier('openrouter', 'qwen/qwen3-coder:free')).toBe('Medium');
+ // Mistral Large 3 (AA 23) → Medium
+ expect(tier('ollama', 'mistral-large-3:675b')).toBe('Medium');
+ // Gemini 2.5 Pro (AA 35) → Large
+ expect(tier('google', 'gemini-2.5-pro')).toBe('Large');
+ // Nemotron 3 Super 120B (AA 36) → Large
+ expect(tier('nvidia', 'nvidia/nemotron-3-super-120b-a12b')).toBe('Large');
+ });
+
+ it('demotes lapped models down from Large', () => {
+ expect(tier('google', 'gemini-2.5-flash')).toBe('Medium'); // AA 21
+ expect(tier('github', 'gpt-4o')).toBe('Medium'); // AA 17
+ expect(tier('cohere', 'command-a-03-2025')).toBe('Medium'); // AA 13
+ expect(tier('ollama', 'devstral-2:123b')).toBe('Medium'); // AA 22
+ });
+
+ it('promotes under-tiered Gemma 4 and Gemini Flash-Lite up to Large', () => {
+ expect(tier('nvidia', 'google/gemma-4-31b-it')).toBe('Large'); // AA 39
+ expect(tier('google', 'gemini-3.1-flash-lite-preview')).toBe('Large'); // AA 34
+ });
+
+ it('demotes weak models to Small', () => {
+ expect(tier('sambanova', 'gemma-3-12b-it')).toBe('Small'); // AA 9
+ expect(tier('mistral', 'codestral-latest')).toBe('Small'); // AA 8
+ expect(tier('cohere', 'command-r-08-2024')).toBe('Small'); // legacy
+ });
+
+ it('keeps the qwen3-coder vs qwen3-coder-next split correct', () => {
+ expect(tier('ollama', 'qwen3-coder:480b')).toBe('Medium');
+ expect(tier('ollama', 'qwen3-coder-next')).toBe('Large');
+ });
+
+ it('assigns a single consistent tier per model family across providers', () => {
+ // The bug the audit also fixes: same model, different tier per provider.
+ expect(tiersForFamily('%llama-4-scout%')).toEqual(['Medium']);
+ expect(tiersForFamily('%llama-3.3-70b%')).toEqual(['Medium']);
+ expect(tiersForFamily('%gpt-oss-120b%')).toEqual(['Large']);
+ });
+
+ it('leaves models with no published AA Index at their seeded tier', () => {
+ expect(tier('ollama', 'cogito-2.1:671b')).toBe('Frontier');
+ expect(tier('openrouter', 'nousresearch/hermes-3-llama-3.1-405b:free')).toBe('Large');
+ expect(tier('openrouter', 'poolside/laguna-m.1:free')).toBe('Large');
+ });
+
+ it('keeps genuine frontier models at Frontier', () => {
+ expect(tier('google', 'gemini-3.1-pro-preview')).toBe('Frontier');
+ expect(tier('cloudflare', '@cf/moonshotai/kimi-k2.6')).toBe('Frontier');
+ expect(tier('nvidia', 'z-ai/glm-5.1')).toBe('Frontier');
+ });
+});
diff --git a/server/src/db/index.ts b/server/src/db/index.ts
index 3129873f..b4f17029 100644
--- a/server/src/db/index.ts
+++ b/server/src/db/index.ts
@@ -51,6 +51,7 @@ export function initDb(dbPath?: string): Database.Database {
migrateModelsV14(db);
migrateModelsV15(db);
migrateModelsV16Vision(db);
+ migrateModelsV17IntelligenceTiers(db);
ensureUnifiedKey(db);
console.log(`Database initialized at ${resolvedPath}`);
@@ -1364,6 +1365,110 @@ function migrateModelsV16Vision(db: Database.Database) {
apply();
}
+// ── V17: intelligence tier audit (2026-06) ──
+// `size_label` is the cross-provider capability tier that DOMINATES the router's
+// intelligence axis (intelligenceComposite = tier*1000 - intelligence_rank in
+// services/router.ts), so it must track real benchmarks rather than the
+// release-day guesses several rows were seeded with. This normalizes every model
+// family that has a published Artificial Analysis Intelligence Index v4.0 score
+// (served/default mode, June 2026) to the correct tier — and as a side effect
+// fixes cross-provider inconsistencies where the same model family had landed in
+// different tiers per provider (e.g. Llama 4 Scout, Llama 3.3 70B).
+//
+// Tier bands by AA Index v4.0: Frontier ≥45 · Large 26–44 · Medium 13–25 · Small ≤12.
+//
+// Rules are keyed by model_id LIKE patterns so one rule covers a family across
+// all providers. Models with NO published AA Index are intentionally left at
+// their seeded tier (Cogito 2.1, Owl Alpha, Poolside Laguna, Hermes 3 405B,
+// Baidu CoBuddy, Groq Compound, GLM-4.6V Flash, GLM-4.5 Flash, Mistral Small 4,
+// Devstral). intelligence_rank (the within-tier tiebreak, low impact) is left
+// untouched. Idempotent — every statement is an absolute SET, safe to re-run.
+function migrateModelsV17IntelligenceTiers(db: Database.Database) {
+ const apply = db.transaction(() => {
+ // Frontier (AA ≥ 45): genuine frontier-class. Promotes Gemini 3.5 Flash (55)
+ // and Gemini 3 Flash Preview (46) up from Large.
+ db.prepare(`
+ UPDATE models SET size_label = 'Frontier' WHERE
+ LOWER(model_id) LIKE '%gemini-3.1-pro%'
+ OR LOWER(model_id) LIKE '%gemini-3.5-flash%'
+ OR LOWER(model_id) LIKE '%gemini-3-flash%'
+ OR LOWER(model_id) LIKE '%kimi-k2.6%'
+ OR LOWER(model_id) LIKE '%kimi-k2-thinking%'
+ OR LOWER(model_id) LIKE '%deepseek-v4-pro%'
+ OR LOWER(model_id) LIKE '%deepseek-v4-flash%'
+ OR LOWER(model_id) LIKE '%glm-5.1%'
+ OR LOWER(model_id) LIKE '%minimax-m2.7%'
+ `).run();
+
+ // Large (AA 26–44). Demotes Gemini 2.5 Pro (35), Nemotron 3 Super/120B (36),
+ // GLM-4.7 (42), DeepSeek V3.1/V3.2 (28/32), Trinity (32) down from Frontier;
+ // promotes Gemma 4 31B (39) / 26B (31) and Gemini 3.1 Flash-Lite (34) up.
+ db.prepare(`
+ UPDATE models SET size_label = 'Large' WHERE
+ LOWER(model_id) LIKE '%minimax-m2.5%'
+ OR LOWER(model_id) LIKE '%qwen3-next%'
+ OR LOWER(model_id) LIKE '%qwen3-coder-next%'
+ OR LOWER(model_id) LIKE '%gpt-oss-120b%' OR LOWER(model_id) LIKE '%gpt-oss:120b%'
+ OR LOWER(model_id) LIKE '%glm-4.7%'
+ OR LOWER(model_id) LIKE '%nemotron-3-super%' OR LOWER(model_id) LIKE '%nemotron-3-120b%'
+ OR LOWER(model_id) LIKE '%gemini-2.5-pro%'
+ OR LOWER(model_id) LIKE '%deepseek-v3.2%'
+ OR LOWER(model_id) LIKE '%deepseek-v3.1%'
+ OR LOWER(model_id) LIKE '%trinity-large%'
+ OR LOWER(model_id) LIKE '%mistral-medium%'
+ OR LOWER(model_id) LIKE '%magistral-medium%'
+ OR LOWER(model_id) LIKE '%gpt-4.1%'
+ OR LOWER(model_id) LIKE '%gemma-4-31b%' OR LOWER(model_id) LIKE '%gemma4:31b%'
+ OR LOWER(model_id) LIKE '%gemma-4-26b%'
+ OR LOWER(model_id) LIKE '%gemini-3.1-flash-lite%'
+ `).run();
+
+ // Medium (AA 13–25). Demotes Qwen3-Coder 480B (25) and Mistral Large 3 (23)
+ // down from Frontier; Llama 4 Maverick (18), GPT-4o (17), Gemini 2.5 Flash
+ // (21), GLM-4.5 Air (23), DeepSeek R1 Distill (17), Command A/R+ down from
+ // Large; unifies Llama 4 Scout (14) and Llama 3.3 70B (14) across providers.
+ db.prepare(`
+ UPDATE models SET size_label = 'Medium' WHERE
+ (LOWER(model_id) LIKE '%qwen3-coder%' AND LOWER(model_id) NOT LIKE '%qwen3-coder-next%')
+ OR LOWER(model_id) LIKE '%qwen-3-235b%' OR LOWER(model_id) LIKE '%qwen3-235b%'
+ OR LOWER(model_id) LIKE '%mistral-large%'
+ OR LOWER(model_id) LIKE '%gpt-oss-20b%' OR LOWER(model_id) LIKE '%gpt-oss:20b%'
+ OR LOWER(model_id) LIKE '%gpt-oss-safeguard-20b%' OR model_id = 'openai-fast'
+ OR LOWER(model_id) LIKE '%glm-4.5-air%'
+ OR LOWER(model_id) LIKE '%devstral-2%'
+ OR LOWER(model_id) LIKE '%deepseek-r1-distill%'
+ OR LOWER(model_id) LIKE '%qwen3-30b%'
+ OR LOWER(model_id) LIKE '%qwen3-32b%'
+ OR LOWER(model_id) LIKE '%llama-4-maverick%'
+ OR LOWER(model_id) LIKE '%llama-4-scout%'
+ OR LOWER(model_id) LIKE '%llama-3.3-70b%'
+ OR LOWER(model_id) LIKE '%llama-3.1-70b%'
+ OR (LOWER(model_id) LIKE '%gemini-2.5-flash%' AND LOWER(model_id) NOT LIKE '%flash-lite%')
+ OR LOWER(model_id) LIKE '%gemini-2.5-flash-lite%'
+ OR LOWER(model_id) LIKE '%gpt-4o%'
+ OR LOWER(model_id) LIKE '%command-a-03-2025%'
+ OR LOWER(model_id) LIKE '%command-r-plus%'
+ OR LOWER(model_id) LIKE '%nemotron-3-nano%'
+ OR LOWER(model_id) LIKE '%nemotron-nano-9b%'
+ `).run();
+
+ // Small (AA ≤ 12). Demotes Gemma 3 12B (9), Command R 08-2024 (legacy ~7),
+ // and Codestral (8) down from Medium.
+ db.prepare(`
+ UPDATE models SET size_label = 'Small' WHERE
+ LOWER(model_id) LIKE '%gemma-3-12b%'
+ OR LOWER(model_id) LIKE '%command-r-08-2024%'
+ OR LOWER(model_id) LIKE '%codestral%'
+ OR LOWER(model_id) LIKE '%llama-3.1-8b%' OR LOWER(model_id) LIKE '%llama3.1-8b%'
+ OR LOWER(model_id) LIKE '%meta-llama-3.1-8b%'
+ OR LOWER(model_id) LIKE '%ministral-8b%'
+ OR LOWER(model_id) LIKE '%granite-4.0-h-micro%'
+ OR LOWER(model_id) LIKE '%lfm-2.5-1.2b%'
+ `).run();
+ });
+ apply();
+}
+
function ensureUnifiedKey(db: Database.Database) {
const existing = db.prepare("SELECT value FROM settings WHERE key = 'unified_api_key'").get() as { value: string } | undefined;
if (!existing) {