7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.
TypeScript
fix(fusion): size-aware panel selection so a too-small model can't claim a slot (#572)
Chong Jia Zhenb93d98a1b542
diff --git a/server/src/__tests__/routes/fusion.test.ts b/server/src/__tests__/routes/fusion.test.tsindex a7d2395e..35dc8840 100644--- a/server/src/__tests__/routes/fusion.test.ts+++ b/server/src/__tests__/routes/fusion.test.ts@@ -395,16 +395,16 @@ describe('fusion route (/v1/chat/completions, model: "fusion")', () => {try {// Priority mode: ordering is the manual chain order, and stable.setRoutingStrategy('priority');- const p1 = getOrderedFusionChain().map(c => c.modelId);- const p2 = getOrderedFusionChain().map(c => c.modelId);+ const p1 = getOrderedFusionChain(1).map(c => c.modelId);+ const p2 = getOrderedFusionChain(1).map(c => c.modelId);expect(p1.length).toBeGreaterThan(0);expect(p2).toEqual(p1);// Bandit mode: previously Thompson-sampled (random per call); now the// deterministic expected-score ranking, so two calls must be identical.setRoutingStrategy('smartest');- const s1 = getOrderedFusionChain().map(c => c.modelId);- const s2 = getOrderedFusionChain().map(c => c.modelId);+ const s1 = getOrderedFusionChain(1).map(c => c.modelId);+ const s2 = getOrderedFusionChain(1).map(c => c.modelId);expect(s2).toEqual(s1);// The strategy actually drives the ordering: 'smartest' (intelligence)@@ -422,7 +422,7 @@ describe('fusion route (/v1/chat/completions, model: "fusion")', () => {const r = await request(app, 'POST', '/api/keys', { platform: 'groq', key: 'k_groq_only', label: 'only-groq' });expect(r.status).toBe(201);- const candidates = getOrderedFusionChain();+ const candidates = getOrderedFusionChain(1);expect(candidates.length).toBeGreaterThan(0);// Even though the seeded catalog has many higher-ranked models on other// platforms (cerebras, openrouter, opencode…), none are routable without a@@ -436,16 +436,49 @@ describe('fusion route (/v1/chat/completions, model: "fusion")', () => {await request(app, 'POST', '/api/keys', { platform: 'groq', key: 'k_groq_cool', label: 'cool' });const keyId = (db.prepare("SELECT id FROM api_keys WHERE platform = 'groq'").get() as { id: number }).id;- const before = getOrderedFusionChain().filter(c => c.platform === 'groq');+ const before = getOrderedFusionChain(1).filter(c => c.platform === 'groq');expect(before.length).toBeGreaterThan(0);const target = before[0]; // the top groq model under the strategy// Bench that exact model+key (as a 402/429 cooldown would), then re-check.setCooldown('groq', target.modelId, keyId, 60_000);- const after = getOrderedFusionChain();+ const after = getOrderedFusionChain(1);expect(after.find(c => c.modelId === target.modelId)).toBeUndefined();});+ it('auto-panel excludes a model whose context window cannot hold the prompt', async () => {+ const db = getDb();+ db.prepare('DELETE FROM api_keys').run();+ await request(app, 'POST', '/api/keys', { platform: 'groq', key: 'k_groq_ctx', label: 'ctx' });++ const target = getOrderedFusionChain(1).find(c => c.platform === 'groq');+ expect(target).toBeDefined();+ db.prepare('UPDATE models SET context_window = 1000 WHERE platform = ? AND model_id = ?')+ .run('groq', target!.modelId);++ // Fits: a 500-token prompt is well inside the 1000-token window.+ expect(getOrderedFusionChain(500).find(c => c.modelId === target!.modelId)).toBeDefined();+ // Does not fit: the model can NEVER serve this request, so it must not claim+ // a panel slot. Before this gate it was selected anyway and failed at+ // dispatch, reported as the misleading "no available key for model".+ expect(getOrderedFusionChain(5000).find(c => c.modelId === target!.modelId)).toBeUndefined();+ });++ it('auto-panel treats a null context_window as unknown, not as zero', async () => {+ const db = getDb();+ db.prepare('DELETE FROM api_keys').run();+ await request(app, 'POST', '/api/keys', { platform: 'groq', key: 'k_groq_null', label: 'nullctx' });++ const target = getOrderedFusionChain(1).find(c => c.platform === 'groq');+ expect(target).toBeDefined();+ db.prepare('UPDATE models SET context_window = NULL WHERE platform = ? AND model_id = ?')+ .run('groq', target!.modelId);++ // An unspecified window must never itself exclude a model, at any size:+ // same convention the auto-router uses.+ expect(getOrderedFusionChain(5_000_000).find(c => c.modelId === target!.modelId)).toBeDefined();+ });+it('auto-panel refills failed slots from the fallback chain', async () => {// groq is entirely rate-limited; cerebras + openrouter answer. The two groq// panel slots fail, and the panel refills from the next chain models.diff --git a/server/src/services/fusion.ts b/server/src/services/fusion.tsindex d543baf8..c5869bab 100644--- a/server/src/services/fusion.ts+++ b/server/src/services/fusion.ts@@ -429,7 +429,7 @@ export function diversifyChain(ordered: FusionCandidate[]): FusionCandidate[] {* both the panel and its refills span genuinely different perspectives before* doubling up on either axis.*/-function selectPanel(config: FusionConfig, requirements: { requireTools?: boolean } = {}): { panel: FusionCandidate[]; overflow: FusionCandidate[]; dropped: string[] } {+function selectPanel(config: FusionConfig, requirements: { requireTools?: boolean; estimatedTokens: number }): { panel: FusionCandidate[]; overflow: FusionCandidate[]; dropped: string[] } {const maxK = panelMaxK();if (config.models && config.models.length > 0) {@@ -450,7 +450,10 @@ function selectPanel(config: FusionConfig, requirements: { requireTools?: boolea}const k = Math.min(Math.max(config.k ?? panelDefaultK(), 1), maxK);- const ordered = getOrderedFusionChain().filter(c => !requirements.requireTools || c.supportsTools);+ // Size-aware: the chain excludes models that cannot hold a prompt this large,+ // so a too-small model never claims a slot it is guaranteed to fail.+ const ordered = getOrderedFusionChain(requirements.estimatedTokens)+ .filter(c => !requirements.requireTools || c.supportsTools);// Diversity-first ordering of the whole servable chain along provider AND// model family (see diversifyChain). The first K are the panel; the rest are@@ -541,7 +544,7 @@ export async function runFusion(params: {const strategy = config.strategy ?? 'synthesize';const requireTools = (options.tools?.length ?? 0) > 0;- const { panel, overflow, dropped } = selectPanel(config, { requireTools });+ const { panel, overflow, dropped } = selectPanel(config, { requireTools, estimatedTokens });if (panel.length === 0) {throw new FusionError('fusion: no usable models for the panel. Provide `fusion.models` with enabled model ids, or enable models in the Fallback Chain.',diff --git a/server/src/services/router.ts b/server/src/services/router.tsindex efaa2ddc..208f319e 100644--- a/server/src/services/router.ts+++ b/server/src/services/router.ts@@ -1008,7 +1008,7 @@ export interface FusionCandidate {* so the panel's auto-pick draws from the highest-scored models first and the* fusion layer just needs to apply provider-diversity on top.*/-export function getOrderedFusionChain(): FusionCandidate[] {+export function getOrderedFusionChain(estimatedTokens: number): FusionCandidate[] {const db = getDb();const strategy = getRoutingStrategy();if (strategy !== 'priority') refreshStatsCache(db);@@ -1023,6 +1023,13 @@ export function getOrderedFusionChain(): FusionCandidate[] {// currently cooled down (huggingface/Kimi-K2.6) would claim a panel slot it// can't fill — surfacing as "no available key" and pushing out a usable model,// which also makes the panel look like it's ignoring the routing strategy.+ //+ // The SIZE gates matter as much as the key gates: a model whose context window+ // cannot hold the prompt can NEVER fill its slot, yet diversifyChain keeps+ // handing it one on every request when it is its platform's only representative.+ // That leaves one panel slot dead on arrival and reports the failure as the+ // misleading "no available key for model". Passing a placeholder token count+ // here made both size gates no-ops.const usableKeys = db.prepare("SELECT id, platform FROM api_keys WHERE enabled = 1 AND status IN ('healthy', 'unknown')").all() as { id: number; platform: string }[];@@ -1032,6 +1039,9 @@ export function getOrderedFusionChain(): FusionCandidate[] {if (arr) arr.push(k.id); else keysByPlatform.set(k.platform, [k.id]);}const servable = chain.filter(e => {+ // A null context_window means "unknown", not "zero": same convention the+ // auto-router uses, so an unspecified window is never itself a reason to skip.+ if (e.context_window != null && estimatedTokens > e.context_window) return false;const keyIds = keysByPlatform.get(e.platform);if (!keyIds) return false;const limits = { rpm: e.rpm_limit, rpd: e.rpd_limit, tpm: e.tpm_limit, tpd: e.tpd_limit };@@ -1041,7 +1051,7 @@ export function getOrderedFusionChain(): FusionCandidate[] {canUseProvider(e.platform, kid) &&canUseProviderMinute(e.platform, kid) &&canMakeRequest(e.platform, e.model_id, kid, limits) &&- canUseProviderTokens(e.platform, kid, e.model_id, 1),+ canUseProviderTokens(e.platform, kid, e.model_id, estimatedTokens),);});