Skip to content

7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.

TypeScript

fix(fallback): normalize intelligence sort across providers (#141)

Tashfeen55bb7e269522
diff --git a/server/src/__tests__/routes/fallback.test.ts b/server/src/__tests__/routes/fallback.test.ts
index 43d02b0c..66ba468f 100644
--- a/server/src/__tests__/routes/fallback.test.ts
+++ b/server/src/__tests__/routes/fallback.test.ts
@@ -76,14 +76,39 @@ describe('Fallback API', () => {
await request(app, 'PUT', '/api/fallback', restore);
});
- it('POST /api/fallback/sort/intelligence sorts by intelligence', async () => {
+ it('POST /api/fallback/sort/intelligence sorts by cross-provider tier, then rank', async () => {
const { status } = await request(app, 'POST', '/api/fallback/sort/intelligence');
expect(status).toBe(200);
const { body } = await request(app, 'GET', '/api/fallback');
- // Should be sorted ascending by intelligence rank
+
+ // intelligence_rank is per-provider, so the sort normalizes on the
+ // cross-provider capability tier (size_label) first (issue #135).
+ const tier: Record<string, number> = { Frontier: 1, Large: 2, Medium: 3, Small: 4 };
+ const tierOf = (label: string) => tier[label] ?? 5;
+
for (let i = 1; i < body.length; i++) {
- expect(body[i].intelligenceRank).toBeGreaterThanOrEqual(body[i - 1].intelligenceRank);
+ const prevTier = tierOf(body[i - 1].sizeLabel);
+ const curTier = tierOf(body[i].sizeLabel);
+ // Capability tier never decreases...
+ expect(curTier).toBeGreaterThanOrEqual(prevTier);
+ // ...and within the same tier, per-provider rank breaks the tie.
+ if (curTier === prevTier) {
+ expect(body[i].intelligenceRank).toBeGreaterThanOrEqual(body[i - 1].intelligenceRank);
+ }
+ }
+ });
+
+ it('intelligence sort never places a weaker tier above a Frontier model (#135)', async () => {
+ await request(app, 'POST', '/api/fallback/sort/intelligence');
+ const { body } = await request(app, 'GET', '/api/fallback');
+
+ // The last Frontier model must come before the first non-Frontier model —
+ // i.e. no "Intel #1 from a weaker provider" leaks above the frontier tier.
+ const lastFrontier = body.map((m: any) => m.sizeLabel).lastIndexOf('Frontier');
+ const firstNonFrontier = body.findIndex((m: any) => m.sizeLabel !== 'Frontier');
+ if (lastFrontier !== -1 && firstNonFrontier !== -1) {
+ expect(lastFrontier).toBeLessThan(firstNonFrontier);
}
});
diff --git a/server/src/routes/fallback.ts b/server/src/routes/fallback.ts
index ebf8b3d3..7c4e6597 100644
--- a/server/src/routes/fallback.ts
+++ b/server/src/routes/fallback.ts
@@ -83,10 +83,18 @@ fallbackRouter.put('/', (req: Request, res: Response) => {
res.json({ success: true });
});
+// `intelligence_rank` is scoped to each provider's own catalog — a provider's
+// #1 model is not globally #1 (see issue #135: MiniMax's top model outranking
+// Gemini Pro because both read "Intel #1"). `size_label` IS a cross-provider
+// capability tier, so normalize on it first and use intelligence_rank only as
+// an in-tier tiebreaker. Unknown labels sort last.
+const INTELLIGENCE_TIER =
+ "CASE m.size_label WHEN 'Frontier' THEN 1 WHEN 'Large' THEN 2 WHEN 'Medium' THEN 3 WHEN 'Small' THEN 4 ELSE 5 END";
+
// Sort presets — `orderBy` is selected from a fixed whitelist, never from
// user input directly, so the interpolation below is safe.
const SORT_PRESETS: Record<string, string> = {
- intelligence: 'm.intelligence_rank ASC',
+ intelligence: `${INTELLIGENCE_TIER} ASC, m.intelligence_rank ASC`,
speed: 'm.speed_rank ASC',
budget: "CASE m.monthly_token_budget WHEN '~120M' THEN 1 WHEN '~50-100M' THEN 2 WHEN '~30M' THEN 3 WHEN '~18-45M' THEN 4 WHEN '~18M' THEN 5 WHEN '~15M' THEN 6 WHEN '~12M' THEN 7 WHEN '~6M' THEN 8 WHEN '~5-10M' THEN 9 WHEN '~4M' THEN 10 ELSE 11 END ASC",
};