7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.
TypeScript
fix(responses): align the /v1/responses shim with the official Responses API spec (#844)
熬夜砍树358710ac9485
diff --git a/server/src/__tests__/services/router-bandit.test.ts b/server/src/__tests__/services/router-bandit.test.tsindex 37823735..6ec300b3 100644--- a/server/src/__tests__/services/router-bandit.test.ts+++ b/server/src/__tests__/services/router-bandit.test.ts@@ -30,12 +30,13 @@ const ORIGINAL_NODE_ENV = process.env.NODE_ENV;function addModel(opts: {platform: string; modelId: string; name: string;intelligenceRank: number; sizeLabel: string; budget: string; priority: number;+ vision?: boolean;}): number {const db = getDb();db.prepare(`- INSERT INTO models (platform, model_id, display_name, intelligence_rank, speed_rank, size_label, monthly_token_budget, enabled)- VALUES (?, ?, ?, ?, ?, ?, ?, 1)- `).run(opts.platform, opts.modelId, opts.name, opts.intelligenceRank, 1, opts.sizeLabel, opts.budget);+ INSERT INTO models (platform, model_id, display_name, intelligence_rank, speed_rank, size_label, monthly_token_budget, enabled, supports_vision, supports_tools)+ VALUES (?, ?, ?, ?, ?, ?, ?, 1, ?, 1)+ `).run(opts.platform, opts.modelId, opts.name, opts.intelligenceRank, 1, opts.sizeLabel, opts.budget, opts.vision ? 1 : 0);const id = (db.prepare('SELECT id FROM models WHERE platform = ? AND model_id = ?').get(opts.platform, opts.modelId) as { id: number }).id;db.prepare('INSERT INTO fallback_config (model_db_id, priority, enabled) VALUES (?, ?, 1)').run(id, opts.priority);@@ -130,6 +131,21 @@ describe('bandit router', () => {expect(counts['y'] ?? 0).toBeGreaterThan(0);});+ it('vision requests never route to a text-only model, even with exploration on', () => {+ setExploreEnabled(true);+ addModel({ platform: 'google', modelId: 'vision-a', name: 'Vision A', intelligenceRank: 3, sizeLabel: 'Large', budget: '~50M', priority: 1, vision: true });+ addModel({ platform: 'groq', modelId: 'text-b', name: 'Text B', intelligenceRank: 3, sizeLabel: 'Large', budget: '~50M', priority: 2 });+ setRoutingStrategy('balanced');+ refreshStatsCache(getDb(), true);+ const counts: Record<string, number> = {};+ for (let i = 0; i < 300; i++) {+ const r = routeRequest(100, undefined, undefined, /*requireVision=*/ true);+ counts[r.modelId] = (counts[r.modelId] ?? 0) + 1;+ }+ expect(counts['text-b'] ?? 0).toBe(0);+ expect(counts['vision-a'] ?? 0).toBe(300);+ });+it('smartest vs fastest flips which model wins, at equal reliability', () => {// Smart: frontier tier, slow. Fast: small tier, high throughput. Equal success.addModel({ platform: 'google', modelId: 'smart', name: 'Smart', intelligenceRank: 1, sizeLabel: 'Frontier', budget: '~50M', priority: 1 });diff --git a/server/src/services/router.ts b/server/src/services/router.tsindex 52aaa2c6..d90e4d47 100644--- a/server/src/services/router.ts+++ b/server/src/services/router.ts@@ -1557,6 +1557,12 @@ export function routeRequest(estimatedTokens = 1000, skipKeys?: Set<string>, pre// probability, pick one unmeasured model uniformly and try it first; if it// fails, the loop falls through to the scored order as usual. Only for// bandit strategies — Manual is the operator's explicit order.+ // Candidates the main loop would immediately reject for THIS request are+ // excluded up front (ruled-out models plus the request-level capability+ // gates: vision/tools/structured output/context window): promoting a model+ // that can't serve the request ahead of capable ones would just get it+ // skipped a moment later (e.g. an image request must never randomly probe a+ // text-only model).if (strategy !== 'priority' && getExploreEnabled() && Math.random() < EXPLORE_CHANCE) {// A model the operator zeroed out via MODEL_ROUTING_OVERRIDES never wins a// bandit draw, so it would stay under EXPLORE_MIN_SAMPLES forever and become@@ -1565,7 +1571,16 @@ export function routeRequest(estimatedTokens = 1000, skipKeys?: Set<string>, preconst unmeasured = sortedChain.filter(e => {if (overrides.get(e.model_id) === 0) return false;const stats = statsCache?.get(modelStatsKey(e.platform, e.model_id, e.endpoint_scope));- return (stats?.successes ?? 0) + (stats?.failures ?? 0) < EXPLORE_MIN_SAMPLES;+ if ((stats?.successes ?? 0) + (stats?.failures ?? 0) >= EXPLORE_MIN_SAMPLES) return false;+ // Mirror the main loop's gates below so exploration only samples+ // candidates that can actually serve this request.+ if (skipModels?.has(e.model_db_id)) return false;+ if (requireVision && !e.supports_vision) return false;+ if (requireTools && !e.supports_tools) return false;+ if (requireStructured && platformDropsResponseFormat(e.platform)) return false;+ if (e.context_window != null && estimatedTokens > e.context_window) return false;+ if (e.tpm_limit != null && estimatedTokens > e.tpm_limit) return false;+ return true;});if (unmeasured.length > 0) {const probe = unmeasured[Math.floor(Math.random() * unmeasured.length)];