Skip to content

7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.

TypeScript

fix(responses): align the /v1/responses shim with the official Responses API spec (#844)

熬夜砍树358710ac9485
diff --git a/server/src/__tests__/services/router-bandit.test.ts b/server/src/__tests__/services/router-bandit.test.ts
index 37823735..6ec300b3 100644
--- a/server/src/__tests__/services/router-bandit.test.ts
+++ b/server/src/__tests__/services/router-bandit.test.ts
@@ -30,12 +30,13 @@ const ORIGINAL_NODE_ENV = process.env.NODE_ENV;
function addModel(opts: {
platform: string; modelId: string; name: string;
intelligenceRank: number; sizeLabel: string; budget: string; priority: number;
+ vision?: boolean;
}): number {
const db = getDb();
db.prepare(`
- INSERT INTO models (platform, model_id, display_name, intelligence_rank, speed_rank, size_label, monthly_token_budget, enabled)
- VALUES (?, ?, ?, ?, ?, ?, ?, 1)
- `).run(opts.platform, opts.modelId, opts.name, opts.intelligenceRank, 1, opts.sizeLabel, opts.budget);
+ INSERT INTO models (platform, model_id, display_name, intelligence_rank, speed_rank, size_label, monthly_token_budget, enabled, supports_vision, supports_tools)
+ VALUES (?, ?, ?, ?, ?, ?, ?, 1, ?, 1)
+ `).run(opts.platform, opts.modelId, opts.name, opts.intelligenceRank, 1, opts.sizeLabel, opts.budget, opts.vision ? 1 : 0);
const id = (db.prepare('SELECT id FROM models WHERE platform = ? AND model_id = ?')
.get(opts.platform, opts.modelId) as { id: number }).id;
db.prepare('INSERT INTO fallback_config (model_db_id, priority, enabled) VALUES (?, ?, 1)').run(id, opts.priority);
@@ -130,6 +131,21 @@ describe('bandit router', () => {
expect(counts['y'] ?? 0).toBeGreaterThan(0);
});
+ it('vision requests never route to a text-only model, even with exploration on', () => {
+ setExploreEnabled(true);
+ addModel({ platform: 'google', modelId: 'vision-a', name: 'Vision A', intelligenceRank: 3, sizeLabel: 'Large', budget: '~50M', priority: 1, vision: true });
+ addModel({ platform: 'groq', modelId: 'text-b', name: 'Text B', intelligenceRank: 3, sizeLabel: 'Large', budget: '~50M', priority: 2 });
+ setRoutingStrategy('balanced');
+ refreshStatsCache(getDb(), true);
+ const counts: Record<string, number> = {};
+ for (let i = 0; i < 300; i++) {
+ const r = routeRequest(100, undefined, undefined, /*requireVision=*/ true);
+ counts[r.modelId] = (counts[r.modelId] ?? 0) + 1;
+ }
+ expect(counts['text-b'] ?? 0).toBe(0);
+ expect(counts['vision-a'] ?? 0).toBe(300);
+ });
+
it('smartest vs fastest flips which model wins, at equal reliability', () => {
// Smart: frontier tier, slow. Fast: small tier, high throughput. Equal success.
addModel({ platform: 'google', modelId: 'smart', name: 'Smart', intelligenceRank: 1, sizeLabel: 'Frontier', budget: '~50M', priority: 1 });
diff --git a/server/src/services/router.ts b/server/src/services/router.ts
index 52aaa2c6..d90e4d47 100644
--- a/server/src/services/router.ts
+++ b/server/src/services/router.ts
@@ -1557,6 +1557,12 @@ export function routeRequest(estimatedTokens = 1000, skipKeys?: Set<string>, pre
// probability, pick one unmeasured model uniformly and try it first; if it
// fails, the loop falls through to the scored order as usual. Only for
// bandit strategies — Manual is the operator's explicit order.
+ // Candidates the main loop would immediately reject for THIS request are
+ // excluded up front (ruled-out models plus the request-level capability
+ // gates: vision/tools/structured output/context window): promoting a model
+ // that can't serve the request ahead of capable ones would just get it
+ // skipped a moment later (e.g. an image request must never randomly probe a
+ // text-only model).
if (strategy !== 'priority' && getExploreEnabled() && Math.random() < EXPLORE_CHANCE) {
// A model the operator zeroed out via MODEL_ROUTING_OVERRIDES never wins a
// bandit draw, so it would stay under EXPLORE_MIN_SAMPLES forever and become
@@ -1565,7 +1571,16 @@ export function routeRequest(estimatedTokens = 1000, skipKeys?: Set<string>, pre
const unmeasured = sortedChain.filter(e => {
if (overrides.get(e.model_id) === 0) return false;
const stats = statsCache?.get(modelStatsKey(e.platform, e.model_id, e.endpoint_scope));
- return (stats?.successes ?? 0) + (stats?.failures ?? 0) < EXPLORE_MIN_SAMPLES;
+ if ((stats?.successes ?? 0) + (stats?.failures ?? 0) >= EXPLORE_MIN_SAMPLES) return false;
+ // Mirror the main loop's gates below so exploration only samples
+ // candidates that can actually serve this request.
+ if (skipModels?.has(e.model_db_id)) return false;
+ if (requireVision && !e.supports_vision) return false;
+ if (requireTools && !e.supports_tools) return false;
+ if (requireStructured && platformDropsResponseFormat(e.platform)) return false;
+ if (e.context_window != null && estimatedTokens > e.context_window) return false;
+ if (e.tpm_limit != null && estimatedTokens > e.tpm_limit) return false;
+ return true;
});
if (unmeasured.length > 0) {
const probe = unmeasured[Math.floor(Math.random() * unmeasured.length)];