Never stop coding. Free MIT AI gateway: one endpoint, 375 providers (150+ free), 1,400+ models Kimi, Claude, GPT, Gemini, GLM, DeepSeek, MiniMax. Works with Claude Code, Codex, Cursor, OpenCode, Cline & Copilot. Quota-aware auto-fallback, RTK+Caveman compression saves 15-95% tokens, MCP/A2A, Desktop/PWA. Built by 900+ contributors
TypeScript a2aai-agentsai-gatewayanthropicclaudeclaude-codeclinecodexcopilotcursordeepseekfree-aigeminikimillm-gatewaymcpopenaiopenai-proxyqwentoken-saver
/**
* Vision Bridge Auto-Router
* Automatically selects the fastest vision-capable model from available models.
*/
import { getResolvedModelCapabilities } from "@/lib/modelCapabilities";
import { getActiveSyncedCatalog } from "@/lib/db/models/activeSyncedCatalog";
import { PROVIDER_MODELS } from "@omniroute/open-sse/config/providerModels";
import { getRegisteredProviderEffortBaseModelId } from "@omniroute/open-sse/utils/registeredEffortVariants.ts";
import {
hasUsableCredentialsForModel,
getUsableConnectionsForModel,
} from "./visionBridgeCredentials";
import { isVisionBridgeForcedModel } from "@/shared/constants/visionBridgeDefaults";
import { resolveProviderId } from "@/shared/constants/providers";
import {
isModelLocked,
getAllModelLockouts,
} from "@omniroute/open-sse/services/accountFallback.ts";
export interface VisionModelCandidate {
modelId: string;
fullName: string; // provider/model format
priority: number; // lower = better (local models first)
averageLatencyMs: number;
lastUsedAt: number;
successRate: number;
}
export interface LatencyRecord {
modelId: string;
latencyMs: number;
timestamp: number;
success: boolean;
}
export interface VisionBridgeRouterConfig {
/** Fixed model to use (overrides auto-routing) */
fixedModel?: string;
/** Maximum number of fallback attempts */
maxFallbackAttempts: number;
/** Cache TTL for selection decisions (ms) */
selectionCacheTtlMs: number;
/**
* Cache TTL for a "no usable candidate" outcome (ms). `0` disables the
* negative cache and restores the pre-#14161 behaviour of rescanning the
* whole catalog on every call.
*/
noCandidateCacheTtlMs: number;
/** Minimum number of latency samples before trusting average */
minLatencySamples: number;
/** Models to exclude from auto-routing */
excludedModels: string[];
/**
* Partition of the selection caches for an API-key-scoped pick (the key's
* id + policy). A scoped pick must never be served from, or leak into, the
* unscoped cache, and its cached member is re-validated against the scoped
* `hasUsableCredentials` before reuse.
*/
cacheScope?: string;
}
/**
* `OMNIROUTE_VISION_BRIDGE_NEGATIVE_CACHE_MS` overrides how long a
* "no usable candidate" outcome is remembered. Invalid or negative values
* fall back to the default; `0` disables the negative cache entirely.
*/
function readNoCandidateCacheTtlMs(): number {
const raw = process.env.OMNIROUTE_VISION_BRIDGE_NEGATIVE_CACHE_MS;
if (raw == null || raw.trim() === "") return 30_000;
const parsed = Number(raw);
if (!Number.isFinite(parsed) || parsed < 0) return 30_000;
return Math.floor(parsed);
}
const DEFAULT_ROUTER_CONFIG: VisionBridgeRouterConfig = {
maxFallbackAttempts: 3,
selectionCacheTtlMs: 60_000, // 1 minute
noCandidateCacheTtlMs: readNoCandidateCacheTtlMs(),
minLatencySamples: 5,
excludedModels: [],
};
// In-memory latency tracker (would be Redis in production)
const latencyStore = new Map<string, LatencyRecord[]>();
const selectionCache = new Map<string, { modelId: string; expiresAt: number }>();
/**
* Keys that recently resolved to "no vision-capable model has usable
* credentials". `getVisionCapableModels` walks every provider in
* PROVIDER_MODELS and issues a credential lookup per candidate, so without
* this an install with images and no usable describer repeats that scan on
* the request path for every image of every request.
*/
const noCandidateCache = new Map<string, number>();
/**
* Record a latency measurement for a model.
*/
export function recordLatency(modelId: string, latencyMs: number, success: boolean): void {
const records = latencyStore.get(modelId) || [];
records.push({
modelId,
latencyMs,
timestamp: Date.now(),
success,
});
// Keep only last 100 records per model
if (records.length > 100) {
records.splice(0, records.length - 100);
}
latencyStore.set(modelId, records);
}
/**
* Calculate average latency for a model, considering only recent records.
*/
function calculateAverageLatency(modelId: string, windowMs: number = 300_000): number {
const records = latencyStore.get(modelId) || [];
const cutoff = Date.now() - windowMs;
const recentRecords = records.filter((r) => r.timestamp > cutoff && r.success);
if (recentRecords.length === 0) {
return Infinity; // No data = assume slow
}
const sum = recentRecords.reduce((acc, r) => acc + r.latencyMs, 0);
return sum / recentRecords.length;
}
/**
* Calculate success rate for a model.
*/
function calculateSuccessRate(modelId: string): number {
const records = latencyStore.get(modelId) || [];
if (records.length === 0) return 1.0; // No data = assume good
const recentRecords = records.slice(-50); // Last 50 attempts
const successes = recentRecords.filter((r) => r.success).length;
return successes / recentRecords.length;
}
/**
* Injectable dependencies for the router's credential-usability check.
* Defaults to the real `hasUsableCredentialsForModel` (DB-backed). Tests can
* inject a pure stub here instead of mocking the `@/lib/db/providers` module
* boundary — this project's Node native test runner (`node:test`) has no
* supported ESM module-mocking mechanism, so DI is the only way to exercise
* the credential-exclusion branch under `npm run test:unit`.
*/
export interface VisionBridgeRouterDeps {
hasUsableCredentials?: (model: string) => Promise<boolean | null>;
getActiveSyncedCatalog?: (provider: string) => Promise<VisionModelCatalog>;
/**
* (#12111) Per-connection model-lockout check, defaulting to the real
* `accountFallback.isModelLocked`. Injectable for the same reason as
* `hasUsableCredentials`: `node:test` has no supported ESM module-mocking.
*/
isModelLocked?: (provider: string, connectionId: string, model: string) => boolean;
}
export interface VisionModelCatalog {
authoritative: boolean;
models: Array<{ id: string }>;
}
type CatalogAwareRegistryModel = {
id: string;
liveCatalogIds?: readonly string[];
};
async function readActiveCatalog(
providerAlias: string,
deps: VisionBridgeRouterDeps
): Promise<VisionModelCatalog> {
const readCatalog = deps.getActiveSyncedCatalog ?? getActiveSyncedCatalog;
try {
return await readCatalog(providerAlias);
} catch {
return { authoritative: false, models: [] };
}
}
function createCatalogModelPredicate(
providerAlias: string,
catalog: VisionModelCatalog
): (model: CatalogAwareRegistryModel) => boolean {
if (!catalog.authoritative) return () => true;
const liveIds = new Set(catalog.models.map((entry) => entry.id));
return (model) => {
if (liveIds.has(model.id) || model.liveCatalogIds?.some((id) => liveIds.has(id))) {
return true;
}
const effortBaseModelId = getRegisteredProviderEffortBaseModelId(providerAlias, model.id);
return effortBaseModelId !== null && liveIds.has(effortBaseModelId);
};
}
/**
* connectionIds worth probing for a `(providerAlias, modelId)` lockout check:
* the provider's DB-known usable connections, plus any connectionId that
* already has an active lockout entry for this provider (#12111) — a 404
* lock (`accountFallback.lockModel`) can target a connectionId the DB-backed
* lookup does not surface (e.g. it predates a reconnect, or the credential
* check path a caller injected does not go through the same DB rows), and
* missing it would silently fail the exclusion open.
*/
function collectLockoutConnectionIds(providerAlias: string): string[] {
const canonicalProvider = resolveProviderId(providerAlias);
return getAllModelLockouts()
.filter((entry) => entry.provider === canonicalProvider)
.map((entry) => entry.connectionId);
}
/**
* (#12111) True unless `modelId` is locked (a post-404 model lockout, see
* `accountFallback.lockModel`) on every connection that could actually serve
* it. `isModelLocked` is scoped per provider+connection+model, so a single
* locked connection must not exclude a model that's still reachable through
* another connection on the same provider — mirrors
* `isConnectionEligibleForModel` in
* open-sse/services/autoCombo/resilienceCandidateFilter.ts. Fails open (never
* excludes) when nothing is known about the provider's connections, matching
* `hasUsableCredentialsForModel`'s existing fail-open contract — this check
* only narrows an already-credentialed candidate, it never widens the pool.
*/
async function isModelUsableGivenLockouts(
providerAlias: string,
modelId: string,
deps: VisionBridgeRouterDeps
): Promise<boolean> {
const checkLocked = deps.isModelLocked ?? isModelLocked;
const dbConnections = await getUsableConnectionsForModel(`${providerAlias}/${modelId}`);
if (dbConnections === null) return true; // indeterminate credential store — fail open
const candidateIds = new Set(dbConnections.map((conn) => conn.id));
for (const id of collectLockoutConnectionIds(providerAlias)) candidateIds.add(id);
if (candidateIds.size === 0) return true; // nothing known about this provider's connections
for (const id of candidateIds) {
if (!checkLocked(providerAlias, id, modelId)) return true;
}
return false;
}
async function cachedModelRemainsAvailable(
fullModelId: string,
deps: VisionBridgeRouterDeps
): Promise<boolean> {
const separator = fullModelId.indexOf("/");
if (separator < 1) return false;
const providerAlias = fullModelId.slice(0, separator);
const modelId = fullModelId.slice(separator + 1);
const registryModel = PROVIDER_MODELS[providerAlias]?.find((model) => model.id === modelId);
if (!registryModel) return false;
if (!(await isModelUsableGivenLockouts(providerAlias, modelId, deps))) return false;
const catalog = await readActiveCatalog(providerAlias, deps);
return createCatalogModelPredicate(providerAlias, catalog)(registryModel);
}
/**
* Get all vision-capable models from the registry that also have a usable
* active connection on this instance.
*
* Without this credential check, a model with no working connection (e.g. the
* hardcoded default `openai/gpt-4o-mini` on an instance with no `openai`
* provider connected) could win selection, fail the describe call, and leave
* the guardrail's describe-failure fallback to forward the raw image to a
* non-vision backend, which rejects it with an opaque upstream error.
*/
async function getVisionCapableModels(
deps: VisionBridgeRouterDeps = {}
): Promise<VisionModelCandidate[]> {
const checkCreds = deps.hasUsableCredentials ?? hasUsableCredentialsForModel;
const candidatesByProvider = await Promise.all(
Object.entries(PROVIDER_MODELS).map(async ([providerAlias, models]) => {
if (!Array.isArray(models)) return [];
const visionModels = models.filter((model) => {
if (!model?.id) return false;
const fullModelId = `${providerAlias}/${model.id}`;
return (
getResolvedModelCapabilities(fullModelId).supportsVision === true &&
!isVisionBridgeForcedModel(fullModelId)
);
});
if (visionModels.length === 0) return [];
const credentialedModels = (
await Promise.all(
visionModels.map(async (model) =>
(await checkCreds(`${providerAlias}/${model.id}`)) === false ? null : model
)
)
).filter((model): model is (typeof visionModels)[number] => model !== null);
if (credentialedModels.length === 0) return [];
// (#12111) A healthy provider connection does not mean every model on
// it is servable: chatCore.ts locks one specific model for 120s on a
// 404 while leaving the connection active, so the credential check
// above never sees it. Drop only the models locked on every usable
// connection for this provider.
const usableModels = (
await Promise.all(
credentialedModels.map(async (model) =>
(await isModelUsableGivenLockouts(providerAlias, model.id, deps)) ? model : null
)
)
).filter((model): model is (typeof credentialedModels)[number] => model !== null);
if (usableModels.length === 0) return [];
const catalog = await readActiveCatalog(providerAlias, deps);
const modelExistsInCatalog = createCatalogModelPredicate(providerAlias, catalog);
const candidates = usableModels.map((model): VisionModelCandidate | null => {
if (!modelExistsInCatalog(model)) return null;
const fullModelId = `${providerAlias}/${model.id}`;
let priority = 100;
if (providerAlias === "openai" || providerAlias === "anthropic") {
priority = 50;
} else if (providerAlias === "vertex" || providerAlias === "gemini") {
priority = 55;
} else if (providerAlias.startsWith("opencode-")) {
priority = 95;
} else {
priority = 75;
}
return {
modelId: model.id,
fullName: fullModelId,
priority,
averageLatencyMs: calculateAverageLatency(fullModelId),
lastUsedAt: 0,
successRate: calculateSuccessRate(fullModelId),
};
});
return candidates.filter(
(candidate): candidate is VisionModelCandidate => candidate !== null
);
})
);
return candidatesByProvider.flat();
}
/**
* Select the best vision model based on latency, priority, and success rate.
*/
function selectBestModel(
candidates: VisionModelCandidate[],
config: VisionBridgeRouterConfig
): VisionModelCandidate | null {
const filtered = candidates.filter((c) => {
// Exclude explicitly excluded models
if (config.excludedModels.includes(c.fullName)) return false;
if (config.excludedModels.includes(c.modelId)) return false;
// Exclude models with poor success rate (< 50%)
if (c.successRate < 0.5) return false;
return true;
});
if (filtered.length === 0) return null;
// Score each candidate: lower is better
// Score = priority * 1000 + averageLatencyMs
// This prioritizes local models, then fastest latency
const scored = filtered.map((c) => ({
...c,
score: c.priority * 1000 + (c.averageLatencyMs === Infinity ? 10000 : c.averageLatencyMs),
}));
scored.sort((a, b) => a.score - b.score);
return scored[0];
}
/**
* (#12237) `auto` / `auto/*` ids are VIRTUAL combos: there is no provider
* row for "auto", so the credential check always reports `false` for them.
* Member-level credentials are enforced downstream when the combo
* dispatches (mirrors the reroute guard in visionBridge.ts), so a virtual
* combo must not be discarded by the #8430 short-circuit — otherwise the
* combo silently falls through to auto-selection and never rotates. It is
* still subject to the pool check in `getBestVisionModel`: when the ENTIRE
* vision pool is unusable there is nothing the combo could dispatch to, and
* returning the combo id would let a raw image reach a text-only backend
* (#8430).
*
* Returns the combo id when `fixedModel` is virtual, `undefined` otherwise.
*/
function resolveVirtualCombo(fixedModel: string | undefined): string | undefined {
return fixedModel === "auto" || fixedModel?.startsWith("auto/") ? fixedModel : undefined;
}
/**
* Resolve a live selection-cache entry for `cacheKey`.
*
* Returns the id to hand back: the cached member for a concrete target, or
* `virtualCombo` once the cached member proves it still has usable
* credentials (the cache never re-validates credentials, and the caller
* exempts virtual combos from that check). A missing or expired entry yields
* `null`; an entry whose member is no longer available or usable is dropped
* so the pool is rescanned.
*/
async function resolveCachedSelection(
cacheKey: string,
virtualCombo: string | undefined,
deps: VisionBridgeRouterDeps,
scoped = false
): Promise<string | null> {
const cached = selectionCache.get(cacheKey);
if (!cached || cached.expiresAt <= Date.now()) return null;
if (await cachedModelRemainsAvailable(cached.modelId, deps)) {
const checkCreds = deps.hasUsableCredentials ?? hasUsableCredentialsForModel;
if (!virtualCombo && !scoped) return cached.modelId;
if ((await checkCreds(cached.modelId)) !== false) return virtualCombo ?? cached.modelId;
}
selectionCache.delete(cacheKey);
return null;
}
/**
* Get the best vision model for image description.
* Respects fixed model override if configured, but validates it has usable
* credentials before short-circuiting — a fixedModel that is confirmed
* unreachable on this instance falls through to auto-selection.
* Returns `null` when no vision-capable candidate has usable credentials.
*/
export async function getBestVisionModel(
config: Partial<VisionBridgeRouterConfig> = {},
deps: VisionBridgeRouterDeps = {}
): Promise<string | null> {
const fullConfig = { ...DEFAULT_ROUTER_CONFIG, ...config };
const virtualCombo = resolveVirtualCombo(fullConfig.fixedModel);
// If fixed model is configured, validate it has usable credentials first.
// (#8430) An unreachable fixedModel (e.g. the default "openai/gpt-4o-mini"
// on an instance with no OpenAI connection/key) must not short-circuit the
// credential check — fall through to auto-selection instead.
// (#12237) A virtual combo is exempt here and goes through the pool
// selection below instead; see `resolveVirtualCombo`.
if (fullConfig.fixedModel && !virtualCombo) {
const checkCreds = deps.hasUsableCredentials ?? hasUsableCredentialsForModel;
const usable = await checkCreds(fullConfig.fixedModel);
// Only skip credential validation when the check is indeterminate (null).
// A confirmed `false` means fall through to auto-selection.
if (usable !== false) {
return fullConfig.fixedModel;
}
}
// Check selection cache — key includes excluded models to prevent cache pollution
// across different configurations
const baseCacheKey =
fullConfig.excludedModels.length > 0
? `excl:${[...fullConfig.excludedModels].sort().join(",")}`
: "default";
const cacheKey = fullConfig.cacheScope
? `scope:${fullConfig.cacheScope}|${baseCacheKey}`
: baseCacheKey;
const cachedPick = await resolveCachedSelection(
cacheKey,
virtualCombo,
deps,
Boolean(fullConfig.cacheScope)
);
if (cachedPick) return cachedPick;
// A recent scan already proved nothing here is usable. Re-running it costs a
// full PROVIDER_MODELS walk plus a credential lookup per candidate, all on
// the request path, and it cannot succeed until credentials change.
const noCandidateUntil = noCandidateCache.get(cacheKey);
if (noCandidateUntil !== undefined) {
if (noCandidateUntil > Date.now()) return null;
noCandidateCache.delete(cacheKey);
}
// Get all vision-capable candidates
const candidates = await getVisionCapableModels(deps);
// Select best model
const best = selectBestModel(candidates, fullConfig);
if (!best) {
// No vision-capable candidate has usable credentials on this instance
if (fullConfig.noCandidateCacheTtlMs > 0) {
noCandidateCache.set(cacheKey, Date.now() + fullConfig.noCandidateCacheTtlMs);
}
return null;
}
// A usable candidate appeared, so any remembered failure is stale.
noCandidateCache.delete(cacheKey);
// Cache the selection
selectionCache.set(cacheKey, {
modelId: best.fullName,
expiresAt: Date.now() + fullConfig.selectionCacheTtlMs,
});
// A virtual combo is returned as-is once the pool proves at least one
// vision-capable member is usable; it rotates its own members downstream.
return virtualCombo ?? best.fullName;
}
/**
* Get fallback models for retry logic.
*/
export async function getFallbackModels(
excludeModel: string,
config: Partial<VisionBridgeRouterConfig> = {},
deps: VisionBridgeRouterDeps = {}
): Promise<string[]> {
const fullConfig = { ...DEFAULT_ROUTER_CONFIG, ...config };
const candidates = await getVisionCapableModels(deps);
const filtered = candidates.filter(
(c) =>
c.fullName !== excludeModel &&
!fullConfig.excludedModels.includes(c.fullName) &&
c.successRate >= 0.5
);
// Sort by score
const scored = filtered.map((c) => ({
...c,
score: c.priority * 1000 + (c.averageLatencyMs === Infinity ? 10000 : c.averageLatencyMs),
}));
scored.sort((a, b) => a.score - b.score);
return scored.slice(0, fullConfig.maxFallbackAttempts - 1).map((c) => c.fullName);
}
/**
* Clear the selection cache (e.g., after config change).
*/
export function clearSelectionCache(): void {
selectionCache.clear();
noCandidateCache.clear();
}
/**
* Get latency statistics for debugging.
*/
export function getLatencyStats(): Record<
string,
{ avg: number; samples: number; successRate: number }
> {
const stats: Record<string, { avg: number; samples: number; successRate: number }> = {};
for (const [modelId, records] of latencyStore.entries()) {
const recentRecords = records.filter((r) => r.timestamp > Date.now() - 300_000);
if (recentRecords.length === 0) continue;
const avg = recentRecords.reduce((acc, r) => acc + r.latencyMs, 0) / recentRecords.length;
const successRate = recentRecords.filter((r) => r.success).length / recentRecords.length;
stats[modelId] = {
avg: Math.round(avg),
samples: recentRecords.length,
successRate: Math.round(successRate * 100) / 100,
};
}
return stats;
}