Never stop coding. Free MIT AI gateway: one endpoint, 375 providers (150+ free), 1,400+ models Kimi, Claude, GPT, Gemini, GLM, DeepSeek, MiniMax. Works with Claude Code, Codex, Cursor, OpenCode, Cline & Copilot. Quota-aware auto-fallback, RTK+Caveman compression saves 15-95% tokens, MCP/A2A, Desktop/PWA. Built by 900+ contributors
TypeScript a2aai-agentsai-gatewayanthropicclaudeclaude-codeclinecodexcopilotcursordeepseekfree-aigeminikimillm-gatewaymcpopenaiopenai-proxyqwentoken-saver
/**
* Vision Bridge helper functions for image processing.
*/
import { detectMediaParts, type MediaPart } from "@omniroute/open-sse/utils/mediaParts";
import { normalizeDataUri } from "@omniroute/open-sse/utils/imageNormalize";
import { fetchRemoteImage } from "@/shared/network/remoteImageFetch";
import { getRuntimePorts } from "@/lib/runtime/ports";
import { resolveSelfLoopBearer } from "@/shared/middleware/chatBodyAdmission";
import { getBestVisionModel, getFallbackModels, recordLatency } from "./visionBridgeRouter";
import { combineCredentialChecks } from "./visionBridgeKeyScope";
import { REGISTRY } from "@omniroute/open-sse/config/providers";
import { fetch as undiciFetch } from "undici";
/**
* Provider to environment variable mapping for API key resolution.
*/
const PROVIDER_API_KEY_MAP: Record<string, string> = {
anthropic: "ANTHROPIC_API_KEY",
google: "GOOGLE_API_KEY",
openai: "OPENAI_API_KEY",
};
// Providers whose wire format is Anthropic Messages ("claude"). Anthropic
// accepts `source: { type: "url" }` for images, but most claude-format backends
// (MiniMax, Z.AI, …) do NOT — they reject remote URLs (MiniMax: 403 code
// 2013). The vision bridge must deliver images as base64 for these targets,
// both in the describe self-loop and in the rerouted payload.
const CLAUDE_WIRE_PROVIDERS = new Set<string>(
Object.entries(REGISTRY)
.filter(([, entry]) => entry.format === "claude")
.map(([id]) => id.toLowerCase())
);
/**
* True when `provider/model` targets a Claude-Messages wire format backend
* that cannot ingest remote image URLs and needs base64 instead.
*/
export function isClaudeWireFormatModel(model: string | null | undefined): boolean {
if (!model || typeof model !== "string") return false;
const provider = model.includes("/") ? model.split("/")[0].trim().toLowerCase() : "";
return CLAUDE_WIRE_PROVIDERS.has(provider);
}
/**
* Resolve API key based on model provider (issue #2232).
*
* Priority:
* 1. `explicitKey` argument (caller override)
* 2. `VISION_BRIDGE_API_KEY` env var — operator-set, takes precedence over
* per-provider env vars. Used when the operator wants every vision-bridge
* call to go through a single OpenAI-compatible endpoint (e.g.,
* OmniRoute itself, OpenRouter, a Gemini-OpenAI-compat URL).
* 3. Per-provider env var (`ANTHROPIC_API_KEY`, `GOOGLE_API_KEY`,
* `OPENAI_API_KEY`) based on the `provider/` prefix in the model id.
* 4. `OPENAI_API_KEY` as final fallback when the prefix is unrecognized.
*
* @param model - Model identifier (e.g., "anthropic/claude-3-haiku", "openai/gpt-4o-mini")
* @param explicitKey - Explicit API key passed as argument (takes precedence)
* @returns Resolved API key string
*/
export function resolveProviderApiKey(model: string, explicitKey?: string): string {
if (explicitKey) return explicitKey;
const isAnthropic = model.startsWith("anthropic/");
// VISION_BRIDGE_API_KEY only applies to the OpenAI-compatible branch — the
// Anthropic branch keeps its dedicated key, since the wire format differs.
if (!isAnthropic) {
const bridgeKey = (process.env.VISION_BRIDGE_API_KEY || "").trim();
if (bridgeKey) return bridgeKey;
}
const provider = model.includes("/") ? model.split("/")[0] : "";
const envVar = PROVIDER_API_KEY_MAP[provider] || "OPENAI_API_KEY";
return process.env[envVar] || "";
}
/**
* Resolve the OpenAI-compatible base URL for non-Anthropic vision bridge calls
* (issue #2232).
*
* Priority:
* 1. `VISION_BRIDGE_BASE_URL` env var — operator-set, e.g. point this at
* OmniRoute's own `/v1` so the vision model can be any provider
* registered in OmniRoute (`google/gemini-2.0-flash`,
* `openrouter/...`, etc.) instead of being limited to OpenAI/Anthropic.
* 2. `OPENAI_API_URL` env var (legacy)
* 3. OmniRoute self-loop (`http://localhost:20128/v1`) — auto-detected when
* the model uses a known OmniRoute-internal provider (e.g. `kr/`, `if/`,
* `pol/`, `groq/`, etc.) instead of a direct OpenAI/Anthropic endpoint.
* 4. `https://api.openai.com/v1` (fallback when the model is `openai/*` or
* unprefixed — works only when the operator actually has an OpenAI
* account and OPENAI_API_KEY set)
*
* @param model - Optional model identifier used to detect non-standard providers
* that require OmniRoute self-loop routing.
*/
export function resolveVisionBridgeBaseUrl(model?: string): string {
const explicit = (process.env.VISION_BRIDGE_BASE_URL || "").trim();
if (explicit) return explicit.replace(/\/+$/, "");
const legacy = (process.env.OPENAI_API_URL || "").trim();
if (legacy) return legacy.replace(/\/+$/, "");
// When the model has a non-standard provider prefix (not openai/ or
// anthropic/), it can only be resolved through OmniRoute's own router,
// not through a direct OpenAI/Anthropic endpoint. Use the operator-configured
// port via OMNIROUTE_PORT / PORT env vars, falling back to the default 20128.
if (model && model.includes("/")) {
const provider = model.split("/")[0].toLowerCase();
if (provider !== "openai" && provider !== "anthropic") {
const { port } = getRuntimePorts();
return `http://localhost:${port}/v1`;
}
}
return "https://api.openai.com/v1";
}
/** True when `baseUrl` is this OmniRoute instance's own loopback listener. */
function isOwnOmniRouteBaseUrl(baseUrl: string): boolean {
let url: URL;
try {
url = new URL(baseUrl);
} catch {
return false;
}
if (!["localhost", "127.0.0.1", "[::1]"].includes(url.hostname)) return false;
const urlPort = url.port || (url.protocol === "https:" ? "443" : "80");
const { port, apiPort, dashboardPort } = getRuntimePorts();
return [port, apiPort, dashboardPort].some((listenPort) => String(listenPort) === urlPort);
}
export interface ImagePart {
messageIndex: number;
partIndex: number;
imageUrl: string;
imageType: "image_url" | "image" | "url";
/**
* For nested hits (image inside a container part, e.g. a tool_result's
* content array) the path from `message.content[partIndex]` to the image
* object itself — what replaceImageParts walks to splice it. Absent for
* top-level parts (plain partIndex splice).
*/
path?: (string | number)[];
}
export interface RequestMessage {
role?: string;
content?: string | RequestContentPart[];
}
export type RequestContentPart =
| { type: "text"; text: string }
| { type: "input_text"; text: string }
| { type: "image_url"; image_url: { url: string; detail?: string } }
| { type: "input_image"; image_url: string; detail?: string }
| {
type: "image";
source: { type: "base64"; media_type: string; data: string } | { type: "url"; url: string };
};
/**
* Extract image parts from messages array.
* Supports OpenAI image_url format, base64 image format, and Anthropic-style
* image source blocks with either `source.type: "base64"` or `source.type: "url"`.
*
* The URL-source branch mirrors the executor-level handling in
* `open-sse/executors/commandCode.ts::extractImageUrl` — without it, a
* Claude-Code-compatible client (e.g. Zoo Code) sending
* `{ type: "image", source: { type: "url", url } }` was invisible to the
* vision-bridge guardrail, so the image was silently dropped by a text-only
* executor instead of being described.
*/
/**
* Shapes `replaceImageParts` knows how to splice: content parts whose `type`
* is `image_url`, `image`, or `input_image` — at top level or nested (inside
* a container part such as a tool_result's content array; nested hits are
* spliced by walking `MediaPart.path`). Everything else the detector reports
* (`data_uri_string`, `image_indicator`) is combo-filter material only —
* extracting it would desync the positional description consumption in
* visionBridge (descriptions would shift onto the wrong images).
*/
const REPLACEABLE_IMAGE_SHAPES: ReadonlySet<MediaPart["shape"]> = new Set([
"image_url",
"image_base64",
"image_source_url",
"input_image",
]);
export function extractImageParts(messages: RequestMessage[]): ImagePart[] {
// Delegates to the unified detector (open-sse/utils/mediaParts.ts) so the
// guardrail and the combo compatibility filter share one source of truth.
// Extraction is shaped-allowlisted and covers BOTH top-level parts and
// nested hits (image inside a container part, e.g. Claude Code's tool_result
// content array) whose shape replaceImageParts can splice back — the
// extract↔replace contract: every extracted part MUST be replaceable, in the
// same order, or the positional descriptions shift onto the wrong images.
// Nested hits carry `path` so the splice can walk the container; top-level
// hits rely on messageIndex/partIndex alone.
return detectMediaParts(messages)
.filter((p) => p.kind === "image" && REPLACEABLE_IMAGE_SHAPES.has(p.shape))
.map((p) => ({
messageIndex: p.messageIndex,
partIndex: p.partIndex,
imageUrl: p.ref,
imageType:
p.shape === "image_base64" ? "image" : p.shape === "image_source_url" ? "url" : "image_url",
...(p.nested ? { path: p.path } : {}),
}));
}
// Undici fetch with a browser-ish User-Agent: Wikimedia (and other CDNs)
// reject requests without a UA with HTTP 400, silently breaking remote image
// downloads in the describe path.
const VISION_BRIDGE_UA_FETCH: typeof fetch = ((input: RequestInfo | URL, init?: RequestInit) =>
undiciFetch(input as string | URL, {
...(init as Parameters<typeof undiciFetch>[1]),
headers: {
"user-agent": "omniroute-vision-bridge",
...((init?.headers as Record<string, string> | undefined) ?? {}),
},
})) as unknown as typeof fetch;
/**
* Resolve every image part in the body to a base64 data URI when the target
* model speaks the Claude wire format (remote URLs unsupported by most
* claude-format backends, e.g. MiniMax 403 2013). Fail-open: an image that
* cannot be fetched is left untouched.
*/
export async function ensureBase64ImagesForClaudeWire(
body: RequestBody,
model: string,
fetchImpl: typeof fetch = VISION_BRIDGE_UA_FETCH
): Promise<RequestBody> {
if (!isClaudeWireFormatModel(model)) return body;
// The splice below re-walks top-level content parts and swaps
// image_url/image fields by sequential index. Nested hits now carry
// `path` (extractImageParts emits them); this loop only handles top-level
// image_url/image parts, so skip nested hits to keep the index map aligned
// (a nested hit interleaved with top-level hits would desync the map).
const parts = extractImageParts(body.messages as RequestMessage[]).filter((p) => !p.path);
if (parts.length === 0) return body;
const resolved = await Promise.all(
parts.map(async (part) => {
const normalized = resolveImageAsDataUri(part.imageUrl);
if (normalized.startsWith("data:")) return null; // already base64
try {
return await fetchRemoteImageAsDataUri(normalized, new AbortController().signal, fetchImpl);
} catch {
return null; // fail-open: keep the original part
}
})
);
// Map sequential image index → resolved data URI (null = keep original).
const byIndex = new Map<number, string>();
parts.forEach((part, i) => {
if (resolved[i]) byIndex.set(i, resolved[i] as string);
});
if (byIndex.size === 0) return body;
const result = structuredClone(body) as RequestBody;
let imageIndex = 0;
for (const message of result.messages ?? []) {
if (!message || !Array.isArray(message.content)) continue;
for (const part of message.content as RequestContentPart[]) {
if (part.type !== "image_url" && part.type !== "image") continue;
const dataUri = byIndex.get(imageIndex);
imageIndex++;
if (dataUri) {
(part as { image_url?: { url: string } }).image_url = { url: dataUri };
}
}
}
return result;
}
/**
* Resolve image URL to data URI format for vision model.
* - HTTP/HTTPS URLs: passed through as-is
* - Data URIs: passed through as-is
* - Base64 without media type: assumed PNG
*/
export function resolveImageAsDataUri(imageUrl: string): string {
if (!imageUrl || typeof imageUrl !== "string") {
throw new Error("Invalid image URL: must be a non-empty string");
}
// Already a data URI
if (imageUrl.startsWith("data:")) {
return imageUrl;
}
// HTTP/HTTPS URL - vision API will fetch it
if (imageUrl.startsWith("http://") || imageUrl.startsWith("https://")) {
return imageUrl;
}
// Assume it's a base64 string without prefix
// Add PNG as default media type
return `data:image/png;base64,${imageUrl}`;
}
async function fetchRemoteImageAsDataUri(
imageUrl: string,
signal: AbortSignal,
fetchImpl: typeof fetch = VISION_BRIDGE_UA_FETCH
): Promise<string> {
const remoteImage = await fetchRemoteImage(imageUrl, {
// GHSA-34rg-3pqj-35g9: `imageUrl` is caller input (a chat `image_url` part) — pin
// `public-only` explicitly; never the operator outbound policy (`block-metadata` on a
// local-first default install), which would let a request body make the server
// fetch loopback/LAN URLs and inline the bytes into the vision self-call.
guard: "public-only",
// `pinDns` is validation-only here: with `fetchImpl` injected the library validates
// every DNS answer but cannot pin the connection (it never builds its own fetch).
pinDns: true,
signal,
// Bypass the runtime's hooked global fetch (ProxyFetch) — a dead local
// proxy (e.g. 127.0.0.1:8317) would otherwise break the download.
fetchImpl,
});
const mediaType = remoteImage.contentType.split(";")[0]?.trim() || "image/png";
const dataUri = `data:${mediaType};base64,${remoteImage.buffer.toString("base64")}`;
// Downscale to the long-edge cap before handing the image to the vision
// model self-call — scoped to this bridge-fetched image only, never the
// user's raw passthrough payload (opt-in principle, HR#20).
// `normalizeDataUri` never throws and is a passthrough for non-image bytes.
return normalizeDataUri(dataUri);
}
async function normalizeVisionImageInput(
imageInput: string,
isAnthropic: boolean,
signal: AbortSignal,
fetchImpl?: typeof fetch
): Promise<string> {
const normalizedImage = resolveImageAsDataUri(imageInput);
if (
isAnthropic &&
(normalizedImage.startsWith("http://") || normalizedImage.startsWith("https://"))
) {
return fetchRemoteImageAsDataUri(normalizedImage, signal, fetchImpl);
}
return normalizedImage;
}
export interface VisionModelConfig {
model: string;
prompt: string;
timeoutMs: number;
maxImages: number;
/** Route catalog models through OmniRoute so provider connections remain authoritative. */
routeThroughOmniRoute?: boolean;
/** Optional parent deadline/abort propagated by multi-step media bridges. */
signal?: AbortSignal;
/** Injectable fetch (tests). Defaults to undici fetch to bypass the runtime's hooked global fetch. */
fetchImpl?: typeof fetch;
/** Receives the actual successful model while the public return value remains a string. */
onModelUsed?: (model: string) => void;
/**
* Caller API-key scope. Model selection is limited to what the key may use,
* and the describe call goes through the OmniRoute self-loop carrying the key
* id so the key's connection/model policy is enforced on the sub-request.
*/
keyScope?: import("./visionBridgeKeyScope").VisionBridgeKeyScope;
}
/** Self-loop header that scopes an internal describe call to the caller's API key. */
export const SELF_LOOP_KEY_ID_HEADER = "x-omniroute-self-loop-key-id";
/** Task-aware focus hint (codex-vision-proxy pattern): steer the description
* toward what the user actually asked, instead of a generic caption. */
export function composeVisionPrompt(
basePrompt: string,
lastUserText: string | undefined,
taskAware: boolean
): string {
const text = (lastUserText ?? "").trim();
if (!taskAware || !text) return basePrompt;
const hint = text.length > 500 ? `${text.slice(0, 500)}…` : text;
return `${basePrompt}\n\nThe user asked: "${hint}". Focus your description on what is relevant to answering this, and transcribe any text visible in the image.`;
}
/**
* A key-scoped call selects (and falls back) only among models the key may
* use, in its own selection-cache partition. Without a scope the router inputs
* are returned unchanged.
*/
function applyVisionKeyScope(
keyScope: import("./visionBridgeKeyScope").VisionBridgeKeyScope | undefined,
routerConfig: Partial<import("./visionBridgeRouter").VisionBridgeRouterConfig> | undefined,
deps: import("./visionBridgeRouter").VisionBridgeRouterDeps | undefined
): {
routerConfig: Partial<import("./visionBridgeRouter").VisionBridgeRouterConfig> | undefined;
deps: import("./visionBridgeRouter").VisionBridgeRouterDeps | undefined;
} {
if (!keyScope) return { routerConfig, deps };
return {
routerConfig: { ...routerConfig, cacheScope: keyScope.cacheScope },
deps: {
...deps,
hasUsableCredentials: deps?.hasUsableCredentials
? combineCredentialChecks(deps.hasUsableCredentials, keyScope)
: keyScope.hasUsableCredentials,
},
};
}
/**
* Call the vision model to get an image description.
* Supports both OpenAI-compatible and Anthropic API formats.
* Uses auto-routing to select the fastest available model.
*/
export async function callVisionModel(
imageDataUri: string,
config: VisionModelConfig,
apiKey?: string,
routerConfig?: Partial<import("./visionBridgeRouter").VisionBridgeRouterConfig>,
deps?: import("./visionBridgeRouter").VisionBridgeRouterDeps
): Promise<string> {
if (config.signal?.aborted) {
throw new Error("Vision model call aborted");
}
({ routerConfig, deps } = applyVisionKeyScope(config.keyScope, routerConfig, deps));
// Auto-select the best vision model. `deps` is the router's existing
// injectable credential-check seam — without forwarding it, tests (and any
// embedder) cannot keep model selection away from the live connections DB.
const modelToUse = await getBestVisionModel(
{
fixedModel: config.model,
...routerConfig,
},
deps
);
// (#8430) When no vision-capable provider has usable credentials on this
// instance, surface a clear error instead of attempting a describe call that
// would fail with an opaque auth/serde error upstream.
if (!modelToUse) {
throw new Error("No vision-capable provider connected, cannot process image request");
}
let lastError: Error | null = null;
// Try primary model + fallbacks
const modelsToTry = [modelToUse, ...(await getFallbackModels(modelToUse, routerConfig, deps))];
const maxAttempts = Math.min(modelsToTry.length, routerConfig?.maxFallbackAttempts ?? 3);
for (let attempt = 0; attempt < maxAttempts; attempt++) {
if (config.signal?.aborted) {
throw lastError ?? new Error("Vision model call aborted");
}
const currentModel = modelsToTry[attempt];
const attemptStart = Date.now();
try {
const result = await callVisionModelSingle(
imageDataUri,
{ ...config, model: currentModel },
apiKey
);
recordLatency(currentModel, Date.now() - attemptStart, true);
try {
config.onModelUsed?.(currentModel);
} catch {
// Observability callbacks must never turn a successful caption into a retry.
}
return result;
} catch (error) {
recordLatency(currentModel, Date.now() - attemptStart, false);
lastError = error instanceof Error ? error : new Error(String(error));
if (config.signal?.aborted) {
throw lastError;
}
// Continue to next model on failure
}
}
// All models failed
throw lastError || new Error("All vision models failed");
}
/**
* Unwrap the detailed-log/diagnostics envelope that some OmniRoute paths attach
* to provider responses (`{ _streamed, _format, summary: {...} }`). Returns the
* inner `summary` object when present, otherwise the value unchanged.
*/
function unwrapVisionSummary(value: unknown): unknown {
if (value && typeof value === "object" && !Array.isArray(value)) {
const record = value as Record<string, unknown>;
if (record.summary && typeof record.summary === "object") {
return record.summary;
}
}
return value;
}
/**
* Parse a vision-bridge response body that may be:
* 1. Plain JSON (`{ choices: [...] }` / `{ content: [...] }`)
* 2. An SSE stream of `data: {...}` lines (forceStream providers, or
* OmniRoute's self-loop when the `stream` default kicks in)
* 3. The `{ _streamed, _format, summary }` diagnostics envelope
*
* For SSE input, aggregates `delta.content` / `delta.reasoning_content`
* (OpenAI-compatible) and `delta.text` (Anthropic-style `content_block_delta`)
* across all chunks into a single chat.completion-shaped object. Returns `null`
* when the body yields nothing usable.
*/
function parseSseVisionBody(rawBody: string): unknown {
const trimmed = String(rawBody || "").trim();
if (!trimmed) return null;
// Direct JSON (normal non-stream response).
try {
return unwrapVisionSummary(JSON.parse(trimmed));
} catch {
// Fall through to SSE aggregation.
}
const contentParts: string[] = [];
const reasoningParts: string[] = [];
const anthropicTextParts: string[] = [];
let sawChoices = false;
for (const line of trimmed.split(/\r?\n/)) {
const lineTrimmed = line.trim();
if (!lineTrimmed.startsWith("data:")) continue;
const payload = lineTrimmed.slice(5).trim();
if (!payload || payload === "[DONE]") continue;
let chunk: Record<string, unknown>;
try {
chunk = JSON.parse(payload);
} catch {
continue; // Ignore malformed lines and keep scanning.
}
if (!chunk || typeof chunk !== "object") continue;
const unwrapped = unwrapVisionSummary(chunk) as Record<string, unknown>;
// Error-only SSE chunk (`data: {"error":{...}}` with no choices) — surface
// the upstream message instead of a generic "empty or invalid response".
if (unwrapped.error != null && !Array.isArray(unwrapped.choices)) {
const err = unwrapped.error;
let message = "";
if (typeof err === "string") {
message = err;
} else if (typeof err === "object" && !Array.isArray(err)) {
message = (err as { message?: unknown }).message
? String((err as { message?: unknown }).message)
: JSON.stringify(err);
} else {
message = String(err);
}
throw new Error(`Vision API error: ${message}`);
}
const choice = (unwrapped.choices as Array<Record<string, unknown>> | undefined)?.[0];
if (choice) sawChoices = true;
const delta = choice?.delta as Record<string, unknown> | undefined;
if (typeof delta?.content === "string" && delta.content.length > 0) {
contentParts.push(delta.content);
}
if (typeof delta?.reasoning_content === "string" && delta.reasoning_content.length > 0) {
reasoningParts.push(delta.reasoning_content);
}
// opencode-routed gateways (e.g. mimo-v2.5-free) stream chain-of-thought in
// `delta.reasoning` instead of `reasoning_content` (#6623).
if (typeof delta?.reasoning === "string" && delta.reasoning.length > 0) {
reasoningParts.push(delta.reasoning);
}
// Some providers put a full message (not a delta) in the final chunk.
const message = choice?.message as Record<string, unknown> | undefined;
if (typeof message?.content === "string" && message.content.length > 0) {
contentParts.push(message.content);
}
if (typeof message?.reasoning_content === "string" && message.reasoning_content.length > 0) {
reasoningParts.push(message.reasoning_content);
}
if (typeof message?.reasoning === "string" && message.reasoning.length > 0) {
reasoningParts.push(message.reasoning);
}
// Anthropic-style streaming: `content_block_delta` with `delta.text`.
if (Array.isArray(unwrapped.content)) {
for (const block of unwrapped.content as Array<Record<string, unknown>>) {
if (typeof block?.text === "string" && block.text.length > 0) {
anthropicTextParts.push(block.text);
}
}
}
}
if (
contentParts.length === 0 &&
reasoningParts.length === 0 &&
anthropicTextParts.length === 0 &&
!sawChoices
) {
return null;
}
const content = contentParts.join("").trim();
const reasoning = reasoningParts.join("").trim();
const anthropicText = anthropicTextParts.join("").trim();
if (anthropicText && !content) {
return { content: [{ type: "text", text: anthropicText }] };
}
const message: Record<string, unknown> = { role: "assistant", content };
if (reasoning) message.reasoning_content = reasoning;
return { choices: [{ message }] };
}
/**
* Read a vision-model HTTP response body tolerantly: try `json()` first, then
* fall back to text/SSE parsing. Some OpenAI-compatible backends (including
* OmniRoute's own self-loop and forceStream providers) reply with a `data:`
* SSE stream even for `stream: false`, which makes `response.json()` throw
* `Unexpected token 'd'`.
*/
async function readVisionResponseBody(response: Response): Promise<unknown> {
try {
// JSON path — also unwrap the { _streamed, summary } diagnostics envelope
// that some OmniRoute capture paths attach to provider responses.
return unwrapVisionSummary(await response.json());
} catch {
// Not JSON — attempt SSE / envelope parsing from the raw text.
}
let rawText = "";
try {
if (typeof (response as Response & { text?: unknown }).text === "function") {
rawText = await response.text();
}
} catch {
rawText = "";
}
const parsed = parseSseVisionBody(rawText);
if (parsed === null) {
throw new Error("Vision API returned empty or invalid response");
}
return parsed;
}
/**
* Extract the description text from an OpenAI-compatible vision response.
* Falls back to `reasoning_content` then `reasoning` when `content` is empty —
* reasoning models (e.g. xiaomi/mimo-v2.5, opencode/mimo-v2.5-free) can exhaust
* `max_tokens` on chain-of-thought and return `content: null` with a complete
* analysis in a reasoning field. opencode-routed gateways name that field
* `reasoning` rather than `reasoning_content` (#6623 / #10809).
*/
function extractOpenAICompatibleContent(data: unknown): string {
const record = data as {
choices?: Array<{
message?: { content?: unknown; reasoning_content?: unknown; reasoning?: unknown };
}>;
error?: { message?: string };
} | null;
if (!record || typeof record !== "object" || Array.isArray(record)) {
throw new Error("Vision API returned invalid response");
}
if (record.error) {
throw new Error(`Vision API error: ${record.error.message || JSON.stringify(record.error)}`);
}
const message = record.choices?.[0]?.message;
const content = typeof message?.content === "string" ? message.content.trim() : "";
if (content) return content;
const reasoning =
typeof message?.reasoning_content === "string"
? message.reasoning_content.trim()
: typeof message?.reasoning === "string"
? message.reasoning.trim()
: "";
if (reasoning) return reasoning;
throw new Error("Vision API returned empty or invalid response");
}
/**
* Internal function to call a single vision model.
*/
async function callVisionModelSingle(
imageDataUri: string,
config: VisionModelConfig,
apiKey?: string
): Promise<string> {
const controller = new AbortController();
const timeoutId = setTimeout(() => controller.abort(), config.timeoutMs);
const signal = config.signal
? AbortSignal.any([config.signal, controller.signal])
: controller.signal;
// Resolve API key based on provider
const resolvedApiKey = resolveProviderApiKey(config.model, apiKey);
// Production callers (VisionBridgeGuardrail) inject undici fetch to bypass
// the runtime's hooked global fetch (ProxyFetch). Defaults to globalThis.fetch
// so existing unit tests that mock it keep working.
const fetchImpl = config.fetchImpl ?? globalThis.fetch;
// Detect provider from model identifier. Claude-wire targets (minimax, zai,
// …) cannot ingest remote image URLs — normalize to base64 so the self-loop
// body reaches the backend as a data URI (the OpenAI→claude translator only
// preserves data URIs as base64; remote URLs become source.url which these
// backends reject).
// A key-scoped describe always goes through the self-loop: direct provider
// calls with operator env keys would bypass the key's connection policy.
const routeThroughOmniRoute = config.routeThroughOmniRoute === true || Boolean(config.keyScope);
const isAnthropic = !routeThroughOmniRoute && config.model.startsWith("anthropic/");
const requiresBase64 = isAnthropic || isClaudeWireFormatModel(config.model);
try {
// Extract model name from provider/model format
const modelName = config.model.includes("/") ? config.model.split("/")[1] : config.model;
const normalizedImageInput = await normalizeVisionImageInput(
imageDataUri,
requiresBase64,
signal,
fetchImpl
);
let response: Response;
if (isAnthropic) {
// Anthropic API path
const anthropicBaseUrl = process.env.ANTHROPIC_API_URL || "https://api.anthropic.com";
// Parse data URI to extract media type and base64 data
const matches = normalizedImageInput.match(/^data:([^;]+);base64,(.+)$/);
let mediaType = "image/png";
let base64Data = normalizedImageInput;
if (matches) {
mediaType = matches[1];
base64Data = matches[2];
}
response = await fetchImpl(`${anthropicBaseUrl}/v1/messages`, {
method: "POST",
signal,
headers: {
"x-api-key": resolvedApiKey,
"anthropic-version": "2023-06-01",
"content-type": "application/json",
},
body: JSON.stringify({
model: modelName,
messages: [
{
role: "user",
content: [
{
type: "image",
source: {
type: "base64",
media_type: mediaType,
data: base64Data,
},
},
{
type: "text",
text: config.prompt,
},
],
},
],
max_tokens: 300,
}),
});
} else {
// OpenAI-compatible path (default) — issue #2232: honor
// VISION_BRIDGE_BASE_URL so the vision-bridge call can be routed through
// OmniRoute itself or any other OpenAI-compatible endpoint instead of
// hardcoded api.openai.com.
const baseUrl = routeThroughOmniRoute
? `http://localhost:${getRuntimePorts().port}/v1`
: resolveVisionBridgeBaseUrl(config.model);
// When routing through the OmniRoute self-loop (non-standard provider),
// keep the full provider-prefixed model ID so OmniRoute can resolve the
// correct provider backend. Only strip the prefix for direct OpenAI calls.
const useFullModelId =
routeThroughOmniRoute ||
(baseUrl.startsWith("http://localhost") &&
config.model.includes("/") &&
!config.model.startsWith("openai/"));
const requestModel = useFullModelId ? config.model : modelName;
// Build headers with optional recursion guard for self-loop calls.
// When routing through OmniRoute's own API, omit the vision-bridge
// guardrail on the sub-request to prevent infinite recursion.
// OmniRoute's own listener authenticates the self-loop bearer (the env key or the
// per-process secret, both accepted by API-key validation), which the admission
// bypass below also requires. Any other endpoint — api.openai.com, or another
// localhost server set as VISION_BRIDGE_BASE_URL — gets only the provider key and
// never an OmniRoute credential; with no provider key the header is omitted.
// A bare-id call to our own listener keeps an operator VISION_BRIDGE_API_KEY.
const selfLoopBearer =
isOwnOmniRouteBaseUrl(baseUrl) && (useFullModelId || !resolvedApiKey)
? resolveSelfLoopBearer()
: null;
const bearer = selfLoopBearer ?? resolvedApiKey;
const headers: Record<string, string> = {
"Content-Type": "application/json",
// Explicit JSON opt-in: without `Accept: application/json` OmniRoute's
// self-loop defaults to SSE (resolveStreamFlag's legacy default) and the
// describe call would receive a `data:` stream that response.json() can't
// parse (`Unexpected token 'd'`), failing the whole vision-bridge
// describe path. Pair with `stream: false` below.
Accept: "application/json",
};
if (bearer) headers.Authorization = `Bearer ${bearer}`;
// Any call into our own listener carries the recursion guard, including a bare-id one.
if (useFullModelId || selfLoopBearer) {
headers["x-omniroute-disabled-guardrails"] = routeThroughOmniRoute
? "vision-bridge,video-bridge"
: "vision-bridge";
// The compression pipeline must not touch the image payload of the
// self-loop describe call (stacked RTK/Caveman can mangle data URIs).
headers["x-omniroute-compression"] = "off";
}
if (selfLoopBearer) {
// Internal self-loop sub-request: the parent request already holds the
// single heavyweight admission lease (`CHAT_MAX_HEAVY_IN_FLIGHT=1`), so a
// large base64-image describe body would be rejected with 503
// `chat_admission_busy` before it is described. The route only honors
// this header when the bearer is the self-loop bearer (set above), so
// external clients cannot use it to bypass admission.
headers["x-omniroute-admission-bypass"] = "internal";
// Scope the sub-request to the caller's key: the receiving chat request
// applies that key's connection/model policy (honored only alongside the
// self-loop bearer, so it can only narrow access).
if (config.keyScope) headers[SELF_LOOP_KEY_ID_HEADER] = config.keyScope.apiKeyId;
}
response = await fetchImpl(`${baseUrl}/chat/completions`, {
method: "POST",
signal,
headers,
body: JSON.stringify({
model: requestModel,
stream: false,
messages: [
{
role: "user",
content: [
{
// Global, not OpenCode-scoped: this is OmniRoute's own internal
// describe self-loop (VisionBridgeGuardrail), called for every
// caller/provider when the target model lacks vision support —
// there is no client-identity signal at this layer to gate on
// (unlike the OpenCode-only `isOpencodeClient` default in
// `chatCore/upstreamBody.ts`, which forwards the *caller's own*
// image_url.detail and is deliberately scoped). "high" is
// requested unconditionally because the describe prompt asks
// the vision model to transcribe visible text
// (`modalityBridgeVisionTaskAware`, see docs/security/GUARDRAILS.md)
// — low-detail sampling degrades OCR accuracy for every
// describe call, not just OpenCode-originated ones. This path
// only affects the OpenAI-compatible wire format branch; the
// Anthropic branch above has no `detail` concept and is
// unaffected (see the "unaffected for Anthropic" compat
// assertion in visionBridgeHelpers.callVisionModel.test.ts).
type: "image_url",
image_url: {
url: normalizedImageInput,
detail: "high",
},
},
{ type: "text", text: config.prompt },
],
},
],
max_tokens: 300,
}),
});
}
clearTimeout(timeoutId);
if (!response.ok) {
const errorText = await response.text().catch(() => "Unknown error");
throw new Error(`Vision API error ${response.status}: ${errorText}`);
}
const data = await readVisionResponseBody(response);
if (isAnthropic) {
// Anthropic response format: { content: [{ type: "text", text: "..." }] }
const anthropicData = data as {
content?: Array<{ type?: string; text?: string }>;
error?: { message?: string };
};
if (anthropicData.error) {
throw new Error(
`Vision API error: ${anthropicData.error.message || JSON.stringify(anthropicData.error)}`
);
}
const textContent = anthropicData.content?.find((c) => c.type === "text");
const content = textContent?.text;
if (!content || typeof content !== "string") {
throw new Error("Vision API returned empty or invalid response");
}
return content.trim();
} else {
// OpenAI-compatible response format. Falls back to reasoning_content when
// content is null — reasoning models (e.g. xiaomi/mimo-v2.5) can exhaust
// max_tokens on chain-of-thought and return content: null with the full
// analysis in reasoning_content.
return extractOpenAICompatibleContent(data);
}
} catch (error) {
clearTimeout(timeoutId);
if (error instanceof Error && error.name === "AbortError") {
throw new Error(
config.signal?.aborted ? "Vision model call aborted" : "Vision model call timed out"
);
}
throw error;
}
}
export interface RequestBody {
model?: string;
messages?: RequestMessage[];
input?: RequestMessage[];
[key: string]: unknown;
}
export function replaceImageParts(
body: RequestBody,
// #4012: a `null` entry means the describe call failed for that image — keep
// the original image part instead of dropping it / stubbing "(unavailable)".
descriptions: (string | null)[]
): RequestBody {
if (!descriptions || descriptions.length === 0) {
return body;
}
const result = structuredClone(body) as RequestBody;
const usesResponsesInput = !Array.isArray(result.messages) && Array.isArray(result.input);
const requestMessages = Array.isArray(result.messages)
? result.messages
: usesResponsesInput
? result.input
: null;
if (!requestMessages) {
return result;
}
// Splice via the unified detector so nested images (image inside a
// tool_result's content array, etc.) are replaced in the SAME order the
// guardrail extracted them (extract↔replace contract). Nested hits carry
// `path`, which the splice walks; top-level hits swap their content slot.
// `input_image` (Responses API) is read through a widened type but MUST be
// replaceable — extractImageParts allowlists it, and every extracted part
// needs a matching splice here.
const replacementTextType: "text" | "input_text" = usesResponsesInput ? "input_text" : "text";
const mediaParts = detectMediaParts(requestMessages).filter(
(p) => p.kind === "image" && REPLACEABLE_IMAGE_SHAPES.has(p.shape)
);
let descriptionIndex = 0;
for (const part of mediaParts) {
const description =
descriptionIndex < descriptions.length ? descriptions[descriptionIndex++] : null;
if (description == null) {
// #4012: describe failed for this image — preserve the original image
// so a vision-capable upstream can still process it.
continue;
}
const message = requestMessages[part.messageIndex];
if (!message || !Array.isArray(message.content)) continue;
const path = part.path;
if (!path || path.length === 0) {
// Top-level part: swap the content slot itself with a text part.
(message.content as unknown[])[part.partIndex] = {
type: replacementTextType,
text: description,
};
} else {
// Nested hit: walk the container part to the media object and splice it.
const container = message.content[part.partIndex] as Record<string, unknown>;
replaceObjectAtPath(container, path, {
type: replacementTextType,
text: description,
});
}
}
return result;
}
function replaceObjectAtPath(
container: Record<string, unknown>,
path: (string | number)[],
replacement: Record<string, unknown>
): void {
let node: unknown = container;
for (let i = 0; i < path.length - 1; i++) {
const next = (node as Record<string, unknown> | null | undefined)?.[path[i] as string];
if (next == null || typeof next !== "object") return;
node = next;
}
(node as Record<string, unknown>)[path[path.length - 1] as string] = replacement;
}