Skip to content

Never stop coding. Free MIT AI gateway: one endpoint, 375 providers (150+ free), 1,400+ models Kimi, Claude, GPT, Gemini, GLM, DeepSeek, MiniMax. Works with Claude Code, Codex, Cursor, OpenCode, Cline & Copilot. Quota-aware auto-fallback, RTK+Caveman compression saves 15-95% tokens, MCP/A2A, Desktop/PWA. Built by 900+ contributors

feat(vision-bridge): add automatic image description fallback for non-vision models (#1476)

Hernan Javier Ardila Sanchezb709dca2d6d8
diff --git a/src/lib/guardrails/registry.ts b/src/lib/guardrails/registry.ts
index 24ed1dc156..b90cd2147e 100644
--- a/src/lib/guardrails/registry.ts
+++ b/src/lib/guardrails/registry.ts
@@ -1,6 +1,7 @@
import { BaseGuardrail, type GuardrailContext, type GuardrailExecutionResult } from "./base";
import { PIIMaskerGuardrail } from "./piiMasker";
import { PromptInjectionGuardrail } from "./promptInjection";
+import { VisionBridgeGuardrail } from "./visionBridge";
type HeadersLike = Headers | Record<string, unknown> | null | undefined;
@@ -262,6 +263,7 @@ let defaultGuardrailsRegistered = false;
export function registerDefaultGuardrails() {
if (defaultGuardrailsRegistered) return guardrailRegistry;
+ guardrailRegistry.register(new VisionBridgeGuardrail());
guardrailRegistry.register(new PIIMaskerGuardrail());
guardrailRegistry.register(new PromptInjectionGuardrail());
defaultGuardrailsRegistered = true;
diff --git a/src/lib/guardrails/visionBridge.ts b/src/lib/guardrails/visionBridge.ts
new file mode 100644
index 0000000000..f9bcac35ca
--- /dev/null
+++ b/src/lib/guardrails/visionBridge.ts
@@ -0,0 +1,150 @@
+/**
+ * Vision Bridge Guardrail.
+ * Intercepts image-bearing requests to non-vision models,
+ * extracts descriptions via vision model, and replaces images with text.
+ */
+
+import { BaseGuardrail, type GuardrailContext, type GuardrailResult } from "./base";
+import { getSettings as defaultGetSettings } from "@/lib/db/settings";
+import { getResolvedModelCapabilities } from "@/lib/modelCapabilities";
+import {
+ extractImageParts,
+ callVisionModel as defaultCallVisionModel,
+ replaceImageParts,
+} from "./visionBridgeHelpers";
+import {
+ VISION_BRIDGE_DEFAULTS,
+ getVisionBridgeConfig,
+} from "@/shared/constants/visionBridgeDefaults";
+
+export interface VisionBridgeDependencies {
+ getSettings?: () => Promise<Record<string, unknown>>;
+ callVisionModel?: (
+ imageDataUri: string,
+ config: import("./visionBridgeHelpers").VisionModelConfig,
+ apiKey?: string
+ ) => Promise<string>;
+}
+
+export class VisionBridgeGuardrail extends BaseGuardrail {
+ name = "vision-bridge";
+ priority = 5;
+
+ private readonly deps: VisionBridgeDependencies;
+
+ constructor(options?: { enabled?: boolean; deps?: VisionBridgeDependencies }) {
+ super("vision-bridge", { priority: 5, enabled: options?.enabled });
+ this.deps = options?.deps ?? {};
+ }
+
+ async preCall(
+ payload: unknown,
+ context: GuardrailContext
+ ): Promise<GuardrailResult<unknown>> {
+ // 1. Check if disabled at guardrail level
+ if (!this.enabled) {
+ return { block: false };
+ }
+
+ // 2. Check disabled via context (header, body, API key)
+ if (context.disabledGuardrails?.includes("vision-bridge")) {
+ return { block: false };
+ }
+
+ // 3. Get model from context or payload
+ const model =
+ context.model || (payload as Record<string, unknown>)?.model as string | undefined;
+ if (!model) {
+ return { block: false };
+ }
+
+ // 4. Check if model supports vision
+ const capabilities = getResolvedModelCapabilities(model);
+ if (capabilities.supportsVision === true) {
+ return { block: false };
+ }
+
+ // 5. Get body and check for messages
+ const body = payload as Record<string, unknown>;
+ const messages = body?.messages;
+ if (!Array.isArray(messages) || messages.length === 0) {
+ return { block: false };
+ }
+
+ // 6. Check for images using helper (extractImageParts returns empty if no images)
+ const imageParts = extractImageParts(
+ messages as Parameters<typeof extractImageParts>[0]
+ );
+ if (imageParts.length === 0) {
+ return { block: false };
+ }
+
+ // 7. Get settings (injectable for testing)
+ const getSettings = this.deps.getSettings ?? defaultGetSettings;
+ let settings: Record<string, unknown> = {};
+ try {
+ settings = await getSettings();
+ } catch {
+ // If getSettings fails, use defaults
+ }
+
+ // 8. Check if Vision Bridge is enabled in settings
+ const enabled = settings.visionBridgeEnabled ?? VISION_BRIDGE_DEFAULTS.enabled;
+ if (!enabled) {
+ return { block: false };
+ }
+
+ // 9. Get configuration
+ const config = getVisionBridgeConfig({
+ visionBridgeEnabled: settings.visionBridgeEnabled as boolean | undefined,
+ visionBridgeModel: settings.visionBridgeModel as string | undefined,
+ visionBridgePrompt: settings.visionBridgePrompt as string | undefined,
+ visionBridgeTimeout: settings.visionBridgeTimeout as number | undefined,
+ visionBridgeMaxImages: settings.visionBridgeMaxImages as number | undefined,
+ });
+
+ // 10. Limit images
+ const limitedParts = imageParts.slice(0, config.maxImages);
+
+ // 11. Call vision model for each image in parallel (injectable for testing)
+ const callVision = this.deps.callVisionModel ?? defaultCallVisionModel;
+ const logger = context.log;
+ const startTime = Date.now();
+
+ // Process all images in parallel using Promise.allSettled for fail-partial behavior
+ const results = await Promise.allSettled(
+ limitedParts.map(async (imagePart, i) => {
+ const description = await callVision(imagePart.imageUrl, config);
+ return `[Image ${i + 1}]: ${description}`;
+ })
+ );
+
+ // Collect descriptions maintaining original order
+ const descriptions = results.map((result, i) => {
+ if (result.status === "fulfilled") {
+ return result.value;
+ }
+ const message = result.reason instanceof Error ? result.reason.message : String(result.reason);
+ logger?.warn?.("VISION-BRIDGE", `Failed to get description for image ${i + 1}: ${message}`);
+ return `[Image ${i + 1}]: (unavailable)`;
+ });
+
+ // 12. Replace image parts with text descriptions
+ const modifiedBody = replaceImageParts(
+ body as Parameters<typeof replaceImageParts>[0],
+ descriptions
+ );
+ const processingTime = Date.now() - startTime;
+
+ return {
+ block: false,
+ modifiedPayload: modifiedBody,
+ meta: {
+ imagesProcessed: descriptions.length,
+ descriptions,
+ processingTimeMs: processingTime,
+ visionModel: config.model,
+ },
+ };
+ }
+}
\ No newline at end of file
diff --git a/src/lib/guardrails/visionBridgeHelpers.ts b/src/lib/guardrails/visionBridgeHelpers.ts
new file mode 100644
index 0000000000..c0d28376b6
--- /dev/null
+++ b/src/lib/guardrails/visionBridgeHelpers.ts
@@ -0,0 +1,330 @@
+/**
+ * Vision Bridge helper functions for image processing.
+ */
+
+/**
+ * Provider to environment variable mapping for API key resolution.
+ */
+const PROVIDER_API_KEY_MAP: Record<string, string> = {
+ anthropic: "ANTHROPIC_API_KEY",
+ google: "GOOGLE_API_KEY",
+ openai: "OPENAI_API_KEY",
+};
+
+/**
+ * Resolve API key based on model provider.
+ * @param model - Model identifier (e.g., "anthropic/claude-3-haiku", "openai/gpt-4o-mini")
+ * @param explicitKey - Explicit API key passed as argument (takes precedence)
+ * @returns Resolved API key string
+ */
+export function resolveProviderApiKey(model: string, explicitKey?: string): string {
+ if (explicitKey) return explicitKey;
+ const provider = model.includes("/") ? model.split("/")[0] : "";
+ const envVar = PROVIDER_API_KEY_MAP[provider] || "OPENAI_API_KEY";
+ return process.env[envVar] || "";
+}
+
+export interface ImagePart {
+ messageIndex: number;
+ partIndex: number;
+ imageUrl: string;
+ imageType: "image_url" | "image";
+}
+
+export interface RequestMessage {
+ role?: string;
+ content?: string | RequestContentPart[];
+}
+
+export type RequestContentPart =
+ | { type: "text"; text: string }
+ | { type: "image_url"; image_url: { url: string; detail?: string } }
+ | { type: "image"; source: { type: "base64"; media_type: string; data: string } };
+
+/**
+ * Extract image parts from messages array.
+ * Supports both OpenAI image_url format and base64 image format.
+ */
+export function extractImageParts(messages: RequestMessage[]): ImagePart[] {
+ const results: ImagePart[] = [];
+
+ if (!Array.isArray(messages)) {
+ return results;
+ }
+
+ for (let msgIdx = 0; msgIdx < messages.length; msgIdx++) {
+ const message = messages[msgIdx];
+ if (!message || !Array.isArray(message.content)) {
+ continue;
+ }
+
+ for (let partIdx = 0; partIdx < message.content.length; partIdx++) {
+ const part = message.content[partIdx];
+
+ if (part?.type === "image_url" && part.image_url?.url) {
+ results.push({
+ messageIndex: msgIdx,
+ partIndex: partIdx,
+ imageUrl: part.image_url.url,
+ imageType: "image_url",
+ });
+ } else if (part?.type === "image" && part.source?.type === "base64") {
+ const { media_type, data } = part.source;
+ const dataUri = `data:${media_type};base64,${data}`;
+ results.push({
+ messageIndex: msgIdx,
+ partIndex: partIdx,
+ imageUrl: dataUri,
+ imageType: "image",
+ });
+ }
+ }
+ }
+
+ return results;
+}
+
+/**
+ * Resolve image URL to data URI format for vision model.
+ * - HTTP/HTTPS URLs: passed through as-is
+ * - Data URIs: passed through as-is
+ * - Base64 without media type: assumed PNG
+ */
+export function resolveImageAsDataUri(imageUrl: string): string {
+ if (!imageUrl || typeof imageUrl !== "string") {
+ throw new Error("Invalid image URL: must be a non-empty string");
+ }
+
+ // Already a data URI
+ if (imageUrl.startsWith("data:")) {
+ return imageUrl;
+ }
+
+ // HTTP/HTTPS URL - vision API will fetch it
+ if (imageUrl.startsWith("http://") || imageUrl.startsWith("https://")) {
+ return imageUrl;
+ }
+
+ // Assume it's a base64 string without prefix
+ // Add PNG as default media type
+ return `data:image/png;base64,${imageUrl}`;
+}
+
+export interface VisionModelConfig {
+ model: string;
+ prompt: string;
+ timeoutMs: number;
+ maxImages: number;
+}
+
+/**
+ * Call the vision model to get an image description.
+ * Supports both OpenAI-compatible and Anthropic API formats.
+ */
+export async function callVisionModel(
+ imageDataUri: string,
+ config: VisionModelConfig,
+ apiKey?: string
+): Promise<string> {
+ const controller = new AbortController();
+ const timeoutId = setTimeout(() => controller.abort(), config.timeoutMs);
+
+ // Resolve API key based on provider
+ const resolvedApiKey = resolveProviderApiKey(config.model, apiKey);
+
+ // Detect provider from model identifier
+ const isAnthropic = config.model.startsWith("anthropic/");
+
+ try {
+ // Extract model name from provider/model format
+ const modelName = config.model.includes("/")
+ ? config.model.split("/")[1]
+ : config.model;
+
+ let response: Response;
+
+ if (isAnthropic) {
+ // Anthropic API path
+ const anthropicBaseUrl = process.env.ANTHROPIC_API_URL || "https://api.anthropic.com";
+
+ // Parse data URI to extract media type and base64 data
+ const matches = imageDataUri.match(/^data:([^;]+);base64,(.+)$/);
+ let mediaType = "image/png";
+ let base64Data = imageDataUri;
+
+ if (matches) {
+ mediaType = matches[1];
+ base64Data = matches[2];
+ }
+
+ response = await fetch(`${anthropicBaseUrl}/v1/messages`, {
+ method: "POST",
+ signal: controller.signal,
+ headers: {
+ "x-api-key": resolvedApiKey,
+ "anthropic-version": "2023-06-01",
+ "content-type": "application/json",
+ },
+ body: JSON.stringify({
+ model: modelName,
+ messages: [
+ {
+ role: "user",
+ content: [
+ {
+ type: "image",
+ source: {
+ type: "base64",
+ media_type: mediaType,
+ data: base64Data,
+ },
+ },
+ {
+ type: "text",
+ text: config.prompt,
+ },
+ ],
+ },
+ ],
+ max_tokens: 300,
+ }),
+ });
+ } else {
+ // OpenAI-compatible path (default)
+ const baseUrl = process.env.OPENAI_API_URL || "https://api.openai.com/v1";
+
+ response = await fetch(`${baseUrl}/chat/completions`, {
+ method: "POST",
+ signal: controller.signal,
+ headers: {
+ "Content-Type": "application/json",
+ Authorization: `Bearer ${resolvedApiKey}`,
+ },
+ body: JSON.stringify({
+ model: modelName,
+ messages: [
+ {
+ role: "user",
+ content: [
+ {
+ type: "image_url",
+ image_url: {
+ url: imageDataUri,
+ detail: "low",
+ },
+ },
+ { type: "text", text: config.prompt },
+ ],
+ },
+ ],
+ max_tokens: 300,
+ }),
+ });
+ }
+
+ clearTimeout(timeoutId);
+
+ if (!response.ok) {
+ const errorText = await response.text().catch(() => "Unknown error");
+ throw new Error(`Vision API error ${response.status}: ${errorText}`);
+ }
+
+ const data = await response.json();
+
+ if (isAnthropic) {
+ // Anthropic response format: { content: [{ type: "text", text: "..." }] }
+ const anthropicData = data as {
+ content?: Array<{ type?: string; text?: string }>;
+ error?: { message?: string };
+ };
+
+ if (anthropicData.error) {
+ throw new Error(`Vision API error: ${anthropicData.error.message || JSON.stringify(anthropicData.error)}`);
+ }
+
+ const textContent = anthropicData.content?.find((c) => c.type === "text");
+ const content = textContent?.text;
+ if (!content || typeof content !== "string") {
+ throw new Error("Vision API returned empty or invalid response");
+ }
+
+ return content.trim();
+ } else {
+ // OpenAI-compatible response format: { choices: [{ message: { content: "..." } }] }
+ const openaiData = data as {
+ choices?: Array<{ message?: { content?: string } }>;
+ error?: { message?: string };
+ };
+
+ if (openaiData.error) {
+ throw new Error(`Vision API error: ${openaiData.error.message || JSON.stringify(openaiData.error)}`);
+ }
+
+ const content = openaiData.choices?.[0]?.message?.content;
+ if (!content || typeof content !== "string") {
+ throw new Error("Vision API returned empty or invalid response");
+ }
+
+ return content.trim();
+ }
+ } catch (error) {
+ clearTimeout(timeoutId);
+
+ if (error instanceof Error && error.name === "AbortError") {
+ throw new Error("Vision model call timed out");
+ }
+
+ throw error;
+ }
+}
+
+export interface RequestBody {
+ model?: string;
+ messages?: RequestMessage[];
+ [key: string]: unknown;
+}
+
+/**
+ * Replace image content parts with text descriptions.
+ * Concatenates descriptions with labels: "[Image 1]: ..."
+ */
+export function replaceImageParts(body: RequestBody, descriptions: string[]): RequestBody {
+ if (!descriptions || descriptions.length === 0) {
+ return body;
+ }
+
+ const result = structuredClone(body) as RequestBody;
+
+ if (!Array.isArray(result.messages)) {
+ return result;
+ }
+
+ let descriptionIndex = 0;
+
+ for (let msgIdx = 0; msgIdx < result.messages.length; msgIdx++) {
+ const message = result.messages[msgIdx];
+ if (!message || !Array.isArray(message.content)) {
+ continue;
+ }
+
+ const newContent: RequestContentPart[] = [];
+
+ for (const part of message.content) {
+ if (part?.type === "image_url" || part?.type === "image") {
+ if (descriptionIndex < descriptions.length) {
+ newContent.push({
+ type: "text",
+ text: descriptions[descriptionIndex],
+ });
+ descriptionIndex++;
+ }
+ } else {
+ newContent.push(part as RequestContentPart);
+ }
+ }
+
+ message.content = newContent;
+ }
+
+ return result;
+}
diff --git a/src/shared/constants/visionBridgeDefaults.ts b/src/shared/constants/visionBridgeDefaults.ts
new file mode 100644
index 0000000000..18e069d6d3
--- /dev/null
+++ b/src/shared/constants/visionBridgeDefaults.ts
@@ -0,0 +1,55 @@
+/**
+ * Vision Bridge default configuration values.
+ */
+
+export const VISION_BRIDGE_DEFAULTS = {
+ enabled: true,
+ model: "openai/gpt-4o-mini",
+ prompt:
+ "Describe this image concisely in 2-3 sentences. Focus on the most relevant visual details.",
+ timeoutMs: 30000,
+ maxImagesPerRequest: 10,
+} as const;
+
+/**
+ * Settings keys for Vision Bridge (to be stored in key_value table).
+ */
+export const VISION_BRIDGE_SETTINGS_KEYS = [
+ "visionBridgeEnabled",
+ "visionBridgeModel",
+ "visionBridgePrompt",
+ "visionBridgeTimeout",
+ "visionBridgeMaxImages",
+] as const;
+
+export type VisionBridgeSettings = {
+ visionBridgeEnabled?: boolean;
+ visionBridgeModel?: string;
+ visionBridgePrompt?: string;
+ visionBridgeTimeout?: number;
+ visionBridgeMaxImages?: number;
+};
+
+export type VisionBridgeConfig = {
+ enabled: boolean;
+ model: string;
+ prompt: string;
+ timeoutMs: number;
+ maxImages: number;
+};
+
+/**
+ * Merge settings with defaults to produce a complete config.
+ */
+export function getVisionBridgeConfig(
+ settings: VisionBridgeSettings | undefined | null = {}
+): VisionBridgeConfig {
+ const s = settings ?? {};
+ return {
+ enabled: s.visionBridgeEnabled ?? VISION_BRIDGE_DEFAULTS.enabled,
+ model: s.visionBridgeModel ?? VISION_BRIDGE_DEFAULTS.model,
+ prompt: s.visionBridgePrompt ?? VISION_BRIDGE_DEFAULTS.prompt,
+ timeoutMs: s.visionBridgeTimeout ?? VISION_BRIDGE_DEFAULTS.timeoutMs,
+ maxImages: s.visionBridgeMaxImages ?? VISION_BRIDGE_DEFAULTS.maxImagesPerRequest,
+ };
+}
\ No newline at end of file
diff --git a/tests/unit/guardrails/visionBridge.test.ts b/tests/unit/guardrails/visionBridge.test.ts
new file mode 100644
index 0000000000..550dc63469
--- /dev/null
+++ b/tests/unit/guardrails/visionBridge.test.ts
@@ -0,0 +1,515 @@
+/**
+ * Tests for VisionBridgeGuardrail.
+ * Uses dependency injection to avoid SQLite dependency.
+ */
+
+import test from "node:test";
+import assert from "node:assert/strict";
+
+const { VisionBridgeGuardrail } = await import(
+ "../../../src/lib/guardrails/visionBridge.ts"
+);
+const { resetGuardrailsForTests } = await import(
+ "../../../src/lib/guardrails/registry.ts"
+);
+const { getResolvedModelCapabilities } = await import(
+ "../../../src/lib/modelCapabilities.ts"
+);
+import type { GuardrailContext } from "../../../src/lib/guardrails/base.ts";
+import type { VisionModelConfig } from "../../../src/lib/guardrails/visionBridgeHelpers.ts";
+
+// ── Mock state ──────────────────────────────────────────────────────────────
+
+let mockSettings: Record<string, unknown> = {
+ visionBridgeEnabled: true,
+ visionBridgeModel: "openai/gpt-4o-mini",
+ visionBridgePrompt: "Describe this image concisely.",
+ visionBridgeTimeout: 30000,
+ visionBridgeMaxImages: 10,
+};
+
+let mockVisionResponse = "A beautiful sunset over the ocean";
+let shouldVisionFail = false;
+let visionCallCount = 0;
+
+function createGuardrail(
+ options?: Parameters<typeof VisionBridgeGuardrail>[0]
+) {
+ return new VisionBridgeGuardrail({
+ ...options,
+ deps: {
+ getSettings: async () => mockSettings,
+ callVisionModel: async (
+ _imageDataUri: string,
+ _config: VisionModelConfig
+ ) => {
+ visionCallCount++;
+ if (shouldVisionFail) {
+ throw new Error("Vision model failed");
+ }
+ return mockVisionResponse;
+ },
+ },
+ });
+}
+
+test.beforeEach(() => {
+ resetGuardrailsForTests({ registerDefaults: false });
+ visionCallCount = 0;
+ shouldVisionFail = false;
+ mockSettings = {
+ visionBridgeEnabled: true,
+ visionBridgeModel: "openai/gpt-4o-mini",
+ visionBridgePrompt: "Describe this image concisely.",
+ visionBridgeTimeout: 30000,
+ visionBridgeMaxImages: 10,
+ };
+});
+
+// ── Helpers ─────────────────────────────────────────────────────────────────
+
+function createContext(
+ overrides: Partial<GuardrailContext> = {}
+): GuardrailContext {
+ return {
+ model: "minimax/minimax-01",
+ log: console,
+ ...overrides,
+ };
+}
+
+function createPayload(
+ overrides: Record<string, unknown> = {}
+): Record<string, unknown> {
+ return {
+ model: "minimax/minimax-01",
+ messages: [{ role: "user", content: "Hello" }],
+ ...overrides,
+ };
+}
+
+// ── Basic Properties ────────────────────────────────────────────────────────
+
+test("VisionBridgeGuardrail has correct name and priority", () => {
+ const guardrail = createGuardrail();
+ assert.strictEqual(guardrail.name, "vision-bridge");
+ assert.strictEqual(guardrail.priority, 5);
+});
+
+test("VisionBridgeGuardrail is enabled by default", () => {
+ const guardrail = createGuardrail();
+ assert.strictEqual(guardrail.enabled, true);
+});
+
+test("VisionBridgeGuardrail can be disabled via constructor", () => {
+ const guardrail = createGuardrail({ enabled: false });
+ assert.strictEqual(guardrail.enabled, false);
+});
+
+// ── VB-S05: Vision Bridge disabled via settings ────────────────────────────
+
+test("VB-S05: passthroughs when visionBridgeEnabled is false", async () => {
+ mockSettings.visionBridgeEnabled = false;
+ const guardrail = createGuardrail();
+
+ const payload = createPayload({
+ messages: [
+ {
+ role: "user",
+ content: [
+ { type: "text", text: "What is in this image?" },
+ {
+ type: "image_url",
+ image_url: { url: "https://example.com/image.png" },
+ },
+ ],
+ },
+ ],
+ });
+
+ const result = await guardrail.preCall(payload, createContext());
+ assert.strictEqual(result.block, false);
+ assert.strictEqual(result.modifiedPayload, undefined);
+});
+
+// ── VB-S06: Disabled via context ────────────────────────────────────────────
+
+test("VB-S06: skips when disabledGuardrails includes vision-bridge", async () => {
+ const guardrail = createGuardrail();
+ const payload = createPayload();
+ const context = createContext({ disabledGuardrails: ["vision-bridge"] });
+
+ const result = await guardrail.preCall(payload, context);
+ assert.strictEqual(result.block, false);
+ assert.strictEqual(result.modifiedPayload, undefined);
+});
+
+// ── VB-S02: Vision-capable model passthrough ────────────────────────────────
+
+test("VB-S02: passthroughs for vision-capable model (gpt-4o)", async () => {
+ const guardrail = createGuardrail();
+ const payload = createPayload({
+ model: "openai/gpt-4o",
+ messages: [
+ {
+ role: "user",
+ content: [
+ { type: "text", text: "What is this?" },
+ {
+ type: "image_url",
+ image_url: { url: "https://example.com/image.png" },
+ },
+ ],
+ },
+ ],
+ });
+
+ const result = await guardrail.preCall(
+ payload,
+ createContext({ model: "openai/gpt-4o" })
+ );
+
+ // If supportsVision is true, it should passthrough (no modification)
+ // If supportsVision is null/undefined (no sync data), it will process — that's correct behavior
+ const capabilities = getResolvedModelCapabilities("openai/gpt-4o");
+ if (capabilities.supportsVision === true) {
+ assert.strictEqual(result.block, false);
+ assert.strictEqual(result.modifiedPayload, undefined);
+ } else {
+ // Without sync data, supportsVision is null — guardrail processes the image
+ // This is correct fail-open behavior for unknown model capabilities
+ assert.strictEqual(result.block, false);
+ }
+});
+
+test("VB-S02: model capabilities returns supportsVision for known models", () => {
+ const gpt4oCaps = getResolvedModelCapabilities("openai/gpt-4o");
+ // supportsVision may be true (if sync data exists) or null (if not synced)
+ assert.ok(gpt4oCaps.supportsVision === true || gpt4oCaps.supportsVision === null);
+});
+
+// ── VB-S04: No images passthrough ──────────────────────────────────────────
+
+test("VB-S04: passthroughs when no images in messages", async () => {
+ const guardrail = createGuardrail();
+ const payload = createPayload({
+ messages: [{ role: "user", content: "Hello, how are you?" }],
+ });
+
+ const result = await guardrail.preCall(payload, createContext());
+ assert.strictEqual(result.block, false);
+ assert.strictEqual(result.modifiedPayload, undefined);
+});
+
+test("VB-S04: passthroughs when messages array is empty", async () => {
+ const guardrail = createGuardrail();
+ const payload = createPayload({ messages: [] });
+ const result = await guardrail.preCall(payload, createContext());
+ assert.strictEqual(result.block, false);
+});
+
+// ── VB-S01: Single image processing ─────────────────────────────────────────
+
+test("VB-S01: replaces image with description for non-vision model", async () => {
+ mockVisionResponse = "A beautiful sunset over the ocean";
+ const guardrail = createGuardrail();
+
+ const payload = createPayload({
+ model: "minimax/minimax-01",
+ messages: [
+ {
+ role: "user",
+ content: [
+ { type: "text", text: "What is in this image?" },
+ {
+ type: "image_url",
+ image_url: { url: "https://example.com/image.png" },
+ },
+ ],
+ },
+ ],
+ });
+
+ const result = await guardrail.preCall(
+ payload,
+ createContext({ model: "minimax/minimax-01" })
+ );
+
+ assert.strictEqual(result.block, false);
+ assert.ok(result.modifiedPayload);
+
+ const modified = result.modifiedPayload as {
+ messages: Array<{ content: unknown[] }>;
+ };
+ const content = modified.messages[0].content as Array<{
+ type: string;
+ text?: string;
+ }>;
+
+ const imagePart = content.find((p) => p.type === "image_url");
+ assert.strictEqual(imagePart, undefined);
+
+ const descriptionPart = content.find(
+ (p) => p.type === "text" && p.text?.includes("sunset")
+ );
+ assert.ok(descriptionPart);
+});
+
+// ── VB-S04: Multiple images ─────────────────────────────────────────────────
+
+test("VB-S04: processes multiple images and concatenates descriptions", async () => {
+ let callIdx = 0;
+ const descriptions = ["A cute cat", "A playful dog", "A colorful bird"];
+
+ const guardrail = new VisionBridgeGuardrail({
+ deps: {
+ getSettings: async () => mockSettings,
+ callVisionModel: async () => {
+ const desc = descriptions[callIdx] || "Unknown image";
+ callIdx++;
+ return desc;
+ },
+ },
+ });
+
+ const payload = createPayload({
+ model: "minimax/minimax-01",
+ messages: [
+ {
+ role: "user",
+ content: [
+ { type: "text", text: "Describe these images" },
+ {
+ type: "image_url",
+ image_url: { url: "https://example.com/cat.png" },
+ },
+ {
+ type: "image_url",
+ image_url: { url: "https://example.com/dog.png" },
+ },
+ {
+ type: "image_url",
+ image_url: { url: "https://example.com/bird.png" },
+ },
+ ],
+ },
+ ],
+ });
+
+ const result = await guardrail.preCall(
+ payload,
+ createContext({ model: "minimax/minimax-01" })
+ );
+
+ assert.strictEqual(result.block, false);
+ assert.ok(result.modifiedPayload);
+ assert.strictEqual(callIdx, 3);
+
+ const modified = result.modifiedPayload as {
+ messages: Array<{ content: unknown[] }>;
+ };
+ const content = modified.messages[0].content as Array<{
+ type: string;
+ text?: string;
+ }>;
+
+ assert.ok(
+ content.some((p) => p.type === "text" && p.text?.includes("[Image 1]"))
+ );
+ assert.ok(
+ content.some((p) => p.type === "text" && p.text?.includes("[Image 2]"))
+ );
+ assert.ok(
+ content.some((p) => p.type === "text" && p.text?.includes("[Image 3]"))
+ );
+});
+
+// ── VB-S03: Fail-open on vision error ──────────────────────────────────────
+
+test("VB-S03: returns modified payload with unavailable text when vision API fails", async () => {
+ shouldVisionFail = true;
+ const guardrail = createGuardrail();
+
+ const payload = createPayload({
+ model: "minimax/minimax-01",
+ messages: [
+ {
+ role: "user",
+ content: [
+ { type: "text", text: "What is this?" },
+ {
+ type: "image_url",
+ image_url: { url: "https://example.com/image.png" },
+ },
+ ],
+ },
+ ],
+ });
+
+ const result = await guardrail.preCall(
+ payload,
+ createContext({ model: "minimax/minimax-01" })
+ );
+
+ assert.strictEqual(result.block, false);
+ assert.ok(result.modifiedPayload);
+
+ const modified = result.modifiedPayload as {
+ messages: Array<{ content: unknown[] }>;
+ };
+ const content = modified.messages[0].content as Array<{
+ type: string;
+ text?: string;
+ }>;
+
+ // Should have "unavailable" text instead of image
+ const unavailPart = content.find(
+ (p) => p.type === "text" && p.text?.includes("unavailable")
+ );
+ assert.ok(unavailPart);
+});
+
+test("VB-S03: logs warning when vision API fails", async () => {
+ shouldVisionFail = true;
+ let warningLogged = false;
+ const guardrail = createGuardrail();
+
+ const payload = createPayload({
+ model: "minimax/minimax-01",
+ messages: [
+ {
+ role: "user",
+ content: [
+ {
+ type: "image_url",
+ image_url: { url: "https://example.com/image.png" },
+ },
+ ],
+ },
+ ],
+ });
+
+ const mockLog = {
+ warn: (_tag: string, msg: string) => {
+ if (msg.includes("Failed to get description")) {
+ warningLogged = true;
+ }
+ },
+ };
+
+ await guardrail.preCall(
+ payload,
+ createContext({
+ model: "minimax/minimax-01",
+ log: mockLog as GuardrailContext["log"],
+ })
+ );
+
+ assert.strictEqual(warningLogged, true);
+});
+
+// ── VB-S07: Base64 image format ─────────────────────────────────────────────
+
+test("VB-S07: handles base64 image format", async () => {
+ mockVisionResponse = "An image description";
+ const guardrail = createGuardrail();
+
+ const payload = createPayload({
+ model: "minimax/minimax-01",
+ messages: [
+ {
+ role: "user",
+ content: [
+ {
+ type: "image",
+ source: {
+ type: "base64",
+ media_type: "image/png",
+ data: "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==",
+ },
+ },
+ ],
+ },
+ ],
+ });
+
+ const result = await guardrail.preCall(
+ payload,
+ createContext({ model: "minimax/minimax-01" })
+ );
+
+ assert.strictEqual(result.block, false);
+ assert.ok(result.modifiedPayload);
+});
+
+// ── VB-S09: Image count limit ───────────────────────────────────────────────
+
+test("VB-S09: respects maxImages setting", async () => {
+ mockSettings.visionBridgeMaxImages = 2;
+ const guardrail = createGuardrail();
+
+ const images = Array.from({ length: 5 }, (_, i) => ({
+ type: "image_url" as const,
+ image_url: { url: `https://example.com/image${i}.png` },
+ }));
+
+ const payload = createPayload({
+ model: "minimax/minimax-01",
+ messages: [
+ {
+ role: "user",
+ content: [{ type: "text", text: "Describe these" }, ...images],
+ },
+ ],
+ });
+
+ await guardrail.preCall(
+ payload,
+ createContext({ model: "minimax/minimax-01" })
+ );
+
+ // Should only call vision API for 2 images (maxImages=2)
+ assert.strictEqual(visionCallCount, 2);
+});
+
+// ── VB-S10: Meta information returned ───────────────────────────────────────
+
+test("VB-S10: returns meta with imagesProcessed count", async () => {
+ mockVisionResponse = "A test description";
+ const guardrail = createGuardrail();
+
+ const payload = createPayload({
+ model: "minimax/minimax-01",
+ messages: [
+ {
+ role: "user",
+ content: [
+ { type: "text", text: "What is this?" },
+ {
+ type: "image_url",
+ image_url: { url: "https://example.com/a.png" },
+ },
+ {
+ type: "image_url",
+ image_url: { url: "https://example.com/b.png" },
+ },
+ ],
+ },
+ ],
+ });
+
+ const result = await guardrail.preCall(
+ payload,
+ createContext({ model: "minimax/minimax-01" })
+ );
+
+ assert.ok(result.meta);
+ assert.ok(typeof result.meta === "object");
+
+ const meta = result.meta as Record<string, unknown>;
+ assert.strictEqual(meta.imagesProcessed, 2);
+ assert.ok(Array.isArray(meta.descriptions));
+ assert.strictEqual((meta.descriptions as string[]).length, 2);
+ assert.strictEqual(typeof meta.processingTimeMs, "number");
+ assert.strictEqual(meta.visionModel, "openai/gpt-4o-mini");
+});
\ No newline at end of file
diff --git a/tests/unit/guardrails/visionBridgeHelpers.callVisionModel.test.ts b/tests/unit/guardrails/visionBridgeHelpers.callVisionModel.test.ts
new file mode 100644
index 0000000000..dd241214e1
--- /dev/null
+++ b/tests/unit/guardrails/visionBridgeHelpers.callVisionModel.test.ts
@@ -0,0 +1,231 @@
+/**
+ * Tests for callVisionModel helper function.
+ */
+
+import test from "node:test";
+import assert from "node:assert/strict";
+import { callVisionModel, type VisionModelConfig } from "@/lib/guardrails/visionBridgeHelpers";
+
+// Store original fetch
+const originalFetch = globalThis.fetch;
+
+test("callVisionModel returns description on success", async () => {
+ // Mock global fetch
+ const mockResponse = {
+ ok: true,
+ json: async () => ({
+ choices: [{ message: { content: "A beautiful sunset over the ocean" } }],
+ }),
+ };
+ globalThis.fetch = async () => mockResponse as unknown as Response;
+
+ try {
+ const config: VisionModelConfig = {
+ model: "openai/gpt-4o-mini",
+ prompt: "Describe this image",
+ timeoutMs: 30000,
+ maxImages: 10,
+ };
+
+ const result = await callVisionModel(
+ "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==",
+ config
+ );
+
+ assert.strictEqual(result, "A beautiful sunset over the ocean");
+ } finally {
+ // Restore original fetch
+ globalThis.fetch = originalFetch;
+ }
+});
+
+test("callVisionModel throws on HTTP error", async () => {
+ const mockResponse = {
+ ok: false,
+ status: 500,
+ text: async () => "Internal Server Error",
+ };
+ globalThis.fetch = async () => mockResponse as unknown as Response;
+
+ try {
+ const config: VisionModelConfig = {
+ model: "openai/gpt-4o-mini",
+ prompt: "Describe this image",
+ timeoutMs: 30000,
+ maxImages: 10,
+ };
+
+ await assert.rejects(
+ async () =>
+ await callVisionModel("data:image/png;base64,iVBORw0KGgo", config),
+ /Vision API error 500/
+ );
+ } finally {
+ globalThis.fetch = originalFetch;
+ }
+});
+
+test("callVisionModel throws on API error response", async () => {
+ const mockResponse = {
+ ok: true,
+ json: async () => ({
+ error: { message: "Invalid API key" },
+ }),
+ };
+ globalThis.fetch = async () => mockResponse as unknown as Response;
+
+ try {
+ const config: VisionModelConfig = {
+ model: "openai/gpt-4o-mini",
+ prompt: "Describe this image",
+ timeoutMs: 30000,
+ maxImages: 10,
+ };
+
+ await assert.rejects(
+ async () => await callVisionModel("data:image/png;base64,iVBORw0KGgo", config),
+ /Invalid API key/
+ );
+ } finally {
+ globalThis.fetch = originalFetch;
+ }
+});
+
+test("callVisionModel throws on empty response", async () => {
+ const mockResponse = {
+ ok: true,
+ json: async () => ({
+ choices: [{}],
+ }),
+ };
+ globalThis.fetch = async () => mockResponse as unknown as Response;
+
+ try {
+ const config: VisionModelConfig = {
+ model: "openai/gpt-4o-mini",
+ prompt: "Describe this image",
+ timeoutMs: 30000,
+ maxImages: 10,
+ };
+
+ await assert.rejects(
+ async () => await callVisionModel("data:image/png;base64,iVBORw0KGgo", config),
+ /empty or invalid/
+ );
+ } finally {
+ globalThis.fetch = originalFetch;
+ }
+});
+
+test("callVisionModel trims whitespace from response", async () => {
+ const mockResponse = {
+ ok: true,
+ json: async () => ({
+ choices: [{ message: { content: " A test description " } }],
+ }),
+ };
+ globalThis.fetch = async () => mockResponse as unknown as Response;
+
+ try {
+ const config: VisionModelConfig = {
+ model: "openai/gpt-4o-mini",
+ prompt: "Describe this image",
+ timeoutMs: 30000,
+ maxImages: 10,
+ };
+
+ const result = await callVisionModel("data:image/png;base64,iVBORw0KGgo", config);
+ assert.strictEqual(result, "A test description");
+ } finally {
+ globalThis.fetch = originalFetch;
+ }
+});
+
+test("callVisionModel passes custom API key", async () => {
+ let capturedHeaders: Record<string, string> = {};
+
+ const mockResponse = {
+ ok: true,
+ json: async () => ({
+ choices: [{ message: { content: "Description" } }],
+ }),
+ };
+
+ globalThis.fetch = async (url: URL | RequestInfo, init?: RequestInit) => {
+ if (init?.headers) {
+ capturedHeaders = init.headers as Record<string, string>;
+ }
+ return mockResponse as unknown as Response;
+ };
+
+ try {
+ const config: VisionModelConfig = {
+ model: "openai/gpt-4o-mini",
+ prompt: "Describe this image",
+ timeoutMs: 30000,
+ maxImages: 10,
+ };
+
+ await callVisionModel("data:image/png;base64,iVBORw0KGgo", config, "sk-custom-key");
+
+ assert.strictEqual(
+ capturedHeaders["Authorization"],
+ "Bearer sk-custom-key"
+ );
+ } finally {
+ globalThis.fetch = originalFetch;
+ }
+});
+
+test("callVisionModel uses correct request body format", async () => {
+ let capturedBody: Record<string, unknown> = {};
+
+ const mockResponse = {
+ ok: true,
+ json: async () => ({
+ choices: [{ message: { content: "Description" } }],
+ }),
+ };
+
+ globalThis.fetch = async (url: URL | RequestInfo, init?: RequestInit) => {
+ if (init?.body) {
+ capturedBody = JSON.parse(init.body as string);
+ }
+ return mockResponse as unknown as Response;
+ };
+
+ try {
+ const config: VisionModelConfig = {
+ model: "openai/gpt-4o-mini",
+ prompt: "What is in this image?",
+ timeoutMs: 30000,
+ maxImages: 10,
+ };
+
+ const imageUri = "data:image/png;base64,test123";
+ await callVisionModel(imageUri, config);
+
+ // Verify request structure
+ assert.strictEqual(capturedBody.model, "gpt-4o-mini");
+ assert.ok(Array.isArray(capturedBody.messages));
+ assert.strictEqual((capturedBody.messages as unknown[]).length, 1);
+
+ const message = (capturedBody.messages as Array<{role: string; content: unknown[]}>)[0];
+ assert.strictEqual(message.role, "user");
+ assert.ok(Array.isArray(message.content));
+ assert.strictEqual(message.content.length, 2);
+
+ // First content is image_url
+ const imagePart = message.content[0] as {type: string; image_url: {url: string; detail: string}};
+ assert.strictEqual(imagePart.type, "image_url");
+ assert.strictEqual(imagePart.image_url.url, imageUri);
+ assert.strictEqual(imagePart.image_url.detail, "low");
+
+ // Second content is text prompt
+ const textPart = message.content[1] as {type: string; text: string};
+ assert.strictEqual(textPart.type, "text");
+ assert.strictEqual(textPart.text, "What is in this image?");
+ } finally {
+ globalThis.fetch = originalFetch;
+ }
+});
diff --git a/tests/unit/guardrails/visionBridgeHelpers.extractImageParts.test.ts b/tests/unit/guardrails/visionBridgeHelpers.extractImageParts.test.ts
new file mode 100644
index 0000000000..e061548c45
--- /dev/null
+++ b/tests/unit/guardrails/visionBridgeHelpers.extractImageParts.test.ts
@@ -0,0 +1,138 @@
+/**
+ * Tests for extractImageParts helper function.
+ */
+
+import test from "node:test";
+import assert from "node:assert/strict";
+import { extractImageParts } from "@/lib/guardrails/visionBridgeHelpers";
+
+interface RequestMessage {
+ role?: string;
+ content?: string | RequestContentPart[];
+}
+
+type RequestContentPart =
+ | { type: "text"; text: string }
+ | { type: "image_url"; image_url: { url: string; detail?: string } }
+ | { type: "image"; source: { type: "base64"; media_type: string; data: string } };
+
+test("extractImageParts returns empty array for messages without images", () => {
+ const messages: RequestMessage[] = [
+ { role: "user", content: "Hello, how are you?" },
+ ];
+ const result = extractImageParts(messages);
+ assert.deepStrictEqual(result, []);
+});
+
+test("extractImageParts detects image_url format", () => {
+ const messages: RequestMessage[] = [
+ {
+ role: "user",
+ content: [
+ { type: "text", text: "What is in this image?" },
+ { type: "image_url", image_url: { url: "https://example.com/image.png" } },
+ ],
+ },
+ ];
+ const result = extractImageParts(messages);
+ assert.strictEqual(result.length, 1);
+ assert.strictEqual(result[0].messageIndex, 0);
+ assert.strictEqual(result[0].partIndex, 1);
+ assert.strictEqual(result[0].imageUrl, "https://example.com/image.png");
+ assert.strictEqual(result[0].imageType, "image_url");
+});
+
+test("extractImageParts detects base64 image format", () => {
+ const messages: RequestMessage[] = [
+ {
+ role: "user",
+ content: [
+ {
+ type: "image",
+ source: {
+ type: "base64",
+ media_type: "image/png",
+ data: "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==",
+ },
+ },
+ ],
+ },
+ ];
+ const result = extractImageParts(messages);
+ assert.strictEqual(result.length, 1);
+ assert.strictEqual(result[0].imageType, "image");
+ assert.ok(result[0].imageUrl.startsWith("data:image/png;base64,"));
+});
+
+test("extractImageParts handles multiple images in single message", () => {
+ const messages: RequestMessage[] = [
+ {
+ role: "user",
+ content: [
+ { type: "text", text: "Compare these images" },
+ { type: "image_url", image_url: { url: "https://example.com/image1.png" } },
+ { type: "image_url", image_url: { url: "https://example.com/image2.png" } },
+ ],
+ },
+ ];
+ const result = extractImageParts(messages);
+ assert.strictEqual(result.length, 2);
+ assert.strictEqual(result[0].partIndex, 1);
+ assert.strictEqual(result[1].partIndex, 2);
+});
+
+test("extractImageParts handles images across multiple messages", () => {
+ const messages: RequestMessage[] = [
+ { role: "user", content: [{ type: "image_url", image_url: { url: "https://example.com/image1.png" } }] },
+ { role: "assistant", content: "Here is analysis of the first image." },
+ { role: "user", content: [{ type: "image_url", image_url: { url: "https://example.com/image2.png" } }] },
+ ];
+ const result = extractImageParts(messages);
+ assert.strictEqual(result.length, 2);
+ assert.strictEqual(result[0].messageIndex, 0);
+ assert.strictEqual(result[1].messageIndex, 2);
+});
+
+test("extractImageParts handles empty messages array", () => {
+ const result = extractImageParts([]);
+ assert.deepStrictEqual(result, []);
+});
+
+test("extractImageParts handles messages with null/undefined content", () => {
+ const messages: RequestMessage[] = [
+ { role: "user", content: null as unknown as RequestContentPart[] },
+ { role: "user", content: undefined as unknown as RequestContentPart[] },
+ ];
+ const result = extractImageParts(messages);
+ assert.deepStrictEqual(result, []);
+});
+
+test("extractImageParts handles data URI image_url format", () => {
+ const dataUri = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==";
+ const messages: RequestMessage[] = [
+ { role: "user", content: [{ type: "image_url", image_url: { url: dataUri } }] },
+ ];
+ const result = extractImageParts(messages);
+ assert.strictEqual(result.length, 1);
+ assert.strictEqual(result[0].imageUrl, dataUri);
+});
+
+test("extractImageParts preserves order of images", () => {
+ const messages: RequestMessage[] = [
+ {
+ role: "user",
+ content: [
+ { type: "text", text: "First" },
+ { type: "image_url", image_url: { url: "https://example.com/A.png" } },
+ { type: "text", text: "Second" },
+ { type: "image_url", image_url: { url: "https://example.com/B.png" } },
+ { type: "image_url", image_url: { url: "https://example.com/C.png" } },
+ ],
+ },
+ ];
+ const result = extractImageParts(messages);
+ assert.strictEqual(result.length, 3);
+ assert.strictEqual(result[0].partIndex, 1);
+ assert.strictEqual(result[1].partIndex, 3);
+ assert.strictEqual(result[2].partIndex, 4);
+});
diff --git a/tests/unit/guardrails/visionBridgeHelpers.replaceImageParts.test.ts b/tests/unit/guardrails/visionBridgeHelpers.replaceImageParts.test.ts
new file mode 100644
index 0000000000..263e3dbfb3
--- /dev/null
+++ b/tests/unit/guardrails/visionBridgeHelpers.replaceImageParts.test.ts
@@ -0,0 +1,254 @@
+/**
+ * Tests for replaceImageParts helper function.
+ */
+
+import test from "node:test";
+import assert from "node:assert/strict";
+import { replaceImageParts } from "@/lib/guardrails/visionBridgeHelpers";
+
+test("replaceImageParts replaces single image with description", () => {
+ const body = {
+ model: "minimax/minimax-01",
+ messages: [
+ {
+ role: "user",
+ content: [
+ { type: "text", text: "What is in this image?" },
+ { type: "image_url", image_url: { url: "https://example.com/image.png" } },
+ ],
+ },
+ ],
+ };
+
+ const descriptions = ["[Image 1]: A beautiful sunset"];
+
+ const result = replaceImageParts(body, descriptions);
+
+ // Should have original text preserved
+ const content = result.messages[0].content as Array<{type: string; text?: string}>;
+ assert.strictEqual(content[0].type, "text");
+ assert.strictEqual(content[0].text, "What is in this image?");
+
+ // Should have description instead of image
+ assert.strictEqual(content[1].type, "text");
+ assert.strictEqual(content[1].text, "[Image 1]: A beautiful sunset");
+});
+
+test("replaceImageParts replaces multiple images with descriptions", () => {
+ const body = {
+ model: "minimax/minimax-01",
+ messages: [
+ {
+ role: "user",
+ content: [
+ { type: "text", text: "Compare these images" },
+ { type: "image_url", image_url: { url: "https://example.com/A.png" } },
+ { type: "image_url", image_url: { url: "https://example.com/B.png" } },
+ { type: "image_url", image_url: { url: "https://example.com/C.png" } },
+ ],
+ },
+ ],
+ };
+
+ const descriptions = [
+ "[Image 1]: A cat",
+ "[Image 2]: A dog",
+ "[Image 3]: A bird",
+ ];
+
+ const result = replaceImageParts(body, descriptions);
+
+ const content = result.messages[0].content as Array<{type: string; text?: string}>;
+ assert.strictEqual(content[0].type, "text");
+ assert.strictEqual(content[0].text, "Compare these images");
+ assert.strictEqual(content[1].type, "text");
+ assert.strictEqual(content[1].text, "[Image 1]: A cat");
+ assert.strictEqual(content[2].type, "text");
+ assert.strictEqual(content[2].text, "[Image 2]: A dog");
+ assert.strictEqual(content[3].type, "text");
+ assert.strictEqual(content[3].text, "[Image 3]: A bird");
+});
+
+test("replaceImageParts handles empty descriptions array", () => {
+ const body = {
+ model: "minimax/minimax-01",
+ messages: [
+ {
+ role: "user",
+ content: [
+ { type: "text", text: "Hello" },
+ { type: "image_url", image_url: { url: "https://example.com/image.png" } },
+ ],
+ },
+ ],
+ };
+
+ const result = replaceImageParts(body, []);
+
+ // Original should be unchanged
+ const content = result.messages[0].content as Array<{type: string}>;
+ assert.strictEqual(content[1].type, "image_url");
+});
+
+test("replaceImageParts preserves non-image content", () => {
+ const body = {
+ model: "minimax/minimax-01",
+ messages: [
+ {
+ role: "system",
+ content: "You are a helpful assistant.",
+ },
+ {
+ role: "user",
+ content: [
+ { type: "text", text: "Analyze this" },
+ { type: "image_url", image_url: { url: "https://example.com/image.png" } },
+ ],
+ },
+ {
+ role: "assistant",
+ content: "I can see the image shows a sunset.",
+ },
+ ],
+ };
+
+ const result = replaceImageParts(body, ["[Image 1]: A sunset over the ocean"]);
+
+ // System message should be unchanged
+ assert.strictEqual(result.messages[0].content, "You are a helpful assistant.");
+
+ // Assistant message should be unchanged
+ assert.strictEqual(result.messages[2].content, "I can see the image shows a sunset.");
+});
+
+test("replaceImageParts handles base64 images", () => {
+ const body = {
+ model: "minimax/minimax-01",
+ messages: [
+ {
+ role: "user",
+ content: [
+ {
+ type: "image",
+ source: {
+ type: "base64",
+ media_type: "image/png",
+ data: "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==",
+ },
+ },
+ ],
+ },
+ ],
+ };
+
+ const descriptions = ["[Image 1]: A red circle"];
+
+ const result = replaceImageParts(body, descriptions);
+
+ const content = result.messages[0].content as Array<{type: string; text?: string}>;
+ assert.strictEqual(content[0].type, "text");
+ assert.strictEqual(content[0].text, "[Image 1]: A red circle");
+});
+
+test("replaceImageParts handles undefined descriptions", () => {
+ const body = {
+ model: "test",
+ messages: [
+ {
+ role: "user",
+ content: [
+ { type: "text", text: "Hello" },
+ { type: "image_url", image_url: { url: "https://example.com/image.png" } },
+ ],
+ },
+ ],
+ };
+
+ const result = replaceImageParts(body, undefined as unknown as string[]);
+
+ // Should return original body when descriptions is undefined
+ const content = result.messages[0].content as Array<{type: string}>;
+ assert.strictEqual(content[1].type, "image_url");
+});
+
+test("replaceImageParts handles empty messages array", () => {
+ const body = {
+ model: "test",
+ messages: [],
+ };
+
+ const descriptions = ["[Image 1]: Description"];
+ const result = replaceImageParts(body, descriptions);
+
+ assert.deepStrictEqual(result.messages, []);
+});
+
+test("replaceImageParts handles messages without content array", () => {
+ const body = {
+ model: "test",
+ messages: [
+ { role: "user", content: "Just a text message" },
+ { role: "user", content: [{ type: "image_url", image_url: { url: "https://example.com/image.png" } }] },
+ ],
+ };
+
+ const descriptions = ["[Image 1]: Description"];
+ const result = replaceImageParts(body, descriptions);
+
+ // First message (string content) should be unchanged
+ assert.strictEqual(result.messages[0].content, "Just a text message");
+
+ // Second message should have image replaced
+ const content = result.messages[1].content as Array<{type: string; text?: string}>;
+ assert.strictEqual(content[0].type, "text");
+ assert.strictEqual(content[0].text, "[Image 1]: Description");
+});
+
+test("replaceImageParts does not modify original body", () => {
+ const body = {
+ model: "minimax/minimax-01",
+ messages: [
+ {
+ role: "user",
+ content: [
+ { type: "text", text: "Original" },
+ { type: "image_url", image_url: { url: "https://example.com/image.png" } },
+ ],
+ },
+ ],
+ };
+
+ const descriptions = ["[Image 1]: Modified"];
+ replaceImageParts(body, descriptions);
+
+ // Original should be unchanged
+ const content = body.messages[0].content as Array<{type: string}>;
+ assert.strictEqual(content[1].type, "image_url");
+});
+
+test("replaceImageParts handles mixed images and text", () => {
+ const body = {
+ model: "test",
+ messages: [
+ {
+ role: "user",
+ content: [
+ { type: "image_url", image_url: { url: "https://example.com/first.png" } },
+ { type: "text", text: "between images" },
+ { type: "image_url", image_url: { url: "https://example.com/second.png" } },
+ ],
+ },
+ ],
+ };
+
+ const descriptions = ["[Image 1]: First image", "[Image 2]: Second image"];
+ const result = replaceImageParts(body, descriptions);
+
+ const content = result.messages[0].content as Array<{type: string; text?: string}>;
+ assert.strictEqual(content[0].type, "text");
+ assert.strictEqual(content[0].text, "[Image 1]: First image");
+ assert.strictEqual(content[1].type, "text");
+ assert.strictEqual(content[1].text, "between images");
+ assert.strictEqual(content[2].type, "text");
+ assert.strictEqual(content[2].text, "[Image 2]: Second image");
+});
diff --git a/tests/unit/guardrails/visionBridgeHelpers.resolveImageAsDataUri.test.ts b/tests/unit/guardrails/visionBridgeHelpers.resolveImageAsDataUri.test.ts
new file mode 100644
index 0000000000..af47178c88
--- /dev/null
+++ b/tests/unit/guardrails/visionBridgeHelpers.resolveImageAsDataUri.test.ts
@@ -0,0 +1,62 @@
+/**
+ * Tests for resolveImageAsDataUri helper function.
+ */
+
+import test from "node:test";
+import assert from "node:assert/strict";
+import { resolveImageAsDataUri } from "@/lib/guardrails/visionBridgeHelpers";
+
+test("resolveImageAsDataUri passes through HTTPS URL as-is", () => {
+ const url = "https://example.com/image.png";
+ const result = resolveImageAsDataUri(url);
+ assert.strictEqual(result, url);
+});
+
+test("resolveImageAsDataUri passes through HTTP URL as-is", () => {
+ const url = "http://example.com/image.png";
+ const result = resolveImageAsDataUri(url);
+ assert.strictEqual(result, url);
+});
+
+test("resolveImageAsDataUri passes through data URI as-is", () => {
+ const dataUri = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==";
+ const result = resolveImageAsDataUri(dataUri);
+ assert.strictEqual(result, dataUri);
+});
+
+test("resolveImageAsDataUri converts base64 string to PNG data URI", () => {
+ const base64 = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==";
+ const result = resolveImageAsDataUri(base64);
+ assert.strictEqual(result, `data:image/png;base64,${base64}`);
+});
+
+test("resolveImageAsDataUri throws for empty string", () => {
+ assert.throws(() => {
+ resolveImageAsDataUri("");
+ }, /Invalid image URL/);
+});
+
+test("resolveImageAsDataUri throws for null", () => {
+ assert.throws(() => {
+ resolveImageAsDataUri(null as unknown as string);
+ }, /Invalid image URL/);
+});
+
+test("resolveImageAsDataUri throws for undefined", () => {
+ assert.throws(() => {
+ resolveImageAsDataUri(undefined as unknown as string);
+ }, /Invalid image URL/);
+});
+
+test("resolveImageAsDataUri handles data URI with different media types", () => {
+ const jpegUri = "data:image/jpeg;base64,/9j/4AAQSkZJRg==";
+ const result = resolveImageAsDataUri(jpegUri);
+ assert.strictEqual(result, jpegUri);
+});
+
+test("resolveImageAsDataUri treats plain base64 with prefix as data URI", () => {
+ // If it doesn't start with data: or http, treat as raw base64
+ const rawBase64 = "abcdef123456==";
+ const result = resolveImageAsDataUri(rawBase64);
+ assert.strictEqual(result, "data:image/png;base64,abcdef123456==");
+});
diff --git a/tests/unit/visionBridgeDefaults.test.ts b/tests/unit/visionBridgeDefaults.test.ts
new file mode 100644
index 0000000000..2d631e05de
--- /dev/null
+++ b/tests/unit/visionBridgeDefaults.test.ts
@@ -0,0 +1,91 @@
+/**
+ * Tests for Vision Bridge default constants.
+ */
+
+import test from "node:test";
+import assert from "node:assert/strict";
+import {
+ VISION_BRIDGE_DEFAULTS,
+ VISION_BRIDGE_SETTINGS_KEYS,
+ getVisionBridgeConfig,
+ type VisionBridgeSettings,
+} from "@/shared/constants/visionBridgeDefaults";
+
+test("VISION_BRIDGE_DEFAULTS exports correct values", () => {
+ assert.strictEqual(VISION_BRIDGE_DEFAULTS.enabled, true);
+ assert.strictEqual(VISION_BRIDGE_DEFAULTS.model, "openai/gpt-4o-mini");
+ assert.strictEqual(
+ VISION_BRIDGE_DEFAULTS.prompt,
+ "Describe this image concisely in 2-3 sentences. Focus on the most relevant visual details."
+ );
+ assert.strictEqual(VISION_BRIDGE_DEFAULTS.timeoutMs, 30000);
+ assert.strictEqual(VISION_BRIDGE_DEFAULTS.maxImagesPerRequest, 10);
+});
+
+test("VISION_BRIDGE_SETTINGS_KEYS exports all expected keys", () => {
+ assert.deepStrictEqual(VISION_BRIDGE_SETTINGS_KEYS, [
+ "visionBridgeEnabled",
+ "visionBridgeModel",
+ "visionBridgePrompt",
+ "visionBridgeTimeout",
+ "visionBridgeMaxImages",
+ ]);
+});
+
+test("getVisionBridgeConfig returns defaults when no settings provided", () => {
+ const config = getVisionBridgeConfig({});
+
+ assert.strictEqual(config.enabled, true);
+ assert.strictEqual(config.model, "openai/gpt-4o-mini");
+ assert.strictEqual(config.prompt, VISION_BRIDGE_DEFAULTS.prompt);
+ assert.strictEqual(config.timeoutMs, 30000);
+ assert.strictEqual(config.maxImages, 10);
+});
+
+test("getVisionBridgeConfig applies custom settings", () => {
+ const customSettings: VisionBridgeSettings = {
+ visionBridgeEnabled: false,
+ visionBridgeModel: "anthropic/claude-3-haiku",
+ visionBridgePrompt: "What is in this image?",
+ visionBridgeTimeout: 60000,
+ visionBridgeMaxImages: 5,
+ };
+
+ const config = getVisionBridgeConfig(customSettings);
+
+ assert.strictEqual(config.enabled, false);
+ assert.strictEqual(config.model, "anthropic/claude-3-haiku");
+ assert.strictEqual(config.prompt, "What is in this image?");
+ assert.strictEqual(config.timeoutMs, 60000);
+ assert.strictEqual(config.maxImages, 5);
+});
+
+test("getVisionBridgeConfig merges partial settings with defaults", () => {
+ const partialSettings: VisionBridgeSettings = {
+ visionBridgeModel: "openai/gpt-4o",
+ };
+
+ const config = getVisionBridgeConfig(partialSettings);
+
+ // Custom value
+ assert.strictEqual(config.model, "openai/gpt-4o");
+ // Default values for the rest
+ assert.strictEqual(config.enabled, true);
+ assert.strictEqual(config.prompt, VISION_BRIDGE_DEFAULTS.prompt);
+ assert.strictEqual(config.timeoutMs, 30000);
+ assert.strictEqual(config.maxImages, 10);
+});
+
+test("getVisionBridgeConfig handles undefined settings", () => {
+ const config = getVisionBridgeConfig(undefined);
+
+ assert.strictEqual(config.enabled, true);
+ assert.strictEqual(config.model, "openai/gpt-4o-mini");
+});
+
+test("getVisionBridgeConfig handles null settings", () => {
+ const config = getVisionBridgeConfig(null as unknown as VisionBridgeSettings);
+
+ assert.strictEqual(config.enabled, true);
+ assert.strictEqual(config.model, "openai/gpt-4o-mini");
+});