Never stop coding. Free MIT AI gateway: one endpoint, 375 providers (150+ free), 1,400+ models Kimi, Claude, GPT, Gemini, GLM, DeepSeek, MiniMax. Works with Claude Code, Codex, Cursor, OpenCode, Cline & Copilot. Quota-aware auto-fallback, RTK+Caveman compression saves 15-95% tokens, MCP/A2A, Desktop/PWA. Built by 900+ contributors
TypeScript a2aai-agentsai-gatewayanthropicclaudeclaude-codeclinecodexcopilotcursordeepseekfree-aigeminikimillm-gatewaymcpopenaiopenai-proxyqwentoken-saver
feat(vision-bridge): add automatic image description fallback for non-vision models (#1476)
Hernan Javier Ardila Sanchezb709dca2d6d8
diff --git a/src/lib/guardrails/registry.ts b/src/lib/guardrails/registry.tsindex 24ed1dc156..b90cd2147e 100644--- a/src/lib/guardrails/registry.ts+++ b/src/lib/guardrails/registry.ts@@ -1,6 +1,7 @@import { BaseGuardrail, type GuardrailContext, type GuardrailExecutionResult } from "./base";import { PIIMaskerGuardrail } from "./piiMasker";import { PromptInjectionGuardrail } from "./promptInjection";+import { VisionBridgeGuardrail } from "./visionBridge";type HeadersLike = Headers | Record<string, unknown> | null | undefined;@@ -262,6 +263,7 @@ let defaultGuardrailsRegistered = false;export function registerDefaultGuardrails() {if (defaultGuardrailsRegistered) return guardrailRegistry;+ guardrailRegistry.register(new VisionBridgeGuardrail());guardrailRegistry.register(new PIIMaskerGuardrail());guardrailRegistry.register(new PromptInjectionGuardrail());defaultGuardrailsRegistered = true;diff --git a/src/lib/guardrails/visionBridge.ts b/src/lib/guardrails/visionBridge.tsnew file mode 100644index 0000000000..f9bcac35ca--- /dev/null+++ b/src/lib/guardrails/visionBridge.ts@@ -0,0 +1,150 @@+/**+ * Vision Bridge Guardrail.+ * Intercepts image-bearing requests to non-vision models,+ * extracts descriptions via vision model, and replaces images with text.+ */++import { BaseGuardrail, type GuardrailContext, type GuardrailResult } from "./base";+import { getSettings as defaultGetSettings } from "@/lib/db/settings";+import { getResolvedModelCapabilities } from "@/lib/modelCapabilities";+import {+ extractImageParts,+ callVisionModel as defaultCallVisionModel,+ replaceImageParts,+} from "./visionBridgeHelpers";+import {+ VISION_BRIDGE_DEFAULTS,+ getVisionBridgeConfig,+} from "@/shared/constants/visionBridgeDefaults";++export interface VisionBridgeDependencies {+ getSettings?: () => Promise<Record<string, unknown>>;+ callVisionModel?: (+ imageDataUri: string,+ config: import("./visionBridgeHelpers").VisionModelConfig,+ apiKey?: string+ ) => Promise<string>;+}++export class VisionBridgeGuardrail extends BaseGuardrail {+ name = "vision-bridge";+ priority = 5;++ private readonly deps: VisionBridgeDependencies;++ constructor(options?: { enabled?: boolean; deps?: VisionBridgeDependencies }) {+ super("vision-bridge", { priority: 5, enabled: options?.enabled });+ this.deps = options?.deps ?? {};+ }++ async preCall(+ payload: unknown,+ context: GuardrailContext+ ): Promise<GuardrailResult<unknown>> {+ // 1. Check if disabled at guardrail level+ if (!this.enabled) {+ return { block: false };+ }++ // 2. Check disabled via context (header, body, API key)+ if (context.disabledGuardrails?.includes("vision-bridge")) {+ return { block: false };+ }++ // 3. Get model from context or payload+ const model =+ context.model || (payload as Record<string, unknown>)?.model as string | undefined;+ if (!model) {+ return { block: false };+ }++ // 4. Check if model supports vision+ const capabilities = getResolvedModelCapabilities(model);+ if (capabilities.supportsVision === true) {+ return { block: false };+ }++ // 5. Get body and check for messages+ const body = payload as Record<string, unknown>;+ const messages = body?.messages;+ if (!Array.isArray(messages) || messages.length === 0) {+ return { block: false };+ }++ // 6. Check for images using helper (extractImageParts returns empty if no images)+ const imageParts = extractImageParts(+ messages as Parameters<typeof extractImageParts>[0]+ );+ if (imageParts.length === 0) {+ return { block: false };+ }++ // 7. Get settings (injectable for testing)+ const getSettings = this.deps.getSettings ?? defaultGetSettings;+ let settings: Record<string, unknown> = {};+ try {+ settings = await getSettings();+ } catch {+ // If getSettings fails, use defaults+ }++ // 8. Check if Vision Bridge is enabled in settings+ const enabled = settings.visionBridgeEnabled ?? VISION_BRIDGE_DEFAULTS.enabled;+ if (!enabled) {+ return { block: false };+ }++ // 9. Get configuration+ const config = getVisionBridgeConfig({+ visionBridgeEnabled: settings.visionBridgeEnabled as boolean | undefined,+ visionBridgeModel: settings.visionBridgeModel as string | undefined,+ visionBridgePrompt: settings.visionBridgePrompt as string | undefined,+ visionBridgeTimeout: settings.visionBridgeTimeout as number | undefined,+ visionBridgeMaxImages: settings.visionBridgeMaxImages as number | undefined,+ });++ // 10. Limit images+ const limitedParts = imageParts.slice(0, config.maxImages);++ // 11. Call vision model for each image in parallel (injectable for testing)+ const callVision = this.deps.callVisionModel ?? defaultCallVisionModel;+ const logger = context.log;+ const startTime = Date.now();++ // Process all images in parallel using Promise.allSettled for fail-partial behavior+ const results = await Promise.allSettled(+ limitedParts.map(async (imagePart, i) => {+ const description = await callVision(imagePart.imageUrl, config);+ return `[Image ${i + 1}]: ${description}`;+ })+ );++ // Collect descriptions maintaining original order+ const descriptions = results.map((result, i) => {+ if (result.status === "fulfilled") {+ return result.value;+ }+ const message = result.reason instanceof Error ? result.reason.message : String(result.reason);+ logger?.warn?.("VISION-BRIDGE", `Failed to get description for image ${i + 1}: ${message}`);+ return `[Image ${i + 1}]: (unavailable)`;+ });++ // 12. Replace image parts with text descriptions+ const modifiedBody = replaceImageParts(+ body as Parameters<typeof replaceImageParts>[0],+ descriptions+ );+ const processingTime = Date.now() - startTime;++ return {+ block: false,+ modifiedPayload: modifiedBody,+ meta: {+ imagesProcessed: descriptions.length,+ descriptions,+ processingTimeMs: processingTime,+ visionModel: config.model,+ },+ };+ }+}\ No newline at end of filediff --git a/src/lib/guardrails/visionBridgeHelpers.ts b/src/lib/guardrails/visionBridgeHelpers.tsnew file mode 100644index 0000000000..c0d28376b6--- /dev/null+++ b/src/lib/guardrails/visionBridgeHelpers.ts@@ -0,0 +1,330 @@+/**+ * Vision Bridge helper functions for image processing.+ */++/**+ * Provider to environment variable mapping for API key resolution.+ */+const PROVIDER_API_KEY_MAP: Record<string, string> = {+ anthropic: "ANTHROPIC_API_KEY",+ google: "GOOGLE_API_KEY",+ openai: "OPENAI_API_KEY",+};++/**+ * Resolve API key based on model provider.+ * @param model - Model identifier (e.g., "anthropic/claude-3-haiku", "openai/gpt-4o-mini")+ * @param explicitKey - Explicit API key passed as argument (takes precedence)+ * @returns Resolved API key string+ */+export function resolveProviderApiKey(model: string, explicitKey?: string): string {+ if (explicitKey) return explicitKey;+ const provider = model.includes("/") ? model.split("/")[0] : "";+ const envVar = PROVIDER_API_KEY_MAP[provider] || "OPENAI_API_KEY";+ return process.env[envVar] || "";+}++export interface ImagePart {+ messageIndex: number;+ partIndex: number;+ imageUrl: string;+ imageType: "image_url" | "image";+}++export interface RequestMessage {+ role?: string;+ content?: string | RequestContentPart[];+}++export type RequestContentPart =+ | { type: "text"; text: string }+ | { type: "image_url"; image_url: { url: string; detail?: string } }+ | { type: "image"; source: { type: "base64"; media_type: string; data: string } };++/**+ * Extract image parts from messages array.+ * Supports both OpenAI image_url format and base64 image format.+ */+export function extractImageParts(messages: RequestMessage[]): ImagePart[] {+ const results: ImagePart[] = [];++ if (!Array.isArray(messages)) {+ return results;+ }++ for (let msgIdx = 0; msgIdx < messages.length; msgIdx++) {+ const message = messages[msgIdx];+ if (!message || !Array.isArray(message.content)) {+ continue;+ }++ for (let partIdx = 0; partIdx < message.content.length; partIdx++) {+ const part = message.content[partIdx];++ if (part?.type === "image_url" && part.image_url?.url) {+ results.push({+ messageIndex: msgIdx,+ partIndex: partIdx,+ imageUrl: part.image_url.url,+ imageType: "image_url",+ });+ } else if (part?.type === "image" && part.source?.type === "base64") {+ const { media_type, data } = part.source;+ const dataUri = `data:${media_type};base64,${data}`;+ results.push({+ messageIndex: msgIdx,+ partIndex: partIdx,+ imageUrl: dataUri,+ imageType: "image",+ });+ }+ }+ }++ return results;+}++/**+ * Resolve image URL to data URI format for vision model.+ * - HTTP/HTTPS URLs: passed through as-is+ * - Data URIs: passed through as-is+ * - Base64 without media type: assumed PNG+ */+export function resolveImageAsDataUri(imageUrl: string): string {+ if (!imageUrl || typeof imageUrl !== "string") {+ throw new Error("Invalid image URL: must be a non-empty string");+ }++ // Already a data URI+ if (imageUrl.startsWith("data:")) {+ return imageUrl;+ }++ // HTTP/HTTPS URL - vision API will fetch it+ if (imageUrl.startsWith("http://") || imageUrl.startsWith("https://")) {+ return imageUrl;+ }++ // Assume it's a base64 string without prefix+ // Add PNG as default media type+ return `data:image/png;base64,${imageUrl}`;+}++export interface VisionModelConfig {+ model: string;+ prompt: string;+ timeoutMs: number;+ maxImages: number;+}++/**+ * Call the vision model to get an image description.+ * Supports both OpenAI-compatible and Anthropic API formats.+ */+export async function callVisionModel(+ imageDataUri: string,+ config: VisionModelConfig,+ apiKey?: string+): Promise<string> {+ const controller = new AbortController();+ const timeoutId = setTimeout(() => controller.abort(), config.timeoutMs);++ // Resolve API key based on provider+ const resolvedApiKey = resolveProviderApiKey(config.model, apiKey);++ // Detect provider from model identifier+ const isAnthropic = config.model.startsWith("anthropic/");++ try {+ // Extract model name from provider/model format+ const modelName = config.model.includes("/")+ ? config.model.split("/")[1]+ : config.model;++ let response: Response;++ if (isAnthropic) {+ // Anthropic API path+ const anthropicBaseUrl = process.env.ANTHROPIC_API_URL || "https://api.anthropic.com";++ // Parse data URI to extract media type and base64 data+ const matches = imageDataUri.match(/^data:([^;]+);base64,(.+)$/);+ let mediaType = "image/png";+ let base64Data = imageDataUri;++ if (matches) {+ mediaType = matches[1];+ base64Data = matches[2];+ }++ response = await fetch(`${anthropicBaseUrl}/v1/messages`, {+ method: "POST",+ signal: controller.signal,+ headers: {+ "x-api-key": resolvedApiKey,+ "anthropic-version": "2023-06-01",+ "content-type": "application/json",+ },+ body: JSON.stringify({+ model: modelName,+ messages: [+ {+ role: "user",+ content: [+ {+ type: "image",+ source: {+ type: "base64",+ media_type: mediaType,+ data: base64Data,+ },+ },+ {+ type: "text",+ text: config.prompt,+ },+ ],+ },+ ],+ max_tokens: 300,+ }),+ });+ } else {+ // OpenAI-compatible path (default)+ const baseUrl = process.env.OPENAI_API_URL || "https://api.openai.com/v1";++ response = await fetch(`${baseUrl}/chat/completions`, {+ method: "POST",+ signal: controller.signal,+ headers: {+ "Content-Type": "application/json",+ Authorization: `Bearer ${resolvedApiKey}`,+ },+ body: JSON.stringify({+ model: modelName,+ messages: [+ {+ role: "user",+ content: [+ {+ type: "image_url",+ image_url: {+ url: imageDataUri,+ detail: "low",+ },+ },+ { type: "text", text: config.prompt },+ ],+ },+ ],+ max_tokens: 300,+ }),+ });+ }++ clearTimeout(timeoutId);++ if (!response.ok) {+ const errorText = await response.text().catch(() => "Unknown error");+ throw new Error(`Vision API error ${response.status}: ${errorText}`);+ }++ const data = await response.json();++ if (isAnthropic) {+ // Anthropic response format: { content: [{ type: "text", text: "..." }] }+ const anthropicData = data as {+ content?: Array<{ type?: string; text?: string }>;+ error?: { message?: string };+ };++ if (anthropicData.error) {+ throw new Error(`Vision API error: ${anthropicData.error.message || JSON.stringify(anthropicData.error)}`);+ }++ const textContent = anthropicData.content?.find((c) => c.type === "text");+ const content = textContent?.text;+ if (!content || typeof content !== "string") {+ throw new Error("Vision API returned empty or invalid response");+ }++ return content.trim();+ } else {+ // OpenAI-compatible response format: { choices: [{ message: { content: "..." } }] }+ const openaiData = data as {+ choices?: Array<{ message?: { content?: string } }>;+ error?: { message?: string };+ };++ if (openaiData.error) {+ throw new Error(`Vision API error: ${openaiData.error.message || JSON.stringify(openaiData.error)}`);+ }++ const content = openaiData.choices?.[0]?.message?.content;+ if (!content || typeof content !== "string") {+ throw new Error("Vision API returned empty or invalid response");+ }++ return content.trim();+ }+ } catch (error) {+ clearTimeout(timeoutId);++ if (error instanceof Error && error.name === "AbortError") {+ throw new Error("Vision model call timed out");+ }++ throw error;+ }+}++export interface RequestBody {+ model?: string;+ messages?: RequestMessage[];+ [key: string]: unknown;+}++/**+ * Replace image content parts with text descriptions.+ * Concatenates descriptions with labels: "[Image 1]: ..."+ */+export function replaceImageParts(body: RequestBody, descriptions: string[]): RequestBody {+ if (!descriptions || descriptions.length === 0) {+ return body;+ }++ const result = structuredClone(body) as RequestBody;++ if (!Array.isArray(result.messages)) {+ return result;+ }++ let descriptionIndex = 0;++ for (let msgIdx = 0; msgIdx < result.messages.length; msgIdx++) {+ const message = result.messages[msgIdx];+ if (!message || !Array.isArray(message.content)) {+ continue;+ }++ const newContent: RequestContentPart[] = [];++ for (const part of message.content) {+ if (part?.type === "image_url" || part?.type === "image") {+ if (descriptionIndex < descriptions.length) {+ newContent.push({+ type: "text",+ text: descriptions[descriptionIndex],+ });+ descriptionIndex++;+ }+ } else {+ newContent.push(part as RequestContentPart);+ }+ }++ message.content = newContent;+ }++ return result;+}diff --git a/src/shared/constants/visionBridgeDefaults.ts b/src/shared/constants/visionBridgeDefaults.tsnew file mode 100644index 0000000000..18e069d6d3--- /dev/null+++ b/src/shared/constants/visionBridgeDefaults.ts@@ -0,0 +1,55 @@+/**+ * Vision Bridge default configuration values.+ */++export const VISION_BRIDGE_DEFAULTS = {+ enabled: true,+ model: "openai/gpt-4o-mini",+ prompt:+ "Describe this image concisely in 2-3 sentences. Focus on the most relevant visual details.",+ timeoutMs: 30000,+ maxImagesPerRequest: 10,+} as const;++/**+ * Settings keys for Vision Bridge (to be stored in key_value table).+ */+export const VISION_BRIDGE_SETTINGS_KEYS = [+ "visionBridgeEnabled",+ "visionBridgeModel",+ "visionBridgePrompt",+ "visionBridgeTimeout",+ "visionBridgeMaxImages",+] as const;++export type VisionBridgeSettings = {+ visionBridgeEnabled?: boolean;+ visionBridgeModel?: string;+ visionBridgePrompt?: string;+ visionBridgeTimeout?: number;+ visionBridgeMaxImages?: number;+};++export type VisionBridgeConfig = {+ enabled: boolean;+ model: string;+ prompt: string;+ timeoutMs: number;+ maxImages: number;+};++/**+ * Merge settings with defaults to produce a complete config.+ */+export function getVisionBridgeConfig(+ settings: VisionBridgeSettings | undefined | null = {}+): VisionBridgeConfig {+ const s = settings ?? {};+ return {+ enabled: s.visionBridgeEnabled ?? VISION_BRIDGE_DEFAULTS.enabled,+ model: s.visionBridgeModel ?? VISION_BRIDGE_DEFAULTS.model,+ prompt: s.visionBridgePrompt ?? VISION_BRIDGE_DEFAULTS.prompt,+ timeoutMs: s.visionBridgeTimeout ?? VISION_BRIDGE_DEFAULTS.timeoutMs,+ maxImages: s.visionBridgeMaxImages ?? VISION_BRIDGE_DEFAULTS.maxImagesPerRequest,+ };+}\ No newline at end of filediff --git a/tests/unit/guardrails/visionBridge.test.ts b/tests/unit/guardrails/visionBridge.test.tsnew file mode 100644index 0000000000..550dc63469--- /dev/null+++ b/tests/unit/guardrails/visionBridge.test.ts@@ -0,0 +1,515 @@+/**+ * Tests for VisionBridgeGuardrail.+ * Uses dependency injection to avoid SQLite dependency.+ */++import test from "node:test";+import assert from "node:assert/strict";++const { VisionBridgeGuardrail } = await import(+ "../../../src/lib/guardrails/visionBridge.ts"+);+const { resetGuardrailsForTests } = await import(+ "../../../src/lib/guardrails/registry.ts"+);+const { getResolvedModelCapabilities } = await import(+ "../../../src/lib/modelCapabilities.ts"+);+import type { GuardrailContext } from "../../../src/lib/guardrails/base.ts";+import type { VisionModelConfig } from "../../../src/lib/guardrails/visionBridgeHelpers.ts";++// ── Mock state ──────────────────────────────────────────────────────────────++let mockSettings: Record<string, unknown> = {+ visionBridgeEnabled: true,+ visionBridgeModel: "openai/gpt-4o-mini",+ visionBridgePrompt: "Describe this image concisely.",+ visionBridgeTimeout: 30000,+ visionBridgeMaxImages: 10,+};++let mockVisionResponse = "A beautiful sunset over the ocean";+let shouldVisionFail = false;+let visionCallCount = 0;++function createGuardrail(+ options?: Parameters<typeof VisionBridgeGuardrail>[0]+) {+ return new VisionBridgeGuardrail({+ ...options,+ deps: {+ getSettings: async () => mockSettings,+ callVisionModel: async (+ _imageDataUri: string,+ _config: VisionModelConfig+ ) => {+ visionCallCount++;+ if (shouldVisionFail) {+ throw new Error("Vision model failed");+ }+ return mockVisionResponse;+ },+ },+ });+}++test.beforeEach(() => {+ resetGuardrailsForTests({ registerDefaults: false });+ visionCallCount = 0;+ shouldVisionFail = false;+ mockSettings = {+ visionBridgeEnabled: true,+ visionBridgeModel: "openai/gpt-4o-mini",+ visionBridgePrompt: "Describe this image concisely.",+ visionBridgeTimeout: 30000,+ visionBridgeMaxImages: 10,+ };+});++// ── Helpers ─────────────────────────────────────────────────────────────────++function createContext(+ overrides: Partial<GuardrailContext> = {}+): GuardrailContext {+ return {+ model: "minimax/minimax-01",+ log: console,+ ...overrides,+ };+}++function createPayload(+ overrides: Record<string, unknown> = {}+): Record<string, unknown> {+ return {+ model: "minimax/minimax-01",+ messages: [{ role: "user", content: "Hello" }],+ ...overrides,+ };+}++// ── Basic Properties ────────────────────────────────────────────────────────++test("VisionBridgeGuardrail has correct name and priority", () => {+ const guardrail = createGuardrail();+ assert.strictEqual(guardrail.name, "vision-bridge");+ assert.strictEqual(guardrail.priority, 5);+});++test("VisionBridgeGuardrail is enabled by default", () => {+ const guardrail = createGuardrail();+ assert.strictEqual(guardrail.enabled, true);+});++test("VisionBridgeGuardrail can be disabled via constructor", () => {+ const guardrail = createGuardrail({ enabled: false });+ assert.strictEqual(guardrail.enabled, false);+});++// ── VB-S05: Vision Bridge disabled via settings ────────────────────────────++test("VB-S05: passthroughs when visionBridgeEnabled is false", async () => {+ mockSettings.visionBridgeEnabled = false;+ const guardrail = createGuardrail();++ const payload = createPayload({+ messages: [+ {+ role: "user",+ content: [+ { type: "text", text: "What is in this image?" },+ {+ type: "image_url",+ image_url: { url: "https://example.com/image.png" },+ },+ ],+ },+ ],+ });++ const result = await guardrail.preCall(payload, createContext());+ assert.strictEqual(result.block, false);+ assert.strictEqual(result.modifiedPayload, undefined);+});++// ── VB-S06: Disabled via context ────────────────────────────────────────────++test("VB-S06: skips when disabledGuardrails includes vision-bridge", async () => {+ const guardrail = createGuardrail();+ const payload = createPayload();+ const context = createContext({ disabledGuardrails: ["vision-bridge"] });++ const result = await guardrail.preCall(payload, context);+ assert.strictEqual(result.block, false);+ assert.strictEqual(result.modifiedPayload, undefined);+});++// ── VB-S02: Vision-capable model passthrough ────────────────────────────────++test("VB-S02: passthroughs for vision-capable model (gpt-4o)", async () => {+ const guardrail = createGuardrail();+ const payload = createPayload({+ model: "openai/gpt-4o",+ messages: [+ {+ role: "user",+ content: [+ { type: "text", text: "What is this?" },+ {+ type: "image_url",+ image_url: { url: "https://example.com/image.png" },+ },+ ],+ },+ ],+ });++ const result = await guardrail.preCall(+ payload,+ createContext({ model: "openai/gpt-4o" })+ );++ // If supportsVision is true, it should passthrough (no modification)+ // If supportsVision is null/undefined (no sync data), it will process — that's correct behavior+ const capabilities = getResolvedModelCapabilities("openai/gpt-4o");+ if (capabilities.supportsVision === true) {+ assert.strictEqual(result.block, false);+ assert.strictEqual(result.modifiedPayload, undefined);+ } else {+ // Without sync data, supportsVision is null — guardrail processes the image+ // This is correct fail-open behavior for unknown model capabilities+ assert.strictEqual(result.block, false);+ }+});++test("VB-S02: model capabilities returns supportsVision for known models", () => {+ const gpt4oCaps = getResolvedModelCapabilities("openai/gpt-4o");+ // supportsVision may be true (if sync data exists) or null (if not synced)+ assert.ok(gpt4oCaps.supportsVision === true || gpt4oCaps.supportsVision === null);+});++// ── VB-S04: No images passthrough ──────────────────────────────────────────++test("VB-S04: passthroughs when no images in messages", async () => {+ const guardrail = createGuardrail();+ const payload = createPayload({+ messages: [{ role: "user", content: "Hello, how are you?" }],+ });++ const result = await guardrail.preCall(payload, createContext());+ assert.strictEqual(result.block, false);+ assert.strictEqual(result.modifiedPayload, undefined);+});++test("VB-S04: passthroughs when messages array is empty", async () => {+ const guardrail = createGuardrail();+ const payload = createPayload({ messages: [] });+ const result = await guardrail.preCall(payload, createContext());+ assert.strictEqual(result.block, false);+});++// ── VB-S01: Single image processing ─────────────────────────────────────────++test("VB-S01: replaces image with description for non-vision model", async () => {+ mockVisionResponse = "A beautiful sunset over the ocean";+ const guardrail = createGuardrail();++ const payload = createPayload({+ model: "minimax/minimax-01",+ messages: [+ {+ role: "user",+ content: [+ { type: "text", text: "What is in this image?" },+ {+ type: "image_url",+ image_url: { url: "https://example.com/image.png" },+ },+ ],+ },+ ],+ });++ const result = await guardrail.preCall(+ payload,+ createContext({ model: "minimax/minimax-01" })+ );++ assert.strictEqual(result.block, false);+ assert.ok(result.modifiedPayload);++ const modified = result.modifiedPayload as {+ messages: Array<{ content: unknown[] }>;+ };+ const content = modified.messages[0].content as Array<{+ type: string;+ text?: string;+ }>;++ const imagePart = content.find((p) => p.type === "image_url");+ assert.strictEqual(imagePart, undefined);++ const descriptionPart = content.find(+ (p) => p.type === "text" && p.text?.includes("sunset")+ );+ assert.ok(descriptionPart);+});++// ── VB-S04: Multiple images ─────────────────────────────────────────────────++test("VB-S04: processes multiple images and concatenates descriptions", async () => {+ let callIdx = 0;+ const descriptions = ["A cute cat", "A playful dog", "A colorful bird"];++ const guardrail = new VisionBridgeGuardrail({+ deps: {+ getSettings: async () => mockSettings,+ callVisionModel: async () => {+ const desc = descriptions[callIdx] || "Unknown image";+ callIdx++;+ return desc;+ },+ },+ });++ const payload = createPayload({+ model: "minimax/minimax-01",+ messages: [+ {+ role: "user",+ content: [+ { type: "text", text: "Describe these images" },+ {+ type: "image_url",+ image_url: { url: "https://example.com/cat.png" },+ },+ {+ type: "image_url",+ image_url: { url: "https://example.com/dog.png" },+ },+ {+ type: "image_url",+ image_url: { url: "https://example.com/bird.png" },+ },+ ],+ },+ ],+ });++ const result = await guardrail.preCall(+ payload,+ createContext({ model: "minimax/minimax-01" })+ );++ assert.strictEqual(result.block, false);+ assert.ok(result.modifiedPayload);+ assert.strictEqual(callIdx, 3);++ const modified = result.modifiedPayload as {+ messages: Array<{ content: unknown[] }>;+ };+ const content = modified.messages[0].content as Array<{+ type: string;+ text?: string;+ }>;++ assert.ok(+ content.some((p) => p.type === "text" && p.text?.includes("[Image 1]"))+ );+ assert.ok(+ content.some((p) => p.type === "text" && p.text?.includes("[Image 2]"))+ );+ assert.ok(+ content.some((p) => p.type === "text" && p.text?.includes("[Image 3]"))+ );+});++// ── VB-S03: Fail-open on vision error ──────────────────────────────────────++test("VB-S03: returns modified payload with unavailable text when vision API fails", async () => {+ shouldVisionFail = true;+ const guardrail = createGuardrail();++ const payload = createPayload({+ model: "minimax/minimax-01",+ messages: [+ {+ role: "user",+ content: [+ { type: "text", text: "What is this?" },+ {+ type: "image_url",+ image_url: { url: "https://example.com/image.png" },+ },+ ],+ },+ ],+ });++ const result = await guardrail.preCall(+ payload,+ createContext({ model: "minimax/minimax-01" })+ );++ assert.strictEqual(result.block, false);+ assert.ok(result.modifiedPayload);++ const modified = result.modifiedPayload as {+ messages: Array<{ content: unknown[] }>;+ };+ const content = modified.messages[0].content as Array<{+ type: string;+ text?: string;+ }>;++ // Should have "unavailable" text instead of image+ const unavailPart = content.find(+ (p) => p.type === "text" && p.text?.includes("unavailable")+ );+ assert.ok(unavailPart);+});++test("VB-S03: logs warning when vision API fails", async () => {+ shouldVisionFail = true;+ let warningLogged = false;+ const guardrail = createGuardrail();++ const payload = createPayload({+ model: "minimax/minimax-01",+ messages: [+ {+ role: "user",+ content: [+ {+ type: "image_url",+ image_url: { url: "https://example.com/image.png" },+ },+ ],+ },+ ],+ });++ const mockLog = {+ warn: (_tag: string, msg: string) => {+ if (msg.includes("Failed to get description")) {+ warningLogged = true;+ }+ },+ };++ await guardrail.preCall(+ payload,+ createContext({+ model: "minimax/minimax-01",+ log: mockLog as GuardrailContext["log"],+ })+ );++ assert.strictEqual(warningLogged, true);+});++// ── VB-S07: Base64 image format ─────────────────────────────────────────────++test("VB-S07: handles base64 image format", async () => {+ mockVisionResponse = "An image description";+ const guardrail = createGuardrail();++ const payload = createPayload({+ model: "minimax/minimax-01",+ messages: [+ {+ role: "user",+ content: [+ {+ type: "image",+ source: {+ type: "base64",+ media_type: "image/png",+ data: "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==",+ },+ },+ ],+ },+ ],+ });++ const result = await guardrail.preCall(+ payload,+ createContext({ model: "minimax/minimax-01" })+ );++ assert.strictEqual(result.block, false);+ assert.ok(result.modifiedPayload);+});++// ── VB-S09: Image count limit ───────────────────────────────────────────────++test("VB-S09: respects maxImages setting", async () => {+ mockSettings.visionBridgeMaxImages = 2;+ const guardrail = createGuardrail();++ const images = Array.from({ length: 5 }, (_, i) => ({+ type: "image_url" as const,+ image_url: { url: `https://example.com/image${i}.png` },+ }));++ const payload = createPayload({+ model: "minimax/minimax-01",+ messages: [+ {+ role: "user",+ content: [{ type: "text", text: "Describe these" }, ...images],+ },+ ],+ });++ await guardrail.preCall(+ payload,+ createContext({ model: "minimax/minimax-01" })+ );++ // Should only call vision API for 2 images (maxImages=2)+ assert.strictEqual(visionCallCount, 2);+});++// ── VB-S10: Meta information returned ───────────────────────────────────────++test("VB-S10: returns meta with imagesProcessed count", async () => {+ mockVisionResponse = "A test description";+ const guardrail = createGuardrail();++ const payload = createPayload({+ model: "minimax/minimax-01",+ messages: [+ {+ role: "user",+ content: [+ { type: "text", text: "What is this?" },+ {+ type: "image_url",+ image_url: { url: "https://example.com/a.png" },+ },+ {+ type: "image_url",+ image_url: { url: "https://example.com/b.png" },+ },+ ],+ },+ ],+ });++ const result = await guardrail.preCall(+ payload,+ createContext({ model: "minimax/minimax-01" })+ );++ assert.ok(result.meta);+ assert.ok(typeof result.meta === "object");++ const meta = result.meta as Record<string, unknown>;+ assert.strictEqual(meta.imagesProcessed, 2);+ assert.ok(Array.isArray(meta.descriptions));+ assert.strictEqual((meta.descriptions as string[]).length, 2);+ assert.strictEqual(typeof meta.processingTimeMs, "number");+ assert.strictEqual(meta.visionModel, "openai/gpt-4o-mini");+});\ No newline at end of filediff --git a/tests/unit/guardrails/visionBridgeHelpers.callVisionModel.test.ts b/tests/unit/guardrails/visionBridgeHelpers.callVisionModel.test.tsnew file mode 100644index 0000000000..dd241214e1--- /dev/null+++ b/tests/unit/guardrails/visionBridgeHelpers.callVisionModel.test.ts@@ -0,0 +1,231 @@+/**+ * Tests for callVisionModel helper function.+ */++import test from "node:test";+import assert from "node:assert/strict";+import { callVisionModel, type VisionModelConfig } from "@/lib/guardrails/visionBridgeHelpers";++// Store original fetch+const originalFetch = globalThis.fetch;++test("callVisionModel returns description on success", async () => {+ // Mock global fetch+ const mockResponse = {+ ok: true,+ json: async () => ({+ choices: [{ message: { content: "A beautiful sunset over the ocean" } }],+ }),+ };+ globalThis.fetch = async () => mockResponse as unknown as Response;++ try {+ const config: VisionModelConfig = {+ model: "openai/gpt-4o-mini",+ prompt: "Describe this image",+ timeoutMs: 30000,+ maxImages: 10,+ };++ const result = await callVisionModel(+ "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==",+ config+ );++ assert.strictEqual(result, "A beautiful sunset over the ocean");+ } finally {+ // Restore original fetch+ globalThis.fetch = originalFetch;+ }+});++test("callVisionModel throws on HTTP error", async () => {+ const mockResponse = {+ ok: false,+ status: 500,+ text: async () => "Internal Server Error",+ };+ globalThis.fetch = async () => mockResponse as unknown as Response;++ try {+ const config: VisionModelConfig = {+ model: "openai/gpt-4o-mini",+ prompt: "Describe this image",+ timeoutMs: 30000,+ maxImages: 10,+ };++ await assert.rejects(+ async () =>+ await callVisionModel("data:image/png;base64,iVBORw0KGgo", config),+ /Vision API error 500/+ );+ } finally {+ globalThis.fetch = originalFetch;+ }+});++test("callVisionModel throws on API error response", async () => {+ const mockResponse = {+ ok: true,+ json: async () => ({+ error: { message: "Invalid API key" },+ }),+ };+ globalThis.fetch = async () => mockResponse as unknown as Response;++ try {+ const config: VisionModelConfig = {+ model: "openai/gpt-4o-mini",+ prompt: "Describe this image",+ timeoutMs: 30000,+ maxImages: 10,+ };++ await assert.rejects(+ async () => await callVisionModel("data:image/png;base64,iVBORw0KGgo", config),+ /Invalid API key/+ );+ } finally {+ globalThis.fetch = originalFetch;+ }+});++test("callVisionModel throws on empty response", async () => {+ const mockResponse = {+ ok: true,+ json: async () => ({+ choices: [{}],+ }),+ };+ globalThis.fetch = async () => mockResponse as unknown as Response;++ try {+ const config: VisionModelConfig = {+ model: "openai/gpt-4o-mini",+ prompt: "Describe this image",+ timeoutMs: 30000,+ maxImages: 10,+ };++ await assert.rejects(+ async () => await callVisionModel("data:image/png;base64,iVBORw0KGgo", config),+ /empty or invalid/+ );+ } finally {+ globalThis.fetch = originalFetch;+ }+});++test("callVisionModel trims whitespace from response", async () => {+ const mockResponse = {+ ok: true,+ json: async () => ({+ choices: [{ message: { content: " A test description " } }],+ }),+ };+ globalThis.fetch = async () => mockResponse as unknown as Response;++ try {+ const config: VisionModelConfig = {+ model: "openai/gpt-4o-mini",+ prompt: "Describe this image",+ timeoutMs: 30000,+ maxImages: 10,+ };++ const result = await callVisionModel("data:image/png;base64,iVBORw0KGgo", config);+ assert.strictEqual(result, "A test description");+ } finally {+ globalThis.fetch = originalFetch;+ }+});++test("callVisionModel passes custom API key", async () => {+ let capturedHeaders: Record<string, string> = {};++ const mockResponse = {+ ok: true,+ json: async () => ({+ choices: [{ message: { content: "Description" } }],+ }),+ };++ globalThis.fetch = async (url: URL | RequestInfo, init?: RequestInit) => {+ if (init?.headers) {+ capturedHeaders = init.headers as Record<string, string>;+ }+ return mockResponse as unknown as Response;+ };++ try {+ const config: VisionModelConfig = {+ model: "openai/gpt-4o-mini",+ prompt: "Describe this image",+ timeoutMs: 30000,+ maxImages: 10,+ };++ await callVisionModel("data:image/png;base64,iVBORw0KGgo", config, "sk-custom-key");++ assert.strictEqual(+ capturedHeaders["Authorization"],+ "Bearer sk-custom-key"+ );+ } finally {+ globalThis.fetch = originalFetch;+ }+});++test("callVisionModel uses correct request body format", async () => {+ let capturedBody: Record<string, unknown> = {};++ const mockResponse = {+ ok: true,+ json: async () => ({+ choices: [{ message: { content: "Description" } }],+ }),+ };++ globalThis.fetch = async (url: URL | RequestInfo, init?: RequestInit) => {+ if (init?.body) {+ capturedBody = JSON.parse(init.body as string);+ }+ return mockResponse as unknown as Response;+ };++ try {+ const config: VisionModelConfig = {+ model: "openai/gpt-4o-mini",+ prompt: "What is in this image?",+ timeoutMs: 30000,+ maxImages: 10,+ };++ const imageUri = "data:image/png;base64,test123";+ await callVisionModel(imageUri, config);++ // Verify request structure+ assert.strictEqual(capturedBody.model, "gpt-4o-mini");+ assert.ok(Array.isArray(capturedBody.messages));+ assert.strictEqual((capturedBody.messages as unknown[]).length, 1);++ const message = (capturedBody.messages as Array<{role: string; content: unknown[]}>)[0];+ assert.strictEqual(message.role, "user");+ assert.ok(Array.isArray(message.content));+ assert.strictEqual(message.content.length, 2);++ // First content is image_url+ const imagePart = message.content[0] as {type: string; image_url: {url: string; detail: string}};+ assert.strictEqual(imagePart.type, "image_url");+ assert.strictEqual(imagePart.image_url.url, imageUri);+ assert.strictEqual(imagePart.image_url.detail, "low");++ // Second content is text prompt+ const textPart = message.content[1] as {type: string; text: string};+ assert.strictEqual(textPart.type, "text");+ assert.strictEqual(textPart.text, "What is in this image?");+ } finally {+ globalThis.fetch = originalFetch;+ }+});diff --git a/tests/unit/guardrails/visionBridgeHelpers.extractImageParts.test.ts b/tests/unit/guardrails/visionBridgeHelpers.extractImageParts.test.tsnew file mode 100644index 0000000000..e061548c45--- /dev/null+++ b/tests/unit/guardrails/visionBridgeHelpers.extractImageParts.test.ts@@ -0,0 +1,138 @@+/**+ * Tests for extractImageParts helper function.+ */++import test from "node:test";+import assert from "node:assert/strict";+import { extractImageParts } from "@/lib/guardrails/visionBridgeHelpers";++interface RequestMessage {+ role?: string;+ content?: string | RequestContentPart[];+}++type RequestContentPart =+ | { type: "text"; text: string }+ | { type: "image_url"; image_url: { url: string; detail?: string } }+ | { type: "image"; source: { type: "base64"; media_type: string; data: string } };++test("extractImageParts returns empty array for messages without images", () => {+ const messages: RequestMessage[] = [+ { role: "user", content: "Hello, how are you?" },+ ];+ const result = extractImageParts(messages);+ assert.deepStrictEqual(result, []);+});++test("extractImageParts detects image_url format", () => {+ const messages: RequestMessage[] = [+ {+ role: "user",+ content: [+ { type: "text", text: "What is in this image?" },+ { type: "image_url", image_url: { url: "https://example.com/image.png" } },+ ],+ },+ ];+ const result = extractImageParts(messages);+ assert.strictEqual(result.length, 1);+ assert.strictEqual(result[0].messageIndex, 0);+ assert.strictEqual(result[0].partIndex, 1);+ assert.strictEqual(result[0].imageUrl, "https://example.com/image.png");+ assert.strictEqual(result[0].imageType, "image_url");+});++test("extractImageParts detects base64 image format", () => {+ const messages: RequestMessage[] = [+ {+ role: "user",+ content: [+ {+ type: "image",+ source: {+ type: "base64",+ media_type: "image/png",+ data: "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==",+ },+ },+ ],+ },+ ];+ const result = extractImageParts(messages);+ assert.strictEqual(result.length, 1);+ assert.strictEqual(result[0].imageType, "image");+ assert.ok(result[0].imageUrl.startsWith("data:image/png;base64,"));+});++test("extractImageParts handles multiple images in single message", () => {+ const messages: RequestMessage[] = [+ {+ role: "user",+ content: [+ { type: "text", text: "Compare these images" },+ { type: "image_url", image_url: { url: "https://example.com/image1.png" } },+ { type: "image_url", image_url: { url: "https://example.com/image2.png" } },+ ],+ },+ ];+ const result = extractImageParts(messages);+ assert.strictEqual(result.length, 2);+ assert.strictEqual(result[0].partIndex, 1);+ assert.strictEqual(result[1].partIndex, 2);+});++test("extractImageParts handles images across multiple messages", () => {+ const messages: RequestMessage[] = [+ { role: "user", content: [{ type: "image_url", image_url: { url: "https://example.com/image1.png" } }] },+ { role: "assistant", content: "Here is analysis of the first image." },+ { role: "user", content: [{ type: "image_url", image_url: { url: "https://example.com/image2.png" } }] },+ ];+ const result = extractImageParts(messages);+ assert.strictEqual(result.length, 2);+ assert.strictEqual(result[0].messageIndex, 0);+ assert.strictEqual(result[1].messageIndex, 2);+});++test("extractImageParts handles empty messages array", () => {+ const result = extractImageParts([]);+ assert.deepStrictEqual(result, []);+});++test("extractImageParts handles messages with null/undefined content", () => {+ const messages: RequestMessage[] = [+ { role: "user", content: null as unknown as RequestContentPart[] },+ { role: "user", content: undefined as unknown as RequestContentPart[] },+ ];+ const result = extractImageParts(messages);+ assert.deepStrictEqual(result, []);+});++test("extractImageParts handles data URI image_url format", () => {+ const dataUri = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==";+ const messages: RequestMessage[] = [+ { role: "user", content: [{ type: "image_url", image_url: { url: dataUri } }] },+ ];+ const result = extractImageParts(messages);+ assert.strictEqual(result.length, 1);+ assert.strictEqual(result[0].imageUrl, dataUri);+});++test("extractImageParts preserves order of images", () => {+ const messages: RequestMessage[] = [+ {+ role: "user",+ content: [+ { type: "text", text: "First" },+ { type: "image_url", image_url: { url: "https://example.com/A.png" } },+ { type: "text", text: "Second" },+ { type: "image_url", image_url: { url: "https://example.com/B.png" } },+ { type: "image_url", image_url: { url: "https://example.com/C.png" } },+ ],+ },+ ];+ const result = extractImageParts(messages);+ assert.strictEqual(result.length, 3);+ assert.strictEqual(result[0].partIndex, 1);+ assert.strictEqual(result[1].partIndex, 3);+ assert.strictEqual(result[2].partIndex, 4);+});diff --git a/tests/unit/guardrails/visionBridgeHelpers.replaceImageParts.test.ts b/tests/unit/guardrails/visionBridgeHelpers.replaceImageParts.test.tsnew file mode 100644index 0000000000..263e3dbfb3--- /dev/null+++ b/tests/unit/guardrails/visionBridgeHelpers.replaceImageParts.test.ts@@ -0,0 +1,254 @@+/**+ * Tests for replaceImageParts helper function.+ */++import test from "node:test";+import assert from "node:assert/strict";+import { replaceImageParts } from "@/lib/guardrails/visionBridgeHelpers";++test("replaceImageParts replaces single image with description", () => {+ const body = {+ model: "minimax/minimax-01",+ messages: [+ {+ role: "user",+ content: [+ { type: "text", text: "What is in this image?" },+ { type: "image_url", image_url: { url: "https://example.com/image.png" } },+ ],+ },+ ],+ };++ const descriptions = ["[Image 1]: A beautiful sunset"];++ const result = replaceImageParts(body, descriptions);++ // Should have original text preserved+ const content = result.messages[0].content as Array<{type: string; text?: string}>;+ assert.strictEqual(content[0].type, "text");+ assert.strictEqual(content[0].text, "What is in this image?");++ // Should have description instead of image+ assert.strictEqual(content[1].type, "text");+ assert.strictEqual(content[1].text, "[Image 1]: A beautiful sunset");+});++test("replaceImageParts replaces multiple images with descriptions", () => {+ const body = {+ model: "minimax/minimax-01",+ messages: [+ {+ role: "user",+ content: [+ { type: "text", text: "Compare these images" },+ { type: "image_url", image_url: { url: "https://example.com/A.png" } },+ { type: "image_url", image_url: { url: "https://example.com/B.png" } },+ { type: "image_url", image_url: { url: "https://example.com/C.png" } },+ ],+ },+ ],+ };++ const descriptions = [+ "[Image 1]: A cat",+ "[Image 2]: A dog",+ "[Image 3]: A bird",+ ];++ const result = replaceImageParts(body, descriptions);++ const content = result.messages[0].content as Array<{type: string; text?: string}>;+ assert.strictEqual(content[0].type, "text");+ assert.strictEqual(content[0].text, "Compare these images");+ assert.strictEqual(content[1].type, "text");+ assert.strictEqual(content[1].text, "[Image 1]: A cat");+ assert.strictEqual(content[2].type, "text");+ assert.strictEqual(content[2].text, "[Image 2]: A dog");+ assert.strictEqual(content[3].type, "text");+ assert.strictEqual(content[3].text, "[Image 3]: A bird");+});++test("replaceImageParts handles empty descriptions array", () => {+ const body = {+ model: "minimax/minimax-01",+ messages: [+ {+ role: "user",+ content: [+ { type: "text", text: "Hello" },+ { type: "image_url", image_url: { url: "https://example.com/image.png" } },+ ],+ },+ ],+ };++ const result = replaceImageParts(body, []);++ // Original should be unchanged+ const content = result.messages[0].content as Array<{type: string}>;+ assert.strictEqual(content[1].type, "image_url");+});++test("replaceImageParts preserves non-image content", () => {+ const body = {+ model: "minimax/minimax-01",+ messages: [+ {+ role: "system",+ content: "You are a helpful assistant.",+ },+ {+ role: "user",+ content: [+ { type: "text", text: "Analyze this" },+ { type: "image_url", image_url: { url: "https://example.com/image.png" } },+ ],+ },+ {+ role: "assistant",+ content: "I can see the image shows a sunset.",+ },+ ],+ };++ const result = replaceImageParts(body, ["[Image 1]: A sunset over the ocean"]);++ // System message should be unchanged+ assert.strictEqual(result.messages[0].content, "You are a helpful assistant.");++ // Assistant message should be unchanged+ assert.strictEqual(result.messages[2].content, "I can see the image shows a sunset.");+});++test("replaceImageParts handles base64 images", () => {+ const body = {+ model: "minimax/minimax-01",+ messages: [+ {+ role: "user",+ content: [+ {+ type: "image",+ source: {+ type: "base64",+ media_type: "image/png",+ data: "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==",+ },+ },+ ],+ },+ ],+ };++ const descriptions = ["[Image 1]: A red circle"];++ const result = replaceImageParts(body, descriptions);++ const content = result.messages[0].content as Array<{type: string; text?: string}>;+ assert.strictEqual(content[0].type, "text");+ assert.strictEqual(content[0].text, "[Image 1]: A red circle");+});++test("replaceImageParts handles undefined descriptions", () => {+ const body = {+ model: "test",+ messages: [+ {+ role: "user",+ content: [+ { type: "text", text: "Hello" },+ { type: "image_url", image_url: { url: "https://example.com/image.png" } },+ ],+ },+ ],+ };++ const result = replaceImageParts(body, undefined as unknown as string[]);++ // Should return original body when descriptions is undefined+ const content = result.messages[0].content as Array<{type: string}>;+ assert.strictEqual(content[1].type, "image_url");+});++test("replaceImageParts handles empty messages array", () => {+ const body = {+ model: "test",+ messages: [],+ };++ const descriptions = ["[Image 1]: Description"];+ const result = replaceImageParts(body, descriptions);++ assert.deepStrictEqual(result.messages, []);+});++test("replaceImageParts handles messages without content array", () => {+ const body = {+ model: "test",+ messages: [+ { role: "user", content: "Just a text message" },+ { role: "user", content: [{ type: "image_url", image_url: { url: "https://example.com/image.png" } }] },+ ],+ };++ const descriptions = ["[Image 1]: Description"];+ const result = replaceImageParts(body, descriptions);++ // First message (string content) should be unchanged+ assert.strictEqual(result.messages[0].content, "Just a text message");++ // Second message should have image replaced+ const content = result.messages[1].content as Array<{type: string; text?: string}>;+ assert.strictEqual(content[0].type, "text");+ assert.strictEqual(content[0].text, "[Image 1]: Description");+});++test("replaceImageParts does not modify original body", () => {+ const body = {+ model: "minimax/minimax-01",+ messages: [+ {+ role: "user",+ content: [+ { type: "text", text: "Original" },+ { type: "image_url", image_url: { url: "https://example.com/image.png" } },+ ],+ },+ ],+ };++ const descriptions = ["[Image 1]: Modified"];+ replaceImageParts(body, descriptions);++ // Original should be unchanged+ const content = body.messages[0].content as Array<{type: string}>;+ assert.strictEqual(content[1].type, "image_url");+});++test("replaceImageParts handles mixed images and text", () => {+ const body = {+ model: "test",+ messages: [+ {+ role: "user",+ content: [+ { type: "image_url", image_url: { url: "https://example.com/first.png" } },+ { type: "text", text: "between images" },+ { type: "image_url", image_url: { url: "https://example.com/second.png" } },+ ],+ },+ ],+ };++ const descriptions = ["[Image 1]: First image", "[Image 2]: Second image"];+ const result = replaceImageParts(body, descriptions);++ const content = result.messages[0].content as Array<{type: string; text?: string}>;+ assert.strictEqual(content[0].type, "text");+ assert.strictEqual(content[0].text, "[Image 1]: First image");+ assert.strictEqual(content[1].type, "text");+ assert.strictEqual(content[1].text, "between images");+ assert.strictEqual(content[2].type, "text");+ assert.strictEqual(content[2].text, "[Image 2]: Second image");+});diff --git a/tests/unit/guardrails/visionBridgeHelpers.resolveImageAsDataUri.test.ts b/tests/unit/guardrails/visionBridgeHelpers.resolveImageAsDataUri.test.tsnew file mode 100644index 0000000000..af47178c88--- /dev/null+++ b/tests/unit/guardrails/visionBridgeHelpers.resolveImageAsDataUri.test.ts@@ -0,0 +1,62 @@+/**+ * Tests for resolveImageAsDataUri helper function.+ */++import test from "node:test";+import assert from "node:assert/strict";+import { resolveImageAsDataUri } from "@/lib/guardrails/visionBridgeHelpers";++test("resolveImageAsDataUri passes through HTTPS URL as-is", () => {+ const url = "https://example.com/image.png";+ const result = resolveImageAsDataUri(url);+ assert.strictEqual(result, url);+});++test("resolveImageAsDataUri passes through HTTP URL as-is", () => {+ const url = "http://example.com/image.png";+ const result = resolveImageAsDataUri(url);+ assert.strictEqual(result, url);+});++test("resolveImageAsDataUri passes through data URI as-is", () => {+ const dataUri = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==";+ const result = resolveImageAsDataUri(dataUri);+ assert.strictEqual(result, dataUri);+});++test("resolveImageAsDataUri converts base64 string to PNG data URI", () => {+ const base64 = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==";+ const result = resolveImageAsDataUri(base64);+ assert.strictEqual(result, `data:image/png;base64,${base64}`);+});++test("resolveImageAsDataUri throws for empty string", () => {+ assert.throws(() => {+ resolveImageAsDataUri("");+ }, /Invalid image URL/);+});++test("resolveImageAsDataUri throws for null", () => {+ assert.throws(() => {+ resolveImageAsDataUri(null as unknown as string);+ }, /Invalid image URL/);+});++test("resolveImageAsDataUri throws for undefined", () => {+ assert.throws(() => {+ resolveImageAsDataUri(undefined as unknown as string);+ }, /Invalid image URL/);+});++test("resolveImageAsDataUri handles data URI with different media types", () => {+ const jpegUri = "data:image/jpeg;base64,/9j/4AAQSkZJRg==";+ const result = resolveImageAsDataUri(jpegUri);+ assert.strictEqual(result, jpegUri);+});++test("resolveImageAsDataUri treats plain base64 with prefix as data URI", () => {+ // If it doesn't start with data: or http, treat as raw base64+ const rawBase64 = "abcdef123456==";+ const result = resolveImageAsDataUri(rawBase64);+ assert.strictEqual(result, "data:image/png;base64,abcdef123456==");+});diff --git a/tests/unit/visionBridgeDefaults.test.ts b/tests/unit/visionBridgeDefaults.test.tsnew file mode 100644index 0000000000..2d631e05de--- /dev/null+++ b/tests/unit/visionBridgeDefaults.test.ts@@ -0,0 +1,91 @@+/**+ * Tests for Vision Bridge default constants.+ */++import test from "node:test";+import assert from "node:assert/strict";+import {+ VISION_BRIDGE_DEFAULTS,+ VISION_BRIDGE_SETTINGS_KEYS,+ getVisionBridgeConfig,+ type VisionBridgeSettings,+} from "@/shared/constants/visionBridgeDefaults";++test("VISION_BRIDGE_DEFAULTS exports correct values", () => {+ assert.strictEqual(VISION_BRIDGE_DEFAULTS.enabled, true);+ assert.strictEqual(VISION_BRIDGE_DEFAULTS.model, "openai/gpt-4o-mini");+ assert.strictEqual(+ VISION_BRIDGE_DEFAULTS.prompt,+ "Describe this image concisely in 2-3 sentences. Focus on the most relevant visual details."+ );+ assert.strictEqual(VISION_BRIDGE_DEFAULTS.timeoutMs, 30000);+ assert.strictEqual(VISION_BRIDGE_DEFAULTS.maxImagesPerRequest, 10);+});++test("VISION_BRIDGE_SETTINGS_KEYS exports all expected keys", () => {+ assert.deepStrictEqual(VISION_BRIDGE_SETTINGS_KEYS, [+ "visionBridgeEnabled",+ "visionBridgeModel",+ "visionBridgePrompt",+ "visionBridgeTimeout",+ "visionBridgeMaxImages",+ ]);+});++test("getVisionBridgeConfig returns defaults when no settings provided", () => {+ const config = getVisionBridgeConfig({});++ assert.strictEqual(config.enabled, true);+ assert.strictEqual(config.model, "openai/gpt-4o-mini");+ assert.strictEqual(config.prompt, VISION_BRIDGE_DEFAULTS.prompt);+ assert.strictEqual(config.timeoutMs, 30000);+ assert.strictEqual(config.maxImages, 10);+});++test("getVisionBridgeConfig applies custom settings", () => {+ const customSettings: VisionBridgeSettings = {+ visionBridgeEnabled: false,+ visionBridgeModel: "anthropic/claude-3-haiku",+ visionBridgePrompt: "What is in this image?",+ visionBridgeTimeout: 60000,+ visionBridgeMaxImages: 5,+ };++ const config = getVisionBridgeConfig(customSettings);++ assert.strictEqual(config.enabled, false);+ assert.strictEqual(config.model, "anthropic/claude-3-haiku");+ assert.strictEqual(config.prompt, "What is in this image?");+ assert.strictEqual(config.timeoutMs, 60000);+ assert.strictEqual(config.maxImages, 5);+});++test("getVisionBridgeConfig merges partial settings with defaults", () => {+ const partialSettings: VisionBridgeSettings = {+ visionBridgeModel: "openai/gpt-4o",+ };++ const config = getVisionBridgeConfig(partialSettings);++ // Custom value+ assert.strictEqual(config.model, "openai/gpt-4o");+ // Default values for the rest+ assert.strictEqual(config.enabled, true);+ assert.strictEqual(config.prompt, VISION_BRIDGE_DEFAULTS.prompt);+ assert.strictEqual(config.timeoutMs, 30000);+ assert.strictEqual(config.maxImages, 10);+});++test("getVisionBridgeConfig handles undefined settings", () => {+ const config = getVisionBridgeConfig(undefined);++ assert.strictEqual(config.enabled, true);+ assert.strictEqual(config.model, "openai/gpt-4o-mini");+});++test("getVisionBridgeConfig handles null settings", () => {+ const config = getVisionBridgeConfig(null as unknown as VisionBridgeSettings);++ assert.strictEqual(config.enabled, true);+ assert.strictEqual(config.model, "openai/gpt-4o-mini");+});