Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix(sdk): preserve Vercel AI SDK file and image parts through compression (#4048)

Shubham Srivastava1668a6c0dd73
diff --git a/sdk/typescript/src/types.ts b/sdk/typescript/src/types.ts
index aef505aad..43951c633 100644
--- a/sdk/typescript/src/types.ts
+++ b/sdk/typescript/src/types.ts
@@ -38,7 +38,11 @@ export interface UserMessage {
export interface AssistantMessage {
role: "assistant";
- content: string | null;
+ /**
+ * A string, or an ordered part array when the assistant turn carries media
+ * (e.g. a Vercel AI SDK file part): text parts plus `image_url` parts.
+ */
+ content: string | ContentPart[] | null;
tool_calls?: ToolCall[];
}
diff --git a/sdk/typescript/src/utils/format.ts b/sdk/typescript/src/utils/format.ts
index d94dd70fb..ec744ad71 100644
--- a/sdk/typescript/src/utils/format.ts
+++ b/sdk/typescript/src/utils/format.ts
@@ -14,6 +14,7 @@
import type {
OpenAIMessage,
AssistantMessage,
+ ContentPart,
ToolCall,
} from "../types.js";
@@ -51,8 +52,14 @@ export function detectFormat(messages: any[]): MessageFormat {
if (part.type === "tool-call" || part.type === "tool-result") return "vercel";
// Anthropic: underscored types
if (part.type === "tool_use" || part.type === "tool_result") return "anthropic";
- // Anthropic: image with source.type
- if (part.type === "image" && part.source?.type) return "anthropic";
+ // Anthropic: image / document with source.type
+ if ((part.type === "image" || part.type === "document") && part.source?.type) return "anthropic";
+ // Vercel: `file` parts carry data/mediaType (OpenAI's `file` part carries a `file` object);
+ // `image` parts carry `image` (Anthropic's carry `source`)
+ if (part && typeof part === "object") {
+ if (part.type === "file" && ("data" in part || "mediaType" in part)) return "vercel";
+ if (part.type === "image" && "image" in part) return "vercel";
+ }
}
}
}
@@ -88,6 +95,74 @@ function openAIImageBlock(url: string): any {
return { type: "image", source: { type: "url", url } };
}
+type AnthropicMediaImageUrl = {
+ url: string;
+ mime_type?: string;
+ // A plain-text document source travels as a data: URI and is tagged so it is restored as
+ // text, not base64; sources the OpenAI shape cannot express (content, file) are carried
+ // verbatim on `source`. The document's own fields are carried on `document`.
+ source_type?: "text";
+ source?: any;
+ document?: Record<string, any>;
+};
+
+const ANTHROPIC_DOCUMENT_FIELDS = ["title", "context", "citations", "cache_control"] as const;
+
+// Anthropic media block -> OpenAI image_url object, or null if unconvertible: image blocks via
+// anthropicImageUrl; document blocks by source — base64 PDF as a data: URI, url with the MIME
+// type carried on the private `mime_type` key, plain text as a tagged data: URI, and content /
+// file sources carried verbatim — plus the document's title / context / citations / cache_control.
+function anthropicMediaImageUrl(block: any): AnthropicMediaImageUrl | null {
+ if (block.type === "image") {
+ const url = anthropicImageUrl(block.source);
+ return url ? { url } : null;
+ }
+ if (block.type !== "document" || !block.source || typeof block.source !== "object") return null;
+ const src = block.source;
+ let out: AnthropicMediaImageUrl;
+ if (src.type === "base64" && src.media_type && src.data) {
+ out = { url: `data:${src.media_type};base64,${src.data}` };
+ } else if (src.type === "url" && typeof src.url === "string") {
+ out = { url: src.url, mime_type: "application/pdf" };
+ } else if (src.type === "text" && typeof src.data === "string") {
+ out = {
+ url: `data:${src.media_type || "text/plain"};base64,${bytesToBase64(new TextEncoder().encode(src.data))}`,
+ source_type: "text",
+ };
+ } else {
+ out = { url: "data:;base64,", source: src };
+ }
+ const fields: Record<string, any> = {};
+ for (const k of ANTHROPIC_DOCUMENT_FIELDS) if (block[k] !== undefined) fields[k] = block[k];
+ if (Object.keys(fields).length > 0) out.document = fields;
+ return out;
+}
+
+// OpenAI image_url -> Anthropic media block, or null when Anthropic has no block for that media:
+// a carried document source or text document is rebuilt as a document block with its fields;
+// application/pdf -> document block (base64 or url source); image/* (or unknown) -> image block.
+function anthropicMediaBlock(imageUrl: any): any {
+ const url: string = imageUrl.url;
+ const m = /^data:([^;,]*);base64,(.*)$/s.exec(url);
+ const mediaType: string | undefined =
+ (m && m[1]) || (typeof imageUrl.mime_type === "string" ? imageUrl.mime_type : undefined);
+ const fields = imageUrl.document && typeof imageUrl.document === "object" ? imageUrl.document : {};
+ if (imageUrl.source && typeof imageUrl.source === "object") {
+ return { type: "document", source: imageUrl.source, ...fields };
+ }
+ if (imageUrl.source_type === "text" && m) {
+ return { type: "document", source: { type: "text", media_type: "text/plain", data: base64ToText(m[2]) }, ...fields };
+ }
+ if (mediaType === "application/pdf") {
+ const source = m
+ ? { type: "base64", media_type: "application/pdf", data: m[2] }
+ : { type: "url", url };
+ return { type: "document", source, ...fields };
+ }
+ if (mediaType === undefined || mediaType.startsWith("image/")) return openAIImageBlock(url);
+ return null;
+}
+
export function anthropicToOpenAI(messages: any[]): OpenAIMessage[] {
const result: OpenAIMessage[] = [];
@@ -99,15 +174,15 @@ export function anthropicToOpenAI(messages: any[]): OpenAIMessage[] {
}
if (Array.isArray(msg.content)) {
const toolResults = msg.content.filter((b: any) => b.type === "tool_result");
- const hasImages = msg.content.some((b: any) => b.type === "image");
+ const hasMedia = msg.content.some((b: any) => b.type === "image" || b.type === "document");
- if (hasImages) {
+ if (hasMedia) {
const parts = msg.content
- .filter((b: any) => b.type === "text" || b.type === "image")
+ .filter((b: any) => b.type === "text" || b.type === "image" || b.type === "document")
.map((b: any) => {
if (b.type === "text") return { type: "text" as const, text: b.text };
- const url = anthropicImageUrl(b.source);
- return url ? { type: "image_url" as const, image_url: { url } } : null;
+ const imageUrl = anthropicMediaImageUrl(b);
+ return imageUrl ? { type: "image_url" as const, image_url: imageUrl } : null;
})
.filter((p: any): p is NonNullable<typeof p> => p !== null);
@@ -147,10 +222,22 @@ export function anthropicToOpenAI(messages: any[]): OpenAIMessage[] {
if (Array.isArray(msg.content)) {
const textBlocks = msg.content.filter((b: any) => b.type === "text");
const toolUseBlocks = msg.content.filter((b: any) => b.type === "tool_use");
+ const hasMedia = msg.content.some((b: any) => b.type === "image" || b.type === "document");
- const content = textBlocks.length > 0
- ? textBlocks.map((b: any) => b.text).join("\n")
- : null;
+ let content: AssistantMessage["content"];
+ if (hasMedia) {
+ // Keep generated media as ordered parts (see AssistantMessage.content).
+ content = msg.content
+ .filter((b: any) => b.type === "text" || b.type === "image" || b.type === "document")
+ .map((b: any) => {
+ if (b.type === "text") return { type: "text" as const, text: b.text };
+ const imageUrl = anthropicMediaImageUrl(b);
+ return imageUrl ? { type: "image_url" as const, image_url: imageUrl } : null;
+ })
+ .filter((p: any): p is NonNullable<typeof p> => p !== null);
+ } else {
+ content = textBlocks.length > 0 ? textBlocks.map((b: any) => b.text).join("\n") : null;
+ }
const openaiMsg: AssistantMessage = { role: "assistant", content };
if (toolUseBlocks.length > 0) {
@@ -190,7 +277,7 @@ export function openAIToAnthropic(messages: OpenAIMessage[]): any[] {
role: "user",
content: msg.content.map((p) => {
if (p.type === "text") return { type: "text", text: p.text };
- if (p.type === "image_url") return openAIImageBlock(p.image_url.url);
+ if (p.type === "image_url") return anthropicMediaBlock(p.image_url) ?? { type: "text", text: "" };
return { type: "text", text: "" };
}),
});
@@ -200,7 +287,17 @@ export function openAIToAnthropic(messages: OpenAIMessage[]): any[] {
if (msg.role === "assistant") {
const blocks: any[] = [];
- if (msg.content) blocks.push({ type: "text", text: msg.content });
+ if (typeof msg.content === "string") {
+ if (msg.content) blocks.push({ type: "text", text: msg.content });
+ } else if (Array.isArray(msg.content)) {
+ for (const g of groupAssistantParts(msg.content)) {
+ if ("text" in g) blocks.push({ type: "text", text: g.text });
+ else {
+ const block = anthropicMediaBlock(g.image_url);
+ if (block) blocks.push(block);
+ }
+ }
+ }
if (msg.tool_calls) {
for (const tc of msg.tool_calls) {
blocks.push({
@@ -236,6 +333,167 @@ export function openAIToAnthropic(messages: OpenAIMessage[]): any[] {
// Vercel AI SDK → OpenAI
// ============================================================
+function bytesToBase64(bytes: Uint8Array): string {
+ if (typeof Buffer !== "undefined") return Buffer.from(bytes).toString("base64");
+ let binary = "";
+ for (let i = 0; i < bytes.length; i += 0x8000) {
+ binary += String.fromCharCode(...bytes.subarray(i, i + 0x8000));
+ }
+ return btoa(binary);
+}
+
+function parseUrl(value: string): URL | null {
+ try {
+ return new URL(value);
+ } catch {
+ return null;
+ }
+}
+
+type VercelMediaImageUrl = {
+ url: string;
+ mime_type?: string;
+ part?: "file";
+ filename?: string;
+ // AI SDK 7 / LanguageModelV4 tagged file data: the tag the input used, so the reverse
+ // conversion restores the same shape; absent for the bare AI SDK 6 forms.
+ data_tag?: "data" | "url" | "text" | "reference";
+ // A provider-managed file reference, carried verbatim (it has no bytes or url to convert).
+ reference?: Record<string, string>;
+};
+
+function base64ToText(b64: string): string {
+ if (typeof Buffer !== "undefined") return Buffer.from(b64, "base64").toString("utf8");
+ const binary = atob(b64);
+ const bytes = new Uint8Array(binary.length);
+ for (let i = 0; i < binary.length; i++) bytes[i] = binary.charCodeAt(i);
+ return new TextDecoder().decode(bytes);
+}
+
+// A bare AI SDK 6 provider reference: a plain string-valued record with no `type`.
+function isProviderReference(v: any): boolean {
+ return (
+ !!v &&
+ typeof v === "object" &&
+ !Array.isArray(v) &&
+ !(v instanceof Uint8Array) &&
+ !(v instanceof ArrayBuffer) &&
+ !(v instanceof URL) &&
+ v.type === undefined &&
+ Object.values(v).every((x) => typeof x === "string")
+ );
+}
+
+// Vercel `image` / `file` part -> OpenAI image_url object, or null if the data is unconvertible.
+// Accepts the bare AI SDK 6 forms (bytes, base64, data URL, URL, provider reference) and the
+// AI SDK 7 / LanguageModelV4 tagged forms ({ type: "data" | "url" | "text" | "reference" }).
+// Bytes, base64 and text travel as a data: URI, which carries the media type; URLs stay URLs
+// with the media type on a private `mime_type` key; references are carried verbatim on
+// `reference`. `part: "file"`, `filename` and `data_tag` are carried so the reverse conversion
+// rebuilds the right part in the same shape. The proxy passes non-text parts through unchanged.
+function vercelMediaImageUrl(p: any): VercelMediaImageUrl | null {
+ let data = p.type === "file" ? p.data : p.image;
+ const mediaType: string | undefined = typeof p.mediaType === "string" ? p.mediaType : p.mimeType;
+ const out: VercelMediaImageUrl = { url: "" };
+ if (p.type === "file") out.part = "file";
+ if (typeof p.filename === "string") out.filename = p.filename;
+
+ if (data && typeof data === "object" && typeof data.type === "string") {
+ if (data.type === "data") {
+ out.data_tag = "data";
+ data = data.data;
+ } else if (data.type === "url") {
+ out.data_tag = "url";
+ data = data.url;
+ } else if (data.type === "text" && typeof data.text === "string") {
+ out.data_tag = "text";
+ out.url = `data:${mediaType ?? "text/plain"};base64,${bytesToBase64(new TextEncoder().encode(data.text))}`;
+ return out;
+ } else if (data.type === "reference" && data.reference && typeof data.reference === "object") {
+ out.data_tag = "reference";
+ out.reference = data.reference;
+ out.url = `data:${mediaType ?? ""};base64,`;
+ return out;
+ } else {
+ return null;
+ }
+ } else if (isProviderReference(data)) {
+ out.reference = data;
+ out.url = `data:${mediaType ?? ""};base64,`;
+ return out;
+ }
+
+ let carryType = false;
+ if (data instanceof Uint8Array || data instanceof ArrayBuffer) {
+ const bytes = data instanceof Uint8Array ? data : new Uint8Array(data);
+ out.url = `data:${mediaType ?? ""};base64,${bytesToBase64(bytes)}`;
+ } else if (data instanceof URL) {
+ out.url = data.toString();
+ carryType = true;
+ } else if (typeof data === "string") {
+ if (data.startsWith("data:")) out.url = data;
+ else if (parseUrl(data)) {
+ out.url = data;
+ carryType = true;
+ } else out.url = `data:${mediaType ?? ""};base64,${data}`;
+ } else {
+ return null;
+ }
+ if (carryType && mediaType) out.mime_type = mediaType;
+ return out;
+}
+
+// OpenAI image_url -> Vercel `image` or `file` part, in the shape the forward conversion saw
+// (bare, or LanguageModelV4-tagged when `data_tag` is carried). A data: URI yields base64 data
+// plus its media type; an http(s) url yields a URL object; anything else stays a string; a
+// carried provider reference comes back verbatim. Never throws.
+function vercelMediaPart(imageUrl: any): any {
+ const url: string = imageUrl.url;
+ const tag: string | undefined = typeof imageUrl.data_tag === "string" ? imageUrl.data_tag : undefined;
+ const m = /^data:([^;,]*);base64,(.*)$/s.exec(url);
+ let mediaType: string | undefined = typeof imageUrl.mime_type === "string" ? imageUrl.mime_type : undefined;
+ if (m && m[1]) mediaType = m[1];
+ let data: any;
+ if (imageUrl.reference && typeof imageUrl.reference === "object") {
+ data = tag === "reference" ? { type: "reference", reference: imageUrl.reference } : imageUrl.reference;
+ } else if (m) {
+ data = m[2];
+ if (tag === "data") data = { type: "data", data: m[2] };
+ else if (tag === "text") data = { type: "text", text: base64ToText(m[2]) };
+ } else {
+ const parsed = parseUrl(url);
+ data = parsed ?? url;
+ if (tag === "url") data = { type: "url", url: parsed ?? url };
+ }
+ const isFile = imageUrl.part === "file" || (mediaType !== undefined && !mediaType.startsWith("image/"));
+ if (isFile) {
+ const part: any = { type: "file", mediaType: mediaType ?? "application/octet-stream", data };
+ if (typeof imageUrl.filename === "string") part.filename = imageUrl.filename;
+ return part;
+ }
+ const part: any = { type: "image", image: data };
+ if (mediaType) part.mediaType = mediaType;
+ return part;
+}
+
+// Assistant content parts for the Anthropic / Gemini reverse converters. A text-only array
+// collapses to one text joined with "\n" (the forward converters join text-only turns the same
+// way); an array carrying media keeps every part separate, because the forward converters keep
+// the exact parts in that case and a round trip must not add or move characters.
+function groupAssistantParts(parts: ContentPart[]): Array<{ text: string } | { image_url: any }> {
+ const hasMedia = parts.some((p) => p.type === "image_url");
+ if (!hasMedia) {
+ const text = parts.filter((p) => p.type === "text").map((p) => (p as any).text).join("\n");
+ return text ? [{ text }] : [];
+ }
+ const out: Array<{ text: string } | { image_url: any }> = [];
+ for (const p of parts) {
+ if (p.type === "text") out.push({ text: p.text });
+ else if (p.type === "image_url") out.push({ image_url: p.image_url });
+ }
+ return out;
+}
+
export function vercelToOpenAI(messages: any[]): OpenAIMessage[] {
const result: OpenAIMessage[] = [];
@@ -252,21 +510,19 @@ export function vercelToOpenAI(messages: any[]): OpenAIMessage[] {
}
const parts = Array.isArray(msg.content) ? msg.content : [];
const textParts = parts.filter((p: any) => p.type === "text");
- const imageParts = parts.filter((p: any) => p.type === "image");
+ const mediaParts = parts.filter((p: any) => p.type === "image" || p.type === "file");
- if (imageParts.length === 0 && textParts.length > 0) {
+ if (mediaParts.length === 0 && textParts.length > 0) {
result.push({ role: "user", content: textParts.map((p: any) => p.text).join("") });
} else {
const openaiParts = parts
- .filter((p: any) => p.type === "text" || p.type === "image")
+ .filter((p: any) => p.type === "text" || p.type === "image" || p.type === "file")
.map((p: any) => {
if (p.type === "text") return { type: "text" as const, text: p.text };
- if (p.type === "image") {
- const url = p.image instanceof URL ? p.image.toString() : String(p.image);
- return { type: "image_url" as const, image_url: { url } };
- }
- return { type: "text" as const, text: "" };
- });
+ const imageUrl = vercelMediaImageUrl(p);
+ return imageUrl ? { type: "image_url" as const, image_url: imageUrl } : null;
+ })
+ .filter((p: any): p is NonNullable<typeof p> => p !== null);
result.push({ role: "user", content: openaiParts });
}
continue;
@@ -280,8 +536,22 @@ export function vercelToOpenAI(messages: any[]): OpenAIMessage[] {
const parts = Array.isArray(msg.content) ? msg.content : [];
const textParts = parts.filter((p: any) => p.type === "text");
const toolCallParts = parts.filter((p: any) => p.type === "tool-call");
+ const hasFiles = parts.some((p: any) => p.type === "file");
- const content = textParts.length > 0 ? textParts.map((p: any) => p.text).join("") : null;
+ let content: AssistantMessage["content"];
+ if (hasFiles) {
+ // AI SDK assistant content may carry generated files; keep them as ordered parts.
+ content = parts
+ .filter((p: any) => p.type === "text" || p.type === "file")
+ .map((p: any) => {
+ if (p.type === "text") return { type: "text" as const, text: p.text };
+ const imageUrl = vercelMediaImageUrl(p);
+ return imageUrl ? { type: "image_url" as const, image_url: imageUrl } : null;
+ })
+ .filter((p: any): p is NonNullable<typeof p> => p !== null);
+ } else {
+ content = textParts.length > 0 ? textParts.map((p: any) => p.text).join("") : null;
+ }
const openaiMsg: AssistantMessage = { role: "assistant", content };
if (toolCallParts.length > 0) {
@@ -343,7 +613,7 @@ export function openAIToVercel(messages: OpenAIMessage[]): any[] {
} else if (Array.isArray(msg.content)) {
const parts = msg.content.map((p) => {
if (p.type === "text") return { type: "text", text: p.text };
- if (p.type === "image_url") return { type: "image", image: new URL(p.image_url.url) };
+ if (p.type === "image_url") return vercelMediaPart(p.image_url);
return { type: "text", text: "" };
});
result.push({ role: "user", content: parts });
@@ -353,7 +623,14 @@ export function openAIToVercel(messages: OpenAIMessage[]): any[] {
if (msg.role === "assistant") {
const parts: any[] = [];
- if (msg.content) parts.push({ type: "text", text: msg.content });
+ if (typeof msg.content === "string") {
+ if (msg.content) parts.push({ type: "text", text: msg.content });
+ } else if (Array.isArray(msg.content)) {
+ for (const p of msg.content) {
+ if (p.type === "text") parts.push({ type: "text", text: p.text });
+ else if (p.type === "image_url") parts.push(vercelMediaPart(p.image_url));
+ }
+ }
if (msg.tool_calls) {
for (const tc of msg.tool_calls) {
toolNames.set(tc.id, tc.function.name);
@@ -480,8 +757,22 @@ export function geminiToOpenAI(messages: any[]): OpenAIMessage[] {
if (role === "assistant") {
const textParts = parts.filter((p: any) => p.text !== undefined);
const funcCalls = parts.filter((p: any) => p.functionCall);
+ const hasMedia = parts.some((p: any) => p.inlineData || p.fileData);
- const content = textParts.length > 0 ? textParts.map((p: any) => p.text).join("\n") : null;
+ let content: AssistantMessage["content"];
+ if (hasMedia) {
+ // Keep generated media (e.g. image output) as ordered parts.
+ content = parts
+ .filter((p: any) => p.text !== undefined || p.inlineData || p.fileData)
+ .map((p: any) => {
+ if (p.text !== undefined) return { type: "text" as const, text: p.text };
+ const imageUrl = geminiMediaImageUrl(p);
+ return imageUrl ? { type: "image_url" as const, image_url: imageUrl } : null;
+ })
+ .filter((p: any): p is NonNullable<typeof p> => p !== null);
+ } else {
+ content = textParts.length > 0 ? textParts.map((p: any) => p.text).join("\n") : null;
+ }
const openaiMsg: AssistantMessage = { role: "assistant", content };
if (funcCalls.length > 0) {
@@ -533,7 +824,14 @@ export function openAIToGemini(messages: OpenAIMessage[]): any[] {
if (msg.role === "assistant") {
const parts: any[] = [];
- if (msg.content) parts.push({ text: msg.content });
+ if (typeof msg.content === "string") {
+ if (msg.content) parts.push({ text: msg.content });
+ } else if (Array.isArray(msg.content)) {
+ for (const g of groupAssistantParts(msg.content)) {
+ if ("text" in g) parts.push({ text: g.text });
+ else parts.push(geminiMediaPart(g.image_url));
+ }
+ }
if (msg.tool_calls) {
for (const tc of msg.tool_calls) {
parts.push({
diff --git a/sdk/typescript/test/adapters/vercel-ai.test.ts b/sdk/typescript/test/adapters/vercel-ai.test.ts
index 82959a346..16581a494 100644
--- a/sdk/typescript/test/adapters/vercel-ai.test.ts
+++ b/sdk/typescript/test/adapters/vercel-ai.test.ts
@@ -1,5 +1,5 @@
import { describe, it, expect, vi, beforeEach } from "vitest";
-import type { LanguageModelV3 } from "@ai-sdk/provider";
+import type { LanguageModelV3, LanguageModelV4FilePart, LanguageModelV4Prompt } from "@ai-sdk/provider";
import {
headroomMiddleware,
compressVercelMessages,
@@ -134,6 +134,149 @@ describe("headroomMiddleware", () => {
const body = JSON.parse(mockFetch.mock.calls[0][1].body);
expect(body.model).toBe("claude-sonnet-4-5-20250929");
});
+
+ it("keeps a provider-prompt file part through compression", async () => {
+ // The proxy only compresses text and passes the media part back untouched.
+ mockFetch.mockResolvedValueOnce(
+ mockCompressResponse([
+ {
+ role: "user",
+ content: [
+ { type: "text", text: "compressed" },
+ { type: "image_url", image_url: { url: "data:image/png;base64,iVBORw==", part: "file" } },
+ ],
+ },
+ ]),
+ );
+ const middleware = headroomMiddleware({ baseUrl: "http://localhost:8787" });
+ const out = await middleware.transformParams({
+ params: {
+ prompt: [
+ {
+ role: "user",
+ content: [
+ { type: "text", text: "what is this?" },
+ { type: "file", mediaType: "image/png", data: new Uint8Array([137, 80, 78, 71]) },
+ ],
+ },
+ ],
+ },
+ model: {},
+ type: "generate",
+ });
+
+ const body = JSON.parse(mockFetch.mock.calls[0][1].body);
+ expect(body.messages[0].content[1]).toEqual({
+ type: "image_url",
+ image_url: { url: "data:image/png;base64,iVBORw==", part: "file" },
+ });
+ expect(out.prompt[0].content[1]).toEqual({ type: "file", mediaType: "image/png", data: "iVBORw==" });
+ });
+
+ it("keeps an assistant file part (generated media) through compression", async () => {
+ mockFetch.mockResolvedValueOnce(
+ mockCompressResponse([
+ { role: "user", content: "draw a cat" },
+ {
+ role: "assistant",
+ content: [
+ { type: "text", text: "generated image" },
+ { type: "image_url", image_url: { url: "data:image/png;base64,iVBORw==", part: "file" } },
+ ],
+ },
+ ]),
+ );
+ const middleware = headroomMiddleware({ baseUrl: "http://localhost:8787" });
+ const out = await middleware.transformParams({
+ params: {
+ prompt: [
+ { role: "user", content: [{ type: "text", text: "draw a cat" }] },
+ {
+ role: "assistant",
+ content: [
+ { type: "text", text: "generated image" },
+ { type: "file", mediaType: "image/png", data: new Uint8Array([137, 80, 78, 71]) },
+ ],
+ },
+ ],
+ },
+ model: {},
+ type: "generate",
+ });
+
+ const body = JSON.parse(mockFetch.mock.calls[0][1].body);
+ expect(body.messages[1]).toEqual({
+ role: "assistant",
+ content: [
+ { type: "text", text: "generated image" },
+ { type: "image_url", image_url: { url: "data:image/png;base64,iVBORw==", part: "file" } },
+ ],
+ });
+ expect(out.prompt[1]).toEqual({
+ role: "assistant",
+ content: [
+ { type: "text", text: "generated image" },
+ { type: "file", mediaType: "image/png", data: "iVBORw==" },
+ ],
+ });
+ });
+
+ it("round-trips a LanguageModelV4 file part (tagged data) through transformParams", async () => {
+ const filePart: LanguageModelV4FilePart = {
+ type: "file",
+ mediaType: "image/png",
+ data: { type: "data", data: new Uint8Array([137, 80, 78, 71]) },
+ };
+ const prompt: LanguageModelV4Prompt = [{ role: "user", content: [{ type: "text", text: "what is this?" }, filePart] }];
+ // The proxy keeps the media part and compresses only the text.
+ mockFetch.mockResolvedValueOnce(
+ mockCompressResponse([
+ {
+ role: "user",
+ content: [
+ { type: "text", text: "compressed" },
+ { type: "image_url", image_url: { url: "data:image/png;base64,iVBORw==", part: "file", data_tag: "data" } },
+ ],
+ },
+ ]),
+ );
+ const middleware = headroomMiddleware({ baseUrl: "http://localhost:8787" });
+ const out = await middleware.transformParams({ params: { prompt }, model: { specificationVersion: "v4" }, type: "generate" });
+
+ const body = JSON.parse(mockFetch.mock.calls[0][1].body);
+ expect(body.messages[0].content[1]).toEqual({
+ type: "image_url",
+ image_url: { url: "data:image/png;base64,iVBORw==", part: "file", data_tag: "data" },
+ });
+ const restored: LanguageModelV4FilePart = out.prompt[0].content[1];
+ expect(restored).toEqual({ type: "file", mediaType: "image/png", data: { type: "data", data: "iVBORw==" } });
+ });
+
+ it("passes a provider file reference through transformParams untouched", async () => {
+ const filePart: LanguageModelV4FilePart = {
+ type: "file",
+ mediaType: "application/pdf",
+ data: { type: "reference", reference: { openai: "file-abc" } },
+ };
+ mockFetch.mockResolvedValueOnce(
+ mockCompressResponse([
+ {
+ role: "user",
+ content: [
+ { type: "text", text: "compressed" },
+ { type: "image_url", image_url: { url: "data:application/pdf;base64,", part: "file", data_tag: "reference", reference: { openai: "file-abc" } } },
+ ],
+ },
+ ]),
+ );
+ const middleware = headroomMiddleware({ baseUrl: "http://localhost:8787" });
+ const out = await middleware.transformParams({
+ params: { prompt: [{ role: "user", content: [{ type: "text", text: "summarise" }, filePart] }] },
+ model: { specificationVersion: "v4" },
+ type: "generate",
+ });
+ expect(out.prompt[0].content[1]).toEqual(filePart);
+ });
});
describe("compressVercelMessages", () => {
@@ -205,6 +348,22 @@ describe("compressVercelMessages", () => {
expect(result.messages[1].content[0].toolName).toBe("getWeather");
});
+
+ it("does not throw on byte image data", async () => {
+ mockFetch.mockResolvedValueOnce(
+ mockCompressResponse([
+ { role: "user", content: [{ type: "image_url", image_url: { url: "data:image/png;base64,iVBORw==" } }] },
+ ]),
+ );
+ await expect(
+ compressVercelMessages(
+ [{ role: "user", content: [{ type: "image", image: new Uint8Array([137, 80, 78, 71]), mediaType: "image/png" }] }],
+ { baseUrl: "http://localhost:8787" },
+ ),
+ ).resolves.toMatchObject({
+ messages: [{ role: "user", content: [{ type: "image", image: "iVBORw==", mediaType: "image/png" }] }],
+ });
+ });
});
describe("withHeadroom", () => {
diff --git a/sdk/typescript/test/compress.test.ts b/sdk/typescript/test/compress.test.ts
index 0bddcce16..49ec2d934 100644
--- a/sdk/typescript/test/compress.test.ts
+++ b/sdk/typescript/test/compress.test.ts
@@ -135,3 +135,70 @@ describe("compress()", () => {
expect(body.config.target_ratio).toBe(0.25);
});
});
+
+describe("compress() autodetects Vercel media-only conversations", () => {
+ beforeEach(() => {
+ mockFetch.mockReset();
+ });
+
+ it("sends binary file parts as image_url data URIs and restores Vercel file parts", async () => {
+ mockFetch.mockResolvedValueOnce(
+ okResponse({
+ messages: [
+ {
+ role: "user",
+ content: [
+ { type: "text", text: "compressed" },
+ { type: "image_url", image_url: { url: "data:image/png;base64,iVBORw==", part: "file" } },
+ ],
+ },
+ ],
+ }),
+ );
+
+ const result = await compress(
+ [
+ {
+ role: "user",
+ content: [
+ { type: "text", text: "what is this?" },
+ { type: "file", mediaType: "image/png", data: new Uint8Array([137, 80, 78, 71]) },
+ ],
+ },
+ ],
+ { baseUrl: "http://localhost:8787" },
+ );
+
+ const body = JSON.parse(mockFetch.mock.calls[0][1].body);
+ expect(body.messages[0].content[1]).toEqual({
+ type: "image_url",
+ image_url: { url: "data:image/png;base64,iVBORw==", part: "file" },
+ });
+ expect(JSON.stringify(body)).not.toContain('"0":137');
+ expect(result.messages[0].content[1]).toEqual({ type: "file", mediaType: "image/png", data: "iVBORw==" });
+ });
+});
+
+describe("compress() keeps Anthropic text documents through the public path", () => {
+ beforeEach(() => {
+ mockFetch.mockReset();
+ });
+
+ it("sends a document-only turn as a media part and restores the document block", async () => {
+ const doc = {
+ type: "document",
+ source: { type: "text", media_type: "text/plain", data: "Critical contract: do not delete account 123" },
+ title: "Contract",
+ citations: { enabled: true },
+ };
+ // The proxy passes non-text parts through unchanged; echo the request's messages back.
+ mockFetch.mockImplementationOnce(async (_url: string, init: any) => okResponse({ messages: JSON.parse(init.body).messages }));
+
+ const result = await compress([{ role: "user", content: [doc] }], { baseUrl: "http://localhost:8787" });
+
+ const body = JSON.parse(mockFetch.mock.calls[0][1].body);
+ expect(body.messages).toHaveLength(1);
+ expect(body.messages[0].content[0].type).toBe("image_url");
+ expect(result.messages).toEqual([{ role: "user", content: [doc] }]);
+ });
+});
diff --git a/sdk/typescript/test/types.test-d.ts b/sdk/typescript/test/types.test-d.ts
index 8b7a47ccb..63b1a7d75 100644
--- a/sdk/typescript/test/types.test-d.ts
+++ b/sdk/typescript/test/types.test-d.ts
@@ -66,9 +66,9 @@ describe("Message types", () => {
>();
});
- it("AssistantMessage content can be string or null", () => {
+ it("AssistantMessage content can be a string, content parts, or null", () => {
expectTypeOf<AssistantMessage["role"]>().toEqualTypeOf<"assistant">();
- expectTypeOf<AssistantMessage["content"]>().toEqualTypeOf<string | null>();
+ expectTypeOf<AssistantMessage["content"]>().toEqualTypeOf<string | ContentPart[] | null>();
});
it("AssistantMessage tool_calls is optional", () => {
diff --git a/sdk/typescript/test/utils/detect-format.test.ts b/sdk/typescript/test/utils/detect-format.test.ts
index d4f49c482..d74bb8216 100644
--- a/sdk/typescript/test/utils/detect-format.test.ts
+++ b/sdk/typescript/test/utils/detect-format.test.ts
@@ -131,3 +131,57 @@ describe("detectFormat", () => {
expect(detectFormat(messages)).toBe("vercel");
});
});
+
+describe("detectFormat: Vercel media parts", () => {
+ it("detects a media-only Vercel turn with a file part", () => {
+ expect(
+ detectFormat([
+ {
+ role: "user",
+ content: [
+ { type: "text", text: "what is this?" },
+ { type: "file", mediaType: "image/png", data: new Uint8Array([137, 80, 78, 71]) },
+ ],
+ },
+ ]),
+ ).toBe("vercel");
+ });
+
+ it("detects a media-only Vercel turn with an image part", () => {
+ expect(
+ detectFormat([
+ { role: "user", content: [{ type: "image", image: new Uint8Array([137, 80, 78, 71]), mediaType: "image/png" }] },
+ ]),
+ ).toBe("vercel");
+ });
+
+ it("still treats OpenAI image_url parts as openai", () => {
+ expect(
+ detectFormat([{ role: "user", content: [{ type: "image_url", image_url: { url: "https://example.com/a.png" } }] }]),
+ ).toBe("openai");
+ });
+
+ it("still treats Anthropic image blocks (with a source) as anthropic", () => {
+ expect(
+ detectFormat([
+ { role: "user", content: [{ type: "image", source: { type: "base64", media_type: "image/png", data: "AAAA" } }] },
+ ]),
+ ).toBe("anthropic");
+ });
+});
+
+describe("detectFormat: Anthropic document blocks", () => {
+ it("detects a turn whose only Anthropic marker is a document block", () => {
+ expect(
+ detectFormat([
+ {
+ role: "assistant",
+ content: [
+ { type: "text", text: "Report" },
+ { type: "document", source: { type: "base64", media_type: "application/pdf", data: "JVBERi0=" } },
+ ],
+ },
+ ]),
+ ).toBe("anthropic");
+ });
+});
diff --git a/sdk/typescript/test/utils/format.test.ts b/sdk/typescript/test/utils/format.test.ts
index 377707246..bb23666c5 100644
--- a/sdk/typescript/test/utils/format.test.ts
+++ b/sdk/typescript/test/utils/format.test.ts
@@ -6,6 +6,9 @@ import {
openAIToAnthropic,
geminiToOpenAI,
openAIToGemini,
+ fromOpenAI,
+ detectFormat,
+ toOpenAI,
} from "../../src/utils/format.js";
import type { OpenAIMessage } from "../../src/types.js";
@@ -838,3 +841,621 @@ describe("openAIToVercel tool names", () => {
});
});
+describe("vercelToOpenAI media parts", () => {
+ it("converts a provider-prompt file part (image bytes) to an image_url data URI", () => {
+ const result = vercelToOpenAI([
+ {
+ role: "user",
+ content: [
+ { type: "text", text: "what is this?" },
+ { type: "file", mediaType: "image/png", data: new Uint8Array([137, 80, 78, 71]) },
+ ],
+ },
+ ]);
+ expect(result).toEqual([
+ {
+ role: "user",
+ content: [
+ { type: "text", text: "what is this?" },
+ { type: "image_url", image_url: { url: "data:image/png;base64,iVBORw==", part: "file" } },
+ ],
+ },
+ ]);
+ });
+
+ it("keeps a file-only turn (pdf) and carries its filename", () => {
+ const result = vercelToOpenAI([
+ { role: "user", content: [{ type: "file", mediaType: "application/pdf", data: "JVBERi0=", filename: "a.pdf" }] },
+ ]);
+ expect(result).toEqual([
+ {
+ role: "user",
+ content: [
+ { type: "image_url", image_url: { url: "data:application/pdf;base64,JVBERi0=", part: "file", filename: "a.pdf" } },
+ ],
+ },
+ ]);
+ });
+
+ it("encodes a Uint8Array image as a data URI instead of String(bytes)", () => {
+ const result = vercelToOpenAI([
+ { role: "user", content: [{ type: "image", image: new Uint8Array([137, 80, 78, 71]), mediaType: "image/png" }] },
+ ]);
+ expect(result).toEqual([
+ { role: "user", content: [{ type: "image_url", image_url: { url: "data:image/png;base64,iVBORw==" } }] },
+ ]);
+ });
+
+ it("treats a bare base64 string image as data, not a URL", () => {
+ const result = vercelToOpenAI([
+ { role: "user", content: [{ type: "image", image: "iVBORw==", mediaType: "image/png" }] },
+ ]);
+ expect(result).toEqual([
+ { role: "user", content: [{ type: "image_url", image_url: { url: "data:image/png;base64,iVBORw==" } }] },
+ ]);
+ });
+
+ it("carries the MIME type of a URL-backed file part", () => {
+ const result = vercelToOpenAI([
+ { role: "user", content: [{ type: "file", mediaType: "application/pdf", data: new URL("https://example.com/a.pdf") }] },
+ ]);
+ expect(result).toEqual([
+ {
+ role: "user",
+ content: [
+ { type: "image_url", image_url: { url: "https://example.com/a.pdf", part: "file", mime_type: "application/pdf" } },
+ ],
+ },
+ ]);
+ });
+});
+
+describe("openAIToVercel media parts", () => {
+ it("restores a data URI marked part=file as a file part with base64 data", () => {
+ const msgs = [
+ {
+ role: "user",
+ content: [
+ { type: "image_url", image_url: { url: "data:application/pdf;base64,JVBERi0=", part: "file", filename: "a.pdf" } },
+ ],
+ },
+ ] as OpenAIMessage[];
+ expect(openAIToVercel(msgs)).toEqual([
+ { role: "user", content: [{ type: "file", mediaType: "application/pdf", data: "JVBERi0=", filename: "a.pdf" }] },
+ ]);
+ });
+
+ it("restores an image data URI as an image part with base64 data and mediaType", () => {
+ const msgs: OpenAIMessage[] = [
+ { role: "user", content: [{ type: "image_url", image_url: { url: "data:image/png;base64,iVBORw==" } }] },
+ ];
+ expect(openAIToVercel(msgs)).toEqual([
+ { role: "user", content: [{ type: "image", image: "iVBORw==", mediaType: "image/png" }] },
+ ]);
+ });
+
+ it("restores a URL-backed file part from the carried MIME type", () => {
+ const msgs = [
+ {
+ role: "user",
+ content: [
+ { type: "image_url", image_url: { url: "https://example.com/a.pdf", part: "file", mime_type: "application/pdf" } },
+ ],
+ },
+ ] as OpenAIMessage[];
+ const part = openAIToVercel(msgs)[0].content[0];
+ expect(part.type).toBe("file");
+ expect(part.mediaType).toBe("application/pdf");
+ expect(part.data).toBeInstanceOf(URL);
+ expect(part.data.toString()).toBe("https://example.com/a.pdf");
+ });
+
+ it("never throws on an unparseable url and keeps it as a string", () => {
+ const msgs: OpenAIMessage[] = [
+ { role: "user", content: [{ type: "image_url", image_url: { url: "137,80,78,71" } }] },
+ ];
+ expect(() => openAIToVercel(msgs)).not.toThrow();
+ expect(openAIToVercel(msgs)[0].content[0]).toEqual({ type: "image", image: "137,80,78,71" });
+ });
+});
+
+describe("round-trip: vercelToOpenAI then openAIToVercel (media)", () => {
+ it("round-trips a provider-prompt text+file turn into a valid file part", () => {
+ const original = [
+ {
+ role: "user",
+ content: [
+ { type: "text", text: "what is this?" },
+ { type: "file", mediaType: "image/png", data: new Uint8Array([137, 80, 78, 71]) },
+ ],
+ },
+ ];
+ expect(openAIToVercel(vercelToOpenAI(original))).toEqual([
+ {
+ role: "user",
+ content: [
+ { type: "text", text: "what is this?" },
+ { type: "file", mediaType: "image/png", data: "iVBORw==" },
+ ],
+ },
+ ]);
+ });
+
+ it("round-trips a byte image without throwing", () => {
+ const original = [
+ { role: "user", content: [{ type: "image", image: new Uint8Array([137, 80, 78, 71]), mediaType: "image/png" }] },
+ ];
+ expect(openAIToVercel(vercelToOpenAI(original))).toEqual([
+ { role: "user", content: [{ type: "image", image: "iVBORw==", mediaType: "image/png" }] },
+ ]);
+ });
+});
+
+describe("vercelToOpenAI / openAIToVercel assistant file parts", () => {
+ it("keeps an assistant file part (generated image) alongside its text", () => {
+ const result = vercelToOpenAI([
+ {
+ role: "assistant",
+ content: [
+ { type: "text", text: "generated image" },
+ { type: "file", mediaType: "image/png", data: new Uint8Array([137, 80, 78, 71]), filename: "cat.png" },
+ ],
+ },
+ ]);
+ expect(result).toEqual([
+ {
+ role: "assistant",
+ content: [
+ { type: "text", text: "generated image" },
+ { type: "image_url", image_url: { url: "data:image/png;base64,iVBORw==", part: "file", filename: "cat.png" } },
+ ],
+ },
+ ]);
+ });
+
+ it("restores an assistant content array as text and file parts with the assistant role", () => {
+ const msgs = [
+ {
+ role: "assistant",
+ content: [
+ { type: "text", text: "generated image" },
+ { type: "image_url", image_url: { url: "data:image/png;base64,iVBORw==", part: "file", filename: "cat.png" } },
+ ],
+ },
+ ] as OpenAIMessage[];
+ expect(openAIToVercel(msgs)).toEqual([
+ {
+ role: "assistant",
+ content: [
+ { type: "text", text: "generated image" },
+ { type: "file", mediaType: "image/png", data: "iVBORw==", filename: "cat.png" },
+ ],
+ },
+ ]);
+ });
+
+ it("round-trips an assistant text+file turn", () => {
+ const original = [
+ {
+ role: "assistant",
+ content: [
+ { type: "text", text: "generated image" },
+ { type: "file", mediaType: "image/png", data: new Uint8Array([137, 80, 78, 71]) },
+ ],
+ },
+ ];
+ expect(openAIToVercel(vercelToOpenAI(original))).toEqual([
+ {
+ role: "assistant",
+ content: [
+ { type: "text", text: "generated image" },
+ { type: "file", mediaType: "image/png", data: "iVBORw==" },
+ ],
+ },
+ ]);
+ });
+
+ it("leaves text-only assistant turns as a string (backward compat)", () => {
+ expect(
+ vercelToOpenAI([{ role: "assistant", content: [{ type: "text", text: "a" }, { type: "text", text: "b" }] }]),
+ ).toEqual([{ role: "assistant", content: "ab" }]);
+ });
+});
+
+describe("assistant media across formats", () => {
+ it("openAIToAnthropic keeps an assistant image as an image block after the text", () => {
+ const msgs = [
+ {
+ role: "assistant",
+ content: [
+ { type: "text", text: "generated image" },
+ { type: "image_url", image_url: { url: "data:image/png;base64,iVBORw==" } },
+ ],
+ },
+ ] as OpenAIMessage[];
+ expect(openAIToAnthropic(msgs)).toEqual([
+ {
+ role: "assistant",
+ content: [
+ { type: "text", text: "generated image" },
+ { type: "image", source: { type: "base64", media_type: "image/png", data: "iVBORw==" } },
+ ],
+ },
+ ]);
+ });
+
+ it("openAIToAnthropic maps an assistant PDF to a document block (base64 and url sources)", () => {
+ const msgs = [
+ {
+ role: "assistant",
+ content: [
+ { type: "image_url", image_url: { url: "data:application/pdf;base64,JVBERi0=", part: "file" } },
+ { type: "image_url", image_url: { url: "https://example.com/a.pdf", part: "file", mime_type: "application/pdf" } },
+ ],
+ },
+ ] as OpenAIMessage[];
+ expect(openAIToAnthropic(msgs)).toEqual([
+ {
+ role: "assistant",
+ content: [
+ { type: "document", source: { type: "base64", media_type: "application/pdf", data: "JVBERi0=" } },
+ { type: "document", source: { type: "url", url: "https://example.com/a.pdf" } },
+ ],
+ },
+ ]);
+ });
+
+ it("openAIToAnthropic leaves a text-only assistant array as a string (backward compat)", () => {
+ const msgs = [{ role: "assistant", content: [{ type: "text", text: "a" }] }] as OpenAIMessage[];
+ expect(openAIToAnthropic(msgs)).toEqual([{ role: "assistant", content: "a" }]);
+ });
+
+ it("openAIToGemini keeps an assistant image as an inlineData part after the text", () => {
+ const msgs = [
+ {
+ role: "assistant",
+ content: [
+ { type: "text", text: "generated image" },
+ { type: "image_url", image_url: { url: "data:image/png;base64,iVBORw==" } },
+ ],
+ },
+ ] as OpenAIMessage[];
+ expect(openAIToGemini(msgs)).toEqual([
+ { role: "model", parts: [{ text: "generated image" }, { inlineData: { mimeType: "image/png", data: "iVBORw==" } }] },
+ ]);
+ });
+
+ it("openAIToGemini maps a URL-backed assistant file to fileData with its MIME type", () => {
+ const msgs = [
+ {
+ role: "assistant",
+ content: [{ type: "image_url", image_url: { url: "https://example.com/a.pdf", part: "file", mime_type: "application/pdf" } }],
+ },
+ ] as OpenAIMessage[];
+ expect(openAIToGemini(msgs)).toEqual([
+ { role: "model", parts: [{ fileData: { mimeType: "application/pdf", fileUri: "https://example.com/a.pdf" } }] },
+ ]);
+ });
+
+ it("anthropicToOpenAI keeps assistant image and document blocks as ordered parts", () => {
+ const result = anthropicToOpenAI([
+ {
+ role: "assistant",
+ content: [
+ { type: "text", text: "generated image" },
+ { type: "image", source: { type: "base64", media_type: "image/png", data: "iVBORw==" } },
+ { type: "document", source: { type: "base64", media_type: "application/pdf", data: "JVBERi0=" } },
+ ],
+ },
+ ]);
+ expect(result).toEqual([
+ {
+ role: "assistant",
+ content: [
+ { type: "text", text: "generated image" },
+ { type: "image_url", image_url: { url: "data:image/png;base64,iVBORw==" } },
+ { type: "image_url", image_url: { url: "data:application/pdf;base64,JVBERi0=" } },
+ ],
+ },
+ ]);
+ });
+
+ it("geminiToOpenAI keeps model-turn inlineData as an ordered image_url part", () => {
+ const result = geminiToOpenAI([
+ { role: "model", parts: [{ text: "generated image" }, { inlineData: { mimeType: "image/png", data: "iVBORw==" } }] },
+ ]);
+ expect(result).toEqual([
+ {
+ role: "assistant",
+ content: [
+ { type: "text", text: "generated image" },
+ { type: "image_url", image_url: { url: "data:image/png;base64,iVBORw==" } },
+ ],
+ },
+ ]);
+ });
+
+ it("round-trips an Anthropic assistant text+image turn exactly", () => {
+ const original = [
+ {
+ role: "assistant",
+ content: [
+ { type: "text", text: "generated image" },
+ { type: "image", source: { type: "base64", media_type: "image/png", data: "iVBORw==" } },
+ ],
+ },
+ ];
+ expect(openAIToAnthropic(anthropicToOpenAI(original))).toEqual(original);
+ });
+
+ it("round-trips a Gemini model text+inlineData turn exactly", () => {
+ const original = [
+ { role: "model", parts: [{ text: "generated image" }, { inlineData: { mimeType: "image/png", data: "iVBORw==" } }] },
+ ];
+ expect(openAIToGemini(geminiToOpenAI(original))).toEqual(original);
+ });
+
+ it("carries a Vercel assistant file into Gemini and Anthropic media parts", () => {
+ const openai = vercelToOpenAI([
+ { role: "assistant", content: [{ type: "text", text: "generated image" }, { type: "file", mediaType: "image/png", data: "iVBORw==" }] },
+ ]);
+ expect(fromOpenAI(openai, "gemini")).toEqual([
+ { role: "model", parts: [{ text: "generated image" }, { inlineData: { mimeType: "image/png", data: "iVBORw==" } }] },
+ ]);
+ expect(fromOpenAI(openai, "anthropic")).toEqual([
+ {
+ role: "assistant",
+ content: [
+ { type: "text", text: "generated image" },
+ { type: "image", source: { type: "base64", media_type: "image/png", data: "iVBORw==" } },
+ ],
+ },
+ ]);
+ });
+});
+
+describe("Vercel V4 tagged file data", () => {
+ it("carries a data-tagged file part and restores the tagged shape", () => {
+ const original = [
+ {
+ role: "user",
+ content: [
+ { type: "text", text: "what is this?" },
+ { type: "file", mediaType: "image/png", data: { type: "data", data: new Uint8Array([137, 80, 78, 71]) } },
+ ],
+ },
+ ];
+ const openai = vercelToOpenAI(original);
+ expect(openai[0].content[1]).toEqual({
+ type: "image_url",
+ image_url: { url: "data:image/png;base64,iVBORw==", part: "file", data_tag: "data" },
+ });
+ expect(openAIToVercel(openai)).toEqual([
+ {
+ role: "user",
+ content: [
+ { type: "text", text: "what is this?" },
+ { type: "file", mediaType: "image/png", data: { type: "data", data: "iVBORw==" } },
+ ],
+ },
+ ]);
+ });
+
+ it("carries a url-tagged file part and restores a URL object inside the tag", () => {
+ const original = [
+ { role: "user", content: [{ type: "file", mediaType: "application/pdf", data: { type: "url", url: new URL("https://example.com/a.pdf") } }] },
+ ];
+ const openai = vercelToOpenAI(original);
+ expect(openai[0].content[0]).toEqual({
+ type: "image_url",
+ image_url: { url: "https://example.com/a.pdf", part: "file", mime_type: "application/pdf", data_tag: "url" },
+ });
+ const part = openAIToVercel(openai)[0].content[0];
+ expect(part.type).toBe("file");
+ expect(part.mediaType).toBe("application/pdf");
+ expect(part.data.type).toBe("url");
+ expect(part.data.url).toBeInstanceOf(URL);
+ expect(part.data.url.toString()).toBe("https://example.com/a.pdf");
+ });
+
+ it("round-trips a text-tagged file part", () => {
+ const original = [
+ { role: "user", content: [{ type: "file", mediaType: "text/plain", data: { type: "text", text: "hello, file" }, filename: "a.txt" }] },
+ ];
+ expect(openAIToVercel(vercelToOpenAI(original))).toEqual(original);
+ });
+
+ it("preserves a provider reference (tagged and bare) instead of dropping it", () => {
+ const tagged = [
+ { role: "user", content: [{ type: "file", mediaType: "application/pdf", data: { type: "reference", reference: { openai: "file-abc" } } }] },
+ ];
+ expect(openAIToVercel(vercelToOpenAI(tagged))).toEqual(tagged);
+ const bare = [{ role: "user", content: [{ type: "file", mediaType: "application/pdf", data: { openai: "file-abc" } }] }];
+ expect(openAIToVercel(vercelToOpenAI(bare))).toEqual(bare);
+ });
+});
+
+describe("assistant text separators across formats", () => {
+ it("openAIToAnthropic joins adjacent assistant text parts with a newline into one block", () => {
+ const msgs = [
+ { role: "assistant", content: [{ type: "text", text: "first" }, { type: "text", text: "second" }] },
+ ] as OpenAIMessage[];
+ expect(openAIToAnthropic(msgs)).toEqual([{ role: "assistant", content: "first\nsecond" }]);
+ });
+
+ it("openAIToAnthropic keeps text parts separate when the array carries media", () => {
+ const msgs = [
+ {
+ role: "assistant",
+ content: [
+ { type: "text", text: "a" },
+ { type: "text", text: "b" },
+ { type: "image_url", image_url: { url: "data:image/png;base64,iVBORw==" } },
+ { type: "text", text: "c" },
+ ],
+ },
+ ] as OpenAIMessage[];
+ expect(openAIToAnthropic(msgs)).toEqual([
+ {
+ role: "assistant",
+ content: [
+ { type: "text", text: "a" },
+ { type: "text", text: "b" },
+ { type: "image", source: { type: "base64", media_type: "image/png", data: "iVBORw==" } },
+ { type: "text", text: "c" },
+ ],
+ },
+ ]);
+ });
+
+ it("round-trips Gemini and Anthropic assistant turns with two adjacent text parts before media exactly", () => {
+ const gemini = [
+ { role: "model", parts: [{ text: "Hello " }, { text: "world" }, { inlineData: { mimeType: "image/png", data: "iVBORw==" } }] },
+ ];
+ expect(openAIToGemini(geminiToOpenAI(gemini))).toEqual(gemini);
+ const anthropic = [
+ {
+ role: "assistant",
+ content: [
+ { type: "text", text: "Hello " },
+ { type: "text", text: "world" },
+ { type: "image", source: { type: "base64", media_type: "image/png", data: "iVBORw==" } },
+ ],
+ },
+ ];
+ expect(openAIToAnthropic(anthropicToOpenAI(anthropic))).toEqual(anthropic);
+ });
+
+ it("openAIToGemini joins adjacent assistant text parts with a newline into one part", () => {
+ const msgs = [
+ { role: "assistant", content: [{ type: "text", text: "first" }, { type: "text", text: "second" }] },
+ ] as OpenAIMessage[];
+ expect(openAIToGemini(msgs)).toEqual([{ role: "model", parts: [{ text: "first\nsecond" }] }]);
+ });
+});
+
+describe("Anthropic user document blocks", () => {
+ it("keeps a user document (PDF) beside its text as an image_url data URI", () => {
+ const result = anthropicToOpenAI([
+ {
+ role: "user",
+ content: [
+ { type: "text", text: "Summarize" },
+ { type: "document", source: { type: "base64", media_type: "application/pdf", data: "JVBERi0=" } },
+ ],
+ },
+ ]);
+ expect(result).toEqual([
+ {
+ role: "user",
+ content: [
+ { type: "text", text: "Summarize" },
+ { type: "image_url", image_url: { url: "data:application/pdf;base64,JVBERi0=" } },
+ ],
+ },
+ ]);
+ });
+
+ it("keeps a document-only user turn instead of dropping it", () => {
+ const result = anthropicToOpenAI([
+ { role: "user", content: [{ type: "document", source: { type: "url", url: "https://example.com/a.pdf" } }] },
+ ]);
+ expect(result).toEqual([
+ { role: "user", content: [{ type: "image_url", image_url: { url: "https://example.com/a.pdf", mime_type: "application/pdf" } }] },
+ ]);
+ });
+
+ it("round-trips a user text+document turn exactly through detection and conversion", () => {
+ const original = [
+ {
+ role: "user",
+ content: [
+ { type: "text", text: "Summarize" },
+ { type: "document", source: { type: "base64", media_type: "application/pdf", data: "JVBERi0=" } },
+ ],
+ },
+ ];
+ expect(detectFormat(original)).toBe("anthropic");
+ expect(fromOpenAI(toOpenAI(original), "anthropic")).toEqual(original);
+ });
+});
+
+describe("Anthropic document sources (text, content, file) and document fields", () => {
+ it("keeps a document-only user turn with a text source, title and citations through the public path", () => {
+ const input = [
+ {
+ role: "user",
+ content: [
+ {
+ type: "document",
+ source: { type: "text", media_type: "text/plain", data: "Critical contract: do not delete account 123" },
+ title: "Contract",
+ citations: { enabled: true },
+ },
+ ],
+ },
+ ];
+ expect(detectFormat(input)).toBe("anthropic");
+ const wire = toOpenAI(input);
+ expect(wire).toHaveLength(1);
+ expect((wire[0].content as any)[0].type).toBe("image_url");
+ expect(fromOpenAI(wire, "anthropic")).toEqual(input);
+ });
+
+ it("round-trips a mixed text + text-document turn with context exactly", () => {
+ const input = [
+ {
+ role: "user",
+ content: [
+ { type: "text", text: "Summarize" },
+ { type: "document", source: { type: "text", media_type: "text/plain", data: "clause 1\nclause 2" }, context: "Contract excerpt" },
+ ],
+ },
+ ];
+ expect(fromOpenAI(toOpenAI(input), "anthropic")).toEqual(input);
+ });
+
+ it("carries content-source and file-source documents verbatim", () => {
+ const input = [
+ {
+ role: "user",
+ content: [
+ { type: "document", source: { type: "content", content: [{ type: "text", text: "clause 1" }] }, title: "Doc" },
+ { type: "document", source: { type: "file", file_id: "file_abc" } },
+ ],
+ },
+ ];
+ expect(fromOpenAI(toOpenAI(input), "anthropic")).toEqual(input);
+ });
+
+ it("restores a PDF document's title, context and citations", () => {
+ const input = [
+ {
+ role: "user",
+ content: [
+ {
+ type: "document",
+ source: { type: "base64", media_type: "application/pdf", data: "JVBERi0=" },
+ title: "Report",
+ context: "Q3",
+ citations: { enabled: true },
+ },
+ ],
+ },
+ ];
+ expect(fromOpenAI(toOpenAI(input), "anthropic")).toEqual(input);
+ });
+
+ it("round-trips an assistant document with a title", () => {
+ const input = [
+ {
+ role: "assistant",
+ content: [
+ { type: "text", text: "see attached" },
+ { type: "document", source: { type: "base64", media_type: "application/pdf", data: "JVBERi0=" }, title: "Report" },
+ ],
+ },
+ ];
+ expect(fromOpenAI(toOpenAI(input), "anthropic")).toEqual(input);
+ });
+});