Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

fix(evals): skip Bedrock URL media interpolation (#18303)

Tobias Wochinger0a9ce1adc423
diff --git a/packages/shared/src/server/llm/mediaMessages.test.ts b/packages/shared/src/server/llm/mediaMessages.test.ts
index 147b2fc0bc..642bcd2112 100644
--- a/packages/shared/src/server/llm/mediaMessages.test.ts
+++ b/packages/shared/src/server/llm/mediaMessages.test.ts
@@ -338,6 +338,65 @@ describe("compileLangfuseMediaMessages", () => {
},
);
+ it("leaves media references as provider text for Bedrock URL transport", async () => {
+ const resolver = vi.fn();
+
+ const result = await compileLangfuseMediaMessages({
+ projectId: "project-1",
+ messages: [userMessage(`inspect ${imageRef}`)],
+ adapter: LLMAdapter.Bedrock,
+ transport: "url",
+ resolveMedia: resolver,
+ });
+
+ expect(result).toEqual({
+ providerMessages: [{ role: "user", content: `inspect ${imageRef}` }],
+ traceMessages: [
+ {
+ role: "user",
+ content: [
+ { type: "text", text: "inspect " },
+ {
+ type: "file",
+ data: imageRef,
+ mediaType: "image/jpeg",
+ },
+ ],
+ },
+ ],
+ });
+ expect(resolver).not.toHaveBeenCalled();
+ });
+
+ it("sends media bytes inline for Bedrock when explicitly configured", async () => {
+ const fetchMedia = vi.fn().mockResolvedValue(new Uint8Array([1, 2, 3]));
+
+ const result = await compileLangfuseMediaMessages({
+ projectId: "project-1",
+ messages: [userMessage(`inspect ${imageRef}`)],
+ adapter: LLMAdapter.Bedrock,
+ transport: "inline",
+ resolveMedia,
+ fetchMedia,
+ });
+
+ expect(result.providerMessages).toEqual([
+ {
+ role: "user",
+ content: [
+ { type: "text", text: "inspect " },
+ {
+ type: "file",
+ data: new Uint8Array([1, 2, 3]),
+ mediaType: "image/jpeg",
+ },
+ ],
+ },
+ ]);
+ expect(resolveMedia).toHaveBeenCalledOnce();
+ expect(fetchMedia).toHaveBeenCalledOnce();
+ });
+
it.each([
[ChatMessageRole.System, ChatMessageType.System],
[ChatMessageRole.Developer, ChatMessageType.Developer],
diff --git a/packages/shared/src/server/llm/mediaMessages.ts b/packages/shared/src/server/llm/mediaMessages.ts
index baa4025c75..efd042de9b 100644
--- a/packages/shared/src/server/llm/mediaMessages.ts
+++ b/packages/shared/src/server/llm/mediaMessages.ts
@@ -56,6 +56,12 @@ const EVALUATOR_MEDIA_TYPES = new Set([
"video/3gpp",
]);
+const EVALUATOR_MEDIA_URL_UNSUPPORTED_ADAPTERS = new Set<LLMAdapter>([
+ // Bedrock Converse accepts inline bytes and S3 locations, but not HTTP(S) URLs.
+ // https://github.com/aws-samples/amazon-bedrock-samples/issues/684
+ LLMAdapter.Bedrock,
+]);
+
export function normalizeEvaluatorMediaType(mediaType: string) {
const normalized = mediaType.trim().toLowerCase();
return MEDIA_TYPE_ALIASES[normalized] ?? normalized;
@@ -235,6 +241,15 @@ export async function compileLangfuseMediaMessages(params: {
traceMessage: { ...message, content: traceContent },
};
}
+ if (
+ transport === "url" &&
+ EVALUATOR_MEDIA_URL_UNSUPPORTED_ADAPTERS.has(params.adapter)
+ ) {
+ return {
+ providerMessage: message,
+ traceMessage: { ...message, content: traceContent },
+ };
+ }
// Resolve and load independent media objects concurrently. Promise.all
// keeps the results in source order even if downloads finish out of order.