🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
fix(evals): skip Bedrock URL media interpolation (#18303)
Tobias Wochinger0a9ce1adc423
diff --git a/packages/shared/src/server/llm/mediaMessages.test.ts b/packages/shared/src/server/llm/mediaMessages.test.tsindex 147b2fc0bc..642bcd2112 100644--- a/packages/shared/src/server/llm/mediaMessages.test.ts+++ b/packages/shared/src/server/llm/mediaMessages.test.ts@@ -338,6 +338,65 @@ describe("compileLangfuseMediaMessages", () => {},);+ it("leaves media references as provider text for Bedrock URL transport", async () => {+ const resolver = vi.fn();++ const result = await compileLangfuseMediaMessages({+ projectId: "project-1",+ messages: [userMessage(`inspect ${imageRef}`)],+ adapter: LLMAdapter.Bedrock,+ transport: "url",+ resolveMedia: resolver,+ });++ expect(result).toEqual({+ providerMessages: [{ role: "user", content: `inspect ${imageRef}` }],+ traceMessages: [+ {+ role: "user",+ content: [+ { type: "text", text: "inspect " },+ {+ type: "file",+ data: imageRef,+ mediaType: "image/jpeg",+ },+ ],+ },+ ],+ });+ expect(resolver).not.toHaveBeenCalled();+ });++ it("sends media bytes inline for Bedrock when explicitly configured", async () => {+ const fetchMedia = vi.fn().mockResolvedValue(new Uint8Array([1, 2, 3]));++ const result = await compileLangfuseMediaMessages({+ projectId: "project-1",+ messages: [userMessage(`inspect ${imageRef}`)],+ adapter: LLMAdapter.Bedrock,+ transport: "inline",+ resolveMedia,+ fetchMedia,+ });++ expect(result.providerMessages).toEqual([+ {+ role: "user",+ content: [+ { type: "text", text: "inspect " },+ {+ type: "file",+ data: new Uint8Array([1, 2, 3]),+ mediaType: "image/jpeg",+ },+ ],+ },+ ]);+ expect(resolveMedia).toHaveBeenCalledOnce();+ expect(fetchMedia).toHaveBeenCalledOnce();+ });+it.each([[ChatMessageRole.System, ChatMessageType.System],[ChatMessageRole.Developer, ChatMessageType.Developer],diff --git a/packages/shared/src/server/llm/mediaMessages.ts b/packages/shared/src/server/llm/mediaMessages.tsindex baa4025c75..efd042de9b 100644--- a/packages/shared/src/server/llm/mediaMessages.ts+++ b/packages/shared/src/server/llm/mediaMessages.ts@@ -56,6 +56,12 @@ const EVALUATOR_MEDIA_TYPES = new Set(["video/3gpp",]);+const EVALUATOR_MEDIA_URL_UNSUPPORTED_ADAPTERS = new Set<LLMAdapter>([+ // Bedrock Converse accepts inline bytes and S3 locations, but not HTTP(S) URLs.+ // https://github.com/aws-samples/amazon-bedrock-samples/issues/684+ LLMAdapter.Bedrock,+]);+export function normalizeEvaluatorMediaType(mediaType: string) {const normalized = mediaType.trim().toLowerCase();return MEDIA_TYPE_ALIASES[normalized] ?? normalized;@@ -235,6 +241,15 @@ export async function compileLangfuseMediaMessages(params: {traceMessage: { ...message, content: traceContent },};}+ if (+ transport === "url" &&+ EVALUATOR_MEDIA_URL_UNSUPPORTED_ADAPTERS.has(params.adapter)+ ) {+ return {+ providerMessage: message,+ traceMessage: { ...message, content: traceContent },+ };+ }// Resolve and load independent media objects concurrently. Promise.all// keeps the results in source order even if downloads finish out of order.