🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
fix(normalized-io): langfuse plugin rendering (#17820)
marliessophie754269adc08a
diff --git a/packages/shared/src/utils/normalized-io/conventions/index.ts b/packages/shared/src/utils/normalized-io/conventions/index.tsindex eea44fc4fc..093f4bb86a 100644--- a/packages/shared/src/utils/normalized-io/conventions/index.ts+++ b/packages/shared/src/utils/normalized-io/conventions/index.ts@@ -4,6 +4,7 @@ import { aiSdkProvider } from "./providers/ai-sdk";import { anthropicProvider } from "./providers/anthropic";import { geminiProvider } from "./providers/gemini";import { langchainProvider } from "./providers/langchain";+import { langfuseAgentPluginsProvider } from "./providers/langfuse-agent-plugins";import { openAiProvider } from "./providers/openai";import { otelGenaiProvider } from "./providers/otel-genai";import { pydanticAiProvider } from "./providers/pydantic-ai";@@ -38,4 +39,5 @@ export const registeredProviders: readonly IOConvention[] = [pydanticAiProvider,semanticKernelProvider,agnoProvider,+ langfuseAgentPluginsProvider,];diff --git a/packages/shared/src/utils/normalized-io/conventions/providers/anthropic/index.ts b/packages/shared/src/utils/normalized-io/conventions/providers/anthropic/index.tsindex 05a306c0ab..6b0825b5c5 100644--- a/packages/shared/src/utils/normalized-io/conventions/providers/anthropic/index.ts+++ b/packages/shared/src/utils/normalized-io/conventions/providers/anthropic/index.ts@@ -31,9 +31,9 @@ import type {/*** Anthropic Messages API convention: this module owns Anthropic's typed- * block vocabulary, its `system` request field, and the `thinking` sibling- * array. One message can still contain parts from several dialects — the- * core folds every provider cumulatively.+ * block vocabulary and its `system` request field. One message can still+ * contain parts from several dialects — the core folds every provider+ * cumulatively.*/// Anthropic `stop_reason` vocabulary -> the canonical FinishReason set.@@ -127,8 +127,12 @@ const normalizeAnthropicMcpToolCall: PartHandler = (value) =>),);-const normalizeAnthropicThinking: PartHandler = (value) =>- claimed(reasoningPart(value.thinking, optionalString(value.signature)));+const normalizeAnthropicThinking: PartHandler = (value) => {+ if (value.thinking === null || value.thinking === undefined) return unmatched;+ return claimed(+ reasoningPart(value.thinking, optionalString(value.signature)),+ );+};const normalizeAnthropicRedactedThinking: PartHandler = (value) => {const data = optionalString(value.data);@@ -235,17 +239,5 @@ export const anthropicProvider = {});return definition ? claimed(definition) : unmatched;},- collectSiblingParts: (- value: Record<string, unknown>,- _baseParts,- context,- ) => {- const parts = context.normalizePartList(- Array.isArray(value.thinking) ? value.thinking : [],- );- return parts.length > 0- ? [{ sourceKey: "thinking", slot: "after-content", parts }]- : [];- },getSystemMessage: anthropicSystemMessage,} satisfies IOConvention;diff --git a/packages/shared/src/utils/normalized-io/conventions/providers/langfuse-agent-plugins/fixtures.ts b/packages/shared/src/utils/normalized-io/conventions/providers/langfuse-agent-plugins/fixtures.tsnew file mode 100644index 0000000000..12cbbeb13c--- /dev/null+++ b/packages/shared/src/utils/normalized-io/conventions/providers/langfuse-agent-plugins/fixtures.ts@@ -0,0 +1,125 @@+import type { NormalizedIOFixture } from "../fixture-types";++const question = {+ role: "user",+ content: "Read README.md and tell me in one line what this repo is.",+};+const reasoning =+ "The user wants me to read README.md and describe the repo in one line.";+const answer = "A scratch repository for the reasoning-on-output review.";++const userTurn: NormalizedIOFixture["expected"]["messages"][number] = {+ role: "user",+ parts: [{ type: "text", text: question.content }],+ source: "input",+};++const answerThenReasoning = (+ signature?: string,+): NormalizedIOFixture["expected"]["messages"][number] => ({+ role: "assistant",+ parts: [+ { type: "text", text: answer },+ {+ type: "reasoning",+ content: {+ kind: "text",+ text: reasoning,+ ...(signature ? { signature } : {}),+ },+ },+ ],+ source: "output",+});++export const langfuseAgentPluginFixtures: NormalizedIOFixture[] = [+ {+ name: "normalizes a `content`-keyed thinking part in the `thinking` sibling array",+ spanIO: {+ input: JSON.stringify([question]),+ output: JSON.stringify({+ role: "assistant",+ content: answer,+ thinking: [+ { type: "thinking", content: reasoning, signature: "sig_sibling" },+ ],+ }),+ metadata: undefined,+ },+ expected: {+ messages: [userTurn, answerThenReasoning("sig_sibling")],+ toolDefinitions: [],+ },+ },+ {+ name: "normalizes a `content`-keyed thinking part inline in `content`",+ spanIO: {+ input: JSON.stringify([question]),+ output: JSON.stringify({+ role: "assistant",+ content: [+ { type: "thinking", content: reasoning, signature: "sig_inline" },+ { type: "text", text: answer },+ ],+ }),+ metadata: undefined,+ },+ expected: {+ messages: [+ userTurn,+ {+ role: "assistant",+ parts: [+ {+ type: "reasoning",+ content: {+ kind: "text",+ text: reasoning,+ signature: "sig_inline",+ },+ },+ { type: "text", text: answer },+ ],+ source: "output",+ },+ ],+ toolDefinitions: [],+ },+ },+ {+ name: "normalizes a thinking part carrying both `thinking` and `content`",+ spanIO: {+ input: JSON.stringify([question]),+ output: JSON.stringify({+ role: "assistant",+ content: answer,+ thinking: [+ { type: "thinking", thinking: reasoning, content: reasoning },+ ],+ }),+ metadata: undefined,+ },+ expected: {+ messages: [userTurn, answerThenReasoning()],+ toolDefinitions: [],+ },+ },+ {+ name: "normalizes an Anthropic thinking block inside a Langfuse plugin thinking sibling array",+ spanIO: {+ input: JSON.stringify([question]),+ output: JSON.stringify({+ role: "assistant",+ content: answer,+ thinking: [+ { type: "thinking", thinking: reasoning, signature: "sig_native" },+ ],+ }),+ metadata: undefined,+ },+ expected: {+ messages: [userTurn, answerThenReasoning("sig_native")],+ toolDefinitions: [],+ },+ },+];diff --git a/packages/shared/src/utils/normalized-io/conventions/providers/langfuse-agent-plugins/index.ts b/packages/shared/src/utils/normalized-io/conventions/providers/langfuse-agent-plugins/index.tsnew file mode 100644index 0000000000..837ea661f7--- /dev/null+++ b/packages/shared/src/utils/normalized-io/conventions/providers/langfuse-agent-plugins/index.ts@@ -0,0 +1,31 @@+import { claimed, unmatched } from "../..";+import { optionalString } from "../../../core/utils/json";+import { reasoningPart } from "../../../core/normalize/message-parts/reasoning";+import type { IOConvention, PartHandler } from "../../io-convention";++const normalizeChatMlThinking: PartHandler = (value) => {+ if (value.thinking !== undefined && value.thinking !== null) return unmatched;+ return claimed(+ reasoningPart(+ value.content ?? value.summary,+ optionalString(value.signature),+ ),+ );+};++export const langfuseAgentPluginsProvider = {+ name: "langfuse-agent-plugins",+ typedParts: { thinking: normalizeChatMlThinking },+ collectSiblingParts: (+ value: Record<string, unknown>,+ _baseParts,+ context,+ ) => {+ const parts = context.normalizePartList(+ Array.isArray(value.thinking) ? value.thinking : [],+ );+ return parts.length > 0+ ? [{ sourceKey: "thinking", slot: "after-content", parts }]+ : [];+ },+} satisfies IOConvention;diff --git a/packages/shared/src/utils/normalized-io/conventions/providers/langfuse-agent-plugins/langfuse-agent-plugins.test.ts b/packages/shared/src/utils/normalized-io/conventions/providers/langfuse-agent-plugins/langfuse-agent-plugins.test.tsnew file mode 100644index 0000000000..ab4ff6f6e8--- /dev/null+++ b/packages/shared/src/utils/normalized-io/conventions/providers/langfuse-agent-plugins/langfuse-agent-plugins.test.ts@@ -0,0 +1,13 @@+import { describe, expect, it } from "vitest";++import { normalizeSpanIO } from "../../../parser";+import { langfuseAgentPluginFixtures } from "./fixtures";++describe("Langfuse agent plugin normalized I/O", () => {+ it.each(langfuseAgentPluginFixtures)("$name", ({ spanIO, expected }) => {+ expect(normalizeSpanIO(spanIO)).toEqual({+ ...expected,+ span: spanIO,+ });+ });+});diff --git a/packages/shared/src/utils/normalized-io/conventions/registry.test.ts b/packages/shared/src/utils/normalized-io/conventions/registry.test.tsindex 4656baa086..159860e13f 100644--- a/packages/shared/src/utils/normalized-io/conventions/registry.test.ts+++ b/packages/shared/src/utils/normalized-io/conventions/registry.test.ts@@ -26,9 +26,9 @@ describe("provider convention registry", () => {expect(new Set(names).size).toBe(names.length);});- it("keeps typed part vocabularies disjoint except for guarded media types", () => {+ it("keeps typed part vocabularies disjoint except for guarded contested types", () => {const owners = new Map<string, string>();- const guardedTypes = new Set(["file", "image"]);+ const guardedTypes = new Set(["file", "image", "thinking"]);for (const provider of registeredProviders) {for (const type of Object.keys(provider.typedParts ?? {})) {diff --git a/web/src/features/traces/hooks/useChatMLParser.clienttest.ts b/web/src/features/traces/hooks/useChatMLParser.clienttest.tsindex 04a39f536e..073e9ea5df 100644--- a/web/src/features/traces/hooks/useChatMLParser.clienttest.ts+++ b/web/src/features/traces/hooks/useChatMLParser.clienttest.ts@@ -1,6 +1,7 @@import { renderHook } from "@testing-library/react";import { normalizeSpanIO } from "@langfuse/shared/src/utils/normalized-io";import { toIOPreview } from "../parsers/toIOPreview";+import { hasThinkingContent } from "../fns/chatMessageUtils";import {parseChatML,type ChatMLParserResult,@@ -171,6 +172,57 @@ describe("useChatMLParser", () => {},);+ it.each(parserImplementations)(+ "$name parser renders a Thinking block for a plugin `thinking` sibling array",+ ({ parse }) => {+ const reasoning = "The user wants the repo described in one line.";++ const result = parse([{ role: "user", content: "What is this repo?" }], {+ role: "assistant",+ content: "A scratch repository.",+ thinking: [+ { type: "thinking", content: reasoning, signature: "sig_sibling" },+ ],+ });++ const assistant = result.allMessages.find(+ (message) => message.role === "assistant",+ );++ expect(assistant && hasThinkingContent(assistant)).toBe(true);+ expect(assistant?.thinking).toEqual([+ { type: "thinking", content: reasoning, signature: "sig_sibling" },+ ]);+ },+ );++ it("normalized parser renders a Thinking block for a plugin thinking part in `content`", () => {+ const reasoning = "The user wants the repo described in one line.";++ const result = toIOPreview(+ normalizeSpanIO({+ input: [{ role: "user", content: "What is this repo?" }],+ output: {+ role: "assistant",+ content: [+ { type: "thinking", content: reasoning, signature: "sig_inline" },+ { type: "text", text: "A scratch repository." },+ ],+ },+ metadata: undefined,+ }),+ );++ const assistant = result.allMessages.find(+ (message) => message.role === "assistant",+ );++ expect(assistant && hasThinkingContent(assistant)).toBe(true);+ expect(assistant?.thinking).toEqual([+ { type: "thinking", content: reasoning, signature: "sig_inline" },+ ]);+ });+it("reuses an already computed parser result", () => {const preparedResult: ChatMLParserResult = {canDisplayAsChat: true,