Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

fix(normalized-io): langfuse plugin rendering (#17820)

marliessophie754269adc08a
diff --git a/packages/shared/src/utils/normalized-io/conventions/index.ts b/packages/shared/src/utils/normalized-io/conventions/index.ts
index eea44fc4fc..093f4bb86a 100644
--- a/packages/shared/src/utils/normalized-io/conventions/index.ts
+++ b/packages/shared/src/utils/normalized-io/conventions/index.ts
@@ -4,6 +4,7 @@ import { aiSdkProvider } from "./providers/ai-sdk";
import { anthropicProvider } from "./providers/anthropic";
import { geminiProvider } from "./providers/gemini";
import { langchainProvider } from "./providers/langchain";
+import { langfuseAgentPluginsProvider } from "./providers/langfuse-agent-plugins";
import { openAiProvider } from "./providers/openai";
import { otelGenaiProvider } from "./providers/otel-genai";
import { pydanticAiProvider } from "./providers/pydantic-ai";
@@ -38,4 +39,5 @@ export const registeredProviders: readonly IOConvention[] = [
pydanticAiProvider,
semanticKernelProvider,
agnoProvider,
+ langfuseAgentPluginsProvider,
];
diff --git a/packages/shared/src/utils/normalized-io/conventions/providers/anthropic/index.ts b/packages/shared/src/utils/normalized-io/conventions/providers/anthropic/index.ts
index 05a306c0ab..6b0825b5c5 100644
--- a/packages/shared/src/utils/normalized-io/conventions/providers/anthropic/index.ts
+++ b/packages/shared/src/utils/normalized-io/conventions/providers/anthropic/index.ts
@@ -31,9 +31,9 @@ import type {
/**
* Anthropic Messages API convention: this module owns Anthropic's typed
- * block vocabulary, its `system` request field, and the `thinking` sibling
- * array. One message can still contain parts from several dialects — the
- * core folds every provider cumulatively.
+ * block vocabulary and its `system` request field. One message can still
+ * contain parts from several dialects — the core folds every provider
+ * cumulatively.
*/
// Anthropic `stop_reason` vocabulary -> the canonical FinishReason set.
@@ -127,8 +127,12 @@ const normalizeAnthropicMcpToolCall: PartHandler = (value) =>
),
);
-const normalizeAnthropicThinking: PartHandler = (value) =>
- claimed(reasoningPart(value.thinking, optionalString(value.signature)));
+const normalizeAnthropicThinking: PartHandler = (value) => {
+ if (value.thinking === null || value.thinking === undefined) return unmatched;
+ return claimed(
+ reasoningPart(value.thinking, optionalString(value.signature)),
+ );
+};
const normalizeAnthropicRedactedThinking: PartHandler = (value) => {
const data = optionalString(value.data);
@@ -235,17 +239,5 @@ export const anthropicProvider = {
});
return definition ? claimed(definition) : unmatched;
},
- collectSiblingParts: (
- value: Record<string, unknown>,
- _baseParts,
- context,
- ) => {
- const parts = context.normalizePartList(
- Array.isArray(value.thinking) ? value.thinking : [],
- );
- return parts.length > 0
- ? [{ sourceKey: "thinking", slot: "after-content", parts }]
- : [];
- },
getSystemMessage: anthropicSystemMessage,
} satisfies IOConvention;
diff --git a/packages/shared/src/utils/normalized-io/conventions/providers/langfuse-agent-plugins/fixtures.ts b/packages/shared/src/utils/normalized-io/conventions/providers/langfuse-agent-plugins/fixtures.ts
new file mode 100644
index 0000000000..12cbbeb13c
--- /dev/null
+++ b/packages/shared/src/utils/normalized-io/conventions/providers/langfuse-agent-plugins/fixtures.ts
@@ -0,0 +1,125 @@
+import type { NormalizedIOFixture } from "../fixture-types";
+
+const question = {
+ role: "user",
+ content: "Read README.md and tell me in one line what this repo is.",
+};
+const reasoning =
+ "The user wants me to read README.md and describe the repo in one line.";
+const answer = "A scratch repository for the reasoning-on-output review.";
+
+const userTurn: NormalizedIOFixture["expected"]["messages"][number] = {
+ role: "user",
+ parts: [{ type: "text", text: question.content }],
+ source: "input",
+};
+
+const answerThenReasoning = (
+ signature?: string,
+): NormalizedIOFixture["expected"]["messages"][number] => ({
+ role: "assistant",
+ parts: [
+ { type: "text", text: answer },
+ {
+ type: "reasoning",
+ content: {
+ kind: "text",
+ text: reasoning,
+ ...(signature ? { signature } : {}),
+ },
+ },
+ ],
+ source: "output",
+});
+
+export const langfuseAgentPluginFixtures: NormalizedIOFixture[] = [
+ {
+ name: "normalizes a `content`-keyed thinking part in the `thinking` sibling array",
+ spanIO: {
+ input: JSON.stringify([question]),
+ output: JSON.stringify({
+ role: "assistant",
+ content: answer,
+ thinking: [
+ { type: "thinking", content: reasoning, signature: "sig_sibling" },
+ ],
+ }),
+ metadata: undefined,
+ },
+ expected: {
+ messages: [userTurn, answerThenReasoning("sig_sibling")],
+ toolDefinitions: [],
+ },
+ },
+ {
+ name: "normalizes a `content`-keyed thinking part inline in `content`",
+ spanIO: {
+ input: JSON.stringify([question]),
+ output: JSON.stringify({
+ role: "assistant",
+ content: [
+ { type: "thinking", content: reasoning, signature: "sig_inline" },
+ { type: "text", text: answer },
+ ],
+ }),
+ metadata: undefined,
+ },
+ expected: {
+ messages: [
+ userTurn,
+ {
+ role: "assistant",
+ parts: [
+ {
+ type: "reasoning",
+ content: {
+ kind: "text",
+ text: reasoning,
+ signature: "sig_inline",
+ },
+ },
+ { type: "text", text: answer },
+ ],
+ source: "output",
+ },
+ ],
+ toolDefinitions: [],
+ },
+ },
+ {
+ name: "normalizes a thinking part carrying both `thinking` and `content`",
+ spanIO: {
+ input: JSON.stringify([question]),
+ output: JSON.stringify({
+ role: "assistant",
+ content: answer,
+ thinking: [
+ { type: "thinking", thinking: reasoning, content: reasoning },
+ ],
+ }),
+ metadata: undefined,
+ },
+ expected: {
+ messages: [userTurn, answerThenReasoning()],
+ toolDefinitions: [],
+ },
+ },
+ {
+ name: "normalizes an Anthropic thinking block inside a Langfuse plugin thinking sibling array",
+ spanIO: {
+ input: JSON.stringify([question]),
+ output: JSON.stringify({
+ role: "assistant",
+ content: answer,
+ thinking: [
+ { type: "thinking", thinking: reasoning, signature: "sig_native" },
+ ],
+ }),
+ metadata: undefined,
+ },
+ expected: {
+ messages: [userTurn, answerThenReasoning("sig_native")],
+ toolDefinitions: [],
+ },
+ },
+];
diff --git a/packages/shared/src/utils/normalized-io/conventions/providers/langfuse-agent-plugins/index.ts b/packages/shared/src/utils/normalized-io/conventions/providers/langfuse-agent-plugins/index.ts
new file mode 100644
index 0000000000..837ea661f7
--- /dev/null
+++ b/packages/shared/src/utils/normalized-io/conventions/providers/langfuse-agent-plugins/index.ts
@@ -0,0 +1,31 @@
+import { claimed, unmatched } from "../..";
+import { optionalString } from "../../../core/utils/json";
+import { reasoningPart } from "../../../core/normalize/message-parts/reasoning";
+import type { IOConvention, PartHandler } from "../../io-convention";
+
+const normalizeChatMlThinking: PartHandler = (value) => {
+ if (value.thinking !== undefined && value.thinking !== null) return unmatched;
+ return claimed(
+ reasoningPart(
+ value.content ?? value.summary,
+ optionalString(value.signature),
+ ),
+ );
+};
+
+export const langfuseAgentPluginsProvider = {
+ name: "langfuse-agent-plugins",
+ typedParts: { thinking: normalizeChatMlThinking },
+ collectSiblingParts: (
+ value: Record<string, unknown>,
+ _baseParts,
+ context,
+ ) => {
+ const parts = context.normalizePartList(
+ Array.isArray(value.thinking) ? value.thinking : [],
+ );
+ return parts.length > 0
+ ? [{ sourceKey: "thinking", slot: "after-content", parts }]
+ : [];
+ },
+} satisfies IOConvention;
diff --git a/packages/shared/src/utils/normalized-io/conventions/providers/langfuse-agent-plugins/langfuse-agent-plugins.test.ts b/packages/shared/src/utils/normalized-io/conventions/providers/langfuse-agent-plugins/langfuse-agent-plugins.test.ts
new file mode 100644
index 0000000000..ab4ff6f6e8
--- /dev/null
+++ b/packages/shared/src/utils/normalized-io/conventions/providers/langfuse-agent-plugins/langfuse-agent-plugins.test.ts
@@ -0,0 +1,13 @@
+import { describe, expect, it } from "vitest";
+
+import { normalizeSpanIO } from "../../../parser";
+import { langfuseAgentPluginFixtures } from "./fixtures";
+
+describe("Langfuse agent plugin normalized I/O", () => {
+ it.each(langfuseAgentPluginFixtures)("$name", ({ spanIO, expected }) => {
+ expect(normalizeSpanIO(spanIO)).toEqual({
+ ...expected,
+ span: spanIO,
+ });
+ });
+});
diff --git a/packages/shared/src/utils/normalized-io/conventions/registry.test.ts b/packages/shared/src/utils/normalized-io/conventions/registry.test.ts
index 4656baa086..159860e13f 100644
--- a/packages/shared/src/utils/normalized-io/conventions/registry.test.ts
+++ b/packages/shared/src/utils/normalized-io/conventions/registry.test.ts
@@ -26,9 +26,9 @@ describe("provider convention registry", () => {
expect(new Set(names).size).toBe(names.length);
});
- it("keeps typed part vocabularies disjoint except for guarded media types", () => {
+ it("keeps typed part vocabularies disjoint except for guarded contested types", () => {
const owners = new Map<string, string>();
- const guardedTypes = new Set(["file", "image"]);
+ const guardedTypes = new Set(["file", "image", "thinking"]);
for (const provider of registeredProviders) {
for (const type of Object.keys(provider.typedParts ?? {})) {
diff --git a/web/src/features/traces/hooks/useChatMLParser.clienttest.ts b/web/src/features/traces/hooks/useChatMLParser.clienttest.ts
index 04a39f536e..073e9ea5df 100644
--- a/web/src/features/traces/hooks/useChatMLParser.clienttest.ts
+++ b/web/src/features/traces/hooks/useChatMLParser.clienttest.ts
@@ -1,6 +1,7 @@
import { renderHook } from "@testing-library/react";
import { normalizeSpanIO } from "@langfuse/shared/src/utils/normalized-io";
import { toIOPreview } from "../parsers/toIOPreview";
+import { hasThinkingContent } from "../fns/chatMessageUtils";
import {
parseChatML,
type ChatMLParserResult,
@@ -171,6 +172,57 @@ describe("useChatMLParser", () => {
},
);
+ it.each(parserImplementations)(
+ "$name parser renders a Thinking block for a plugin `thinking` sibling array",
+ ({ parse }) => {
+ const reasoning = "The user wants the repo described in one line.";
+
+ const result = parse([{ role: "user", content: "What is this repo?" }], {
+ role: "assistant",
+ content: "A scratch repository.",
+ thinking: [
+ { type: "thinking", content: reasoning, signature: "sig_sibling" },
+ ],
+ });
+
+ const assistant = result.allMessages.find(
+ (message) => message.role === "assistant",
+ );
+
+ expect(assistant && hasThinkingContent(assistant)).toBe(true);
+ expect(assistant?.thinking).toEqual([
+ { type: "thinking", content: reasoning, signature: "sig_sibling" },
+ ]);
+ },
+ );
+
+ it("normalized parser renders a Thinking block for a plugin thinking part in `content`", () => {
+ const reasoning = "The user wants the repo described in one line.";
+
+ const result = toIOPreview(
+ normalizeSpanIO({
+ input: [{ role: "user", content: "What is this repo?" }],
+ output: {
+ role: "assistant",
+ content: [
+ { type: "thinking", content: reasoning, signature: "sig_inline" },
+ { type: "text", text: "A scratch repository." },
+ ],
+ },
+ metadata: undefined,
+ }),
+ );
+
+ const assistant = result.allMessages.find(
+ (message) => message.role === "assistant",
+ );
+
+ expect(assistant && hasThinkingContent(assistant)).toBe(true);
+ expect(assistant?.thinking).toEqual([
+ { type: "thinking", content: reasoning, signature: "sig_inline" },
+ ]);
+ });
+
it("reuses an already computed parser result", () => {
const preparedResult: ChatMLParserResult = {
canDisplayAsChat: true,