Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

feat(langchain): add serialized generation result fixture for LangChain messages normalization (#17589)

marliessophieede67ed4b68e
diff --git a/packages/shared/src/utils/normalized-io/conventions/io-convention.ts b/packages/shared/src/utils/normalized-io/conventions/io-convention.ts
index 4110e38780..b722a31211 100644
--- a/packages/shared/src/utils/normalized-io/conventions/io-convention.ts
+++ b/packages/shared/src/utils/normalized-io/conventions/io-convention.ts
@@ -21,7 +21,7 @@ export const claimed = <T>(value: T | null): ConventionResult<T> => ({
export const dropped = { matched: true, value: null } as const;
export type PartHandlerContext = {
- normalizePart(value: unknown): NormalizedMessagePart | null;
+ normalizePartValue(value: unknown): NormalizedMessagePart[];
normalizePartList(values: unknown[]): NormalizedMessagePart[];
};
@@ -89,7 +89,7 @@ export type ToolDefinitionCarrier = {
export type PartHandler = (
value: Record<string, unknown>,
context: PartHandlerContext,
-) => ConventionResult<NormalizedMessagePart>;
+) => ConventionResult<NormalizedMessagePart | NormalizedMessagePart[]>;
export interface IOConvention {
readonly name: string;
diff --git a/packages/shared/src/utils/normalized-io/conventions/providers/langchain/fixtures.ts b/packages/shared/src/utils/normalized-io/conventions/providers/langchain/fixtures.ts
index e8396107b8..622658b408 100644
--- a/packages/shared/src/utils/normalized-io/conventions/providers/langchain/fixtures.ts
+++ b/packages/shared/src/utils/normalized-io/conventions/providers/langchain/fixtures.ts
@@ -1,5 +1,176 @@
import type { NormalizedIOFixture } from "../fixture-types";
+// Reconstructed from a user-provided trace excerpt, not a verbatim capture.
+// https://reference.langchain.com/python/langchain-core/outputs/llm_result/LLMResult
+export const langchainSerializedGenerationResultFixture = {
+ name: "normalizes batched serialized LangChain messages and ChatGeneration output",
+ spanIO: {
+ input: {
+ messages: [
+ [
+ {
+ lc: 1,
+ type: "constructor",
+ id: ["langchain", "schema", "messages", "SystemMessage"],
+ kwargs: {
+ content: "* Never disclose private information *",
+ type: "system",
+ },
+ },
+ {
+ lc: 1,
+ type: "constructor",
+ id: ["langchain", "schema", "messages", "SystemMessage"],
+ kwargs: {
+ content: "Answer questions about office hours.",
+ type: "system",
+ },
+ },
+ {
+ lc: 1,
+ type: "constructor",
+ id: ["langchain", "schema", "messages", "HumanMessage"],
+ kwargs: { content: "office hours", type: "human", id: "123" },
+ },
+ {
+ lc: 1,
+ type: "constructor",
+ id: ["langchain", "schema", "messages", "AIMessage"],
+ kwargs: {
+ content: [{ type: "text", text: "This is my answer" }],
+ response_metadata: {},
+ },
+ },
+ {
+ lc: 1,
+ type: "constructor",
+ id: ["langchain", "schema", "messages", "AIMessage"],
+ kwargs: {
+ content: [
+ {
+ id: "rs_099776",
+ // Schema-derived reasoning fields, not captured model output:
+ // https://github.com/openai/openai-python/blob/main/src/openai/types/responses/response_reasoning_item.py
+ summary: [
+ {
+ type: "summary_text",
+ text: "Checking whether current office hours are available.",
+ },
+ ],
+ type: "reasoning",
+ content: [
+ {
+ type: "reasoning_text",
+ text: "The provided context does not contain current opening times.",
+ },
+ ],
+ index: 0,
+ encrypted_content: "encrypted",
+ },
+ {
+ type: "text",
+ text: "I don't have the latest opening times.",
+ },
+ ],
+ response_metadata: {},
+ },
+ },
+ ],
+ ],
+ },
+ output: {
+ generations: [
+ [
+ {
+ text: "You've asked about our office hours.",
+ generation_info: null,
+ type: "ChatGeneration",
+ message: {
+ lc: 1,
+ type: "constructor",
+ id: ["langchain", "schema", "messages", "AIMessage"],
+ kwargs: {
+ content: [
+ {
+ type: "text",
+ text: "You've asked about our office hours.",
+ },
+ ],
+ response_metadata: {},
+ },
+ },
+ },
+ ],
+ ],
+ llm_output: null,
+ run: null,
+ type: "LLMResult",
+ },
+ metadata: undefined,
+ },
+ expected: {
+ messages: [
+ {
+ source: "input",
+ role: "system",
+ parts: [
+ { type: "text", text: "* Never disclose private information *" },
+ ],
+ },
+ {
+ source: "input",
+ role: "system",
+ parts: [{ type: "text", text: "Answer questions about office hours." }],
+ },
+ {
+ source: "input",
+ role: "user",
+ id: "123",
+ parts: [{ type: "text", text: "office hours" }],
+ },
+ {
+ source: "input",
+ role: "assistant",
+ parts: [{ type: "text", text: "This is my answer" }],
+ },
+ {
+ source: "input",
+ role: "assistant",
+ parts: [
+ {
+ type: "reasoning",
+ content: {
+ kind: "text",
+ text: "Checking whether current office hours are available.",
+ },
+ providerMetadata: { id: "rs_099776", index: 0 },
+ },
+ {
+ type: "reasoning",
+ content: {
+ kind: "text",
+ text: "The provided context does not contain current opening times.",
+ },
+ providerMetadata: { id: "rs_099776", index: 0 },
+ },
+ {
+ type: "reasoning",
+ content: { kind: "encrypted", data: "encrypted" },
+ providerMetadata: { id: "rs_099776", index: 0 },
+ },
+ { type: "text", text: "I don't have the latest opening times." },
+ ],
+ },
+ {
+ source: "output",
+ role: "assistant",
+ parts: [{ type: "text", text: "You've asked about our office hours." }],
+ },
+ ],
+ toolDefinitions: [],
+ },
+} satisfies NormalizedIOFixture;
+
/**
* LangChain serialization envelope: instrumentation that dumps LangChain
* message objects (dumpd) wraps each message in constructor kwargs with the
diff --git a/packages/shared/src/utils/normalized-io/conventions/providers/langchain/index.ts b/packages/shared/src/utils/normalized-io/conventions/providers/langchain/index.ts
index 0cd113c33b..002f5dc2ca 100644
--- a/packages/shared/src/utils/normalized-io/conventions/providers/langchain/index.ts
+++ b/packages/shared/src/utils/normalized-io/conventions/providers/langchain/index.ts
@@ -15,6 +15,7 @@ import type {
ConventionResult,
IOConvention,
MessageEnvelopeContext,
+ MessageSource,
PartHandlerContext,
} from "../../io-convention";
@@ -109,8 +110,32 @@ function langchainCollectSiblingParts(
: [];
}
+function langchainMessages(
+ root: Record<string, unknown>,
+ kind: "input" | "output",
+): MessageSource[] {
+ if (kind !== "output" || !Array.isArray(root.generations)) return [];
+
+ // LLMResult batches generations; ChatResult contains a flat list.
+ const sources: MessageSource[] = [];
+ for (const value of root.generations.flat(1)) {
+ const generation = asRecord(value);
+ if (!generation) return [];
+ const message = asRecord(generation.message);
+ if (!message && typeof generation.text !== "string") return [];
+ sources.push({
+ kind: "single",
+ value: message ?? generation.text,
+ fallbackRole: "assistant",
+ finishReasonCarrier: asRecord(generation.generation_info),
+ });
+ }
+ return sources;
+}
+
export const langchainProvider = {
name: "langchain",
+ claimMessages: langchainMessages,
// Serialized LangChain message classes carry their role as a type string.
roleByMessageType: {
human: "user",
diff --git a/packages/shared/src/utils/normalized-io/conventions/providers/langchain/langchain.test.ts b/packages/shared/src/utils/normalized-io/conventions/providers/langchain/langchain.test.ts
index 073de15bf1..ca63c3b6d3 100644
--- a/packages/shared/src/utils/normalized-io/conventions/providers/langchain/langchain.test.ts
+++ b/packages/shared/src/utils/normalized-io/conventions/providers/langchain/langchain.test.ts
@@ -3,6 +3,7 @@ import { describe, expect, it } from "vitest";
import { normalizeSpanIO } from "../../../parser";
import {
capturedTraceFixtures,
+ langchainSerializedGenerationResultFixture,
langchainBatchedMessagesFixture,
langchainDictToolMessageFixture,
langchainMultiBatchMessagesFixture,
@@ -13,6 +14,7 @@ import {
describe("LangChain normalized I/O", () => {
it.each([
...capturedTraceFixtures,
+ langchainSerializedGenerationResultFixture,
langchainBatchedMessagesFixture,
langchainMultiBatchMessagesFixture,
langchainDictToolMessageFixture,
diff --git a/packages/shared/src/utils/normalized-io/conventions/providers/openai/index.ts b/packages/shared/src/utils/normalized-io/conventions/providers/openai/index.ts
index 687eeaa032..01972eae99 100644
--- a/packages/shared/src/utils/normalized-io/conventions/providers/openai/index.ts
+++ b/packages/shared/src/utils/normalized-io/conventions/providers/openai/index.ts
@@ -274,6 +274,49 @@ const normalizeOpenAiResponsesCustomToolCall: PartHandler = (value) =>
const normalizeOpenAiReasoningText: PartHandler = (value) =>
claimed(reasoningPart(value.text));
+function isReasoningTextArray(
+ value: unknown,
+ expectedType: "summary_text" | "reasoning_text",
+): boolean {
+ if (!Array.isArray(value)) return false;
+
+ return value.every((entry) => {
+ const part = asRecord(entry);
+ return part?.type === expectedType && typeof part.text === "string";
+ });
+}
+
+const normalizeOpenAiReasoning: PartHandler = (value, context) => {
+ if (!isReasoningTextArray(value.summary, "summary_text")) {
+ return unmatched;
+ }
+
+ const hasInvalidContent =
+ value.content !== undefined &&
+ !isReasoningTextArray(value.content, "reasoning_text");
+
+ if (hasInvalidContent) {
+ return unmatched;
+ }
+
+ const encryptedContent = value.encrypted_content;
+ const isValidEncryptedContent =
+ encryptedContent === undefined ||
+ encryptedContent === null ||
+ typeof encryptedContent === "string";
+
+ if (!isValidEncryptedContent) {
+ return unmatched;
+ }
+
+ // Text-based reasoning belongs to the shared handler.
+ if (value.text !== undefined) {
+ return unmatched;
+ }
+
+ return claimed(openAiReasoningParts(value, [], context));
+};
+
const OPENAI_PART_HANDLERS = {
function: normalizeOpenAiFunctionCall,
function_call: normalizeOpenAiFunctionCall,
@@ -286,6 +329,7 @@ const OPENAI_PART_HANDLERS = {
input_file: normalizeOpenAiInputFile,
refusal: normalizeOpenAiRefusal,
reasoning_text: normalizeOpenAiReasoningText,
+ reasoning: normalizeOpenAiReasoning,
summary_text: normalizeOpenAiReasoningText,
mcp_call: normalizeOpenAiMcpCall,
...Object.fromEntries(
@@ -355,20 +399,7 @@ function openAiCollectSiblingParts(
if (audioPart) parts.push(audioPart);
if (value.type === "reasoning") {
- const reasoningValues = (
- baseParts.length === 0 ? [value.summary, value.content] : [value.summary]
- )
- .flatMap((entry) => (Array.isArray(entry) ? entry : [entry]))
- .filter((entry) => entry !== undefined && entry !== null);
- parts.push(...context.normalizePartList(reasoningValues));
-
- const encryptedContent = optionalString(value.encrypted_content);
- if (encryptedContent) {
- parts.push({
- type: "reasoning",
- content: { kind: "encrypted", data: encryptedContent },
- });
- }
+ parts.push(...openAiReasoningParts(value, baseParts, context));
}
return parts.length > 0
@@ -376,6 +407,29 @@ function openAiCollectSiblingParts(
: [];
}
+function openAiReasoningParts(
+ value: Record<string, unknown>,
+ baseParts: readonly NormalizedMessagePart[],
+ context: { normalizePartList(values: unknown[]): NormalizedMessagePart[] },
+): NormalizedMessagePart[] {
+ const parts: NormalizedMessagePart[] = [];
+ const reasoningValues = (
+ baseParts.length === 0 ? [value.summary, value.content] : [value.summary]
+ )
+ .flatMap((entry) => (Array.isArray(entry) ? entry : [entry]))
+ .filter((entry) => entry !== undefined && entry !== null);
+ parts.push(...context.normalizePartList(reasoningValues));
+
+ const encryptedContent = optionalString(value.encrypted_content);
+ if (encryptedContent) {
+ parts.push({
+ type: "reasoning",
+ content: { kind: "encrypted", data: encryptedContent },
+ });
+ }
+ return parts;
+}
+
/**
* OpenAI response envelopes: Chat Completions `choices[]` (finish reason
* per choice) and Responses' `output[]` item array.
diff --git a/packages/shared/src/utils/normalized-io/core/accumulator/collect.ts b/packages/shared/src/utils/normalized-io/core/accumulator/collect.ts
index 9ddae42010..2246bc04d1 100644
--- a/packages/shared/src/utils/normalized-io/core/accumulator/collect.ts
+++ b/packages/shared/src/utils/normalized-io/core/accumulator/collect.ts
@@ -7,7 +7,7 @@ import { addMessage, addToolDefinitionValue } from "./helpers";
import type { NormalizedIOAccumulator } from "./interface";
import type { ParserContext } from "../parser-context";
import {
- normalizePart,
+ normalizePartValue,
normalizeMessage,
normalizeFinishReason,
} from "../normalize";
@@ -143,9 +143,11 @@ function collectMessageSequence(
if (record?.type === "mcp_list_tools") continue;
if (record && !isMessageLike(record)) {
- const part = normalizePart(record, parserContext);
- if (part?.type === "tool-call") {
- standaloneToolCalls.push(part);
+ const parts = normalizePartValue(record, parserContext);
+ const onlyToolCalls =
+ parts.length > 0 && parts.every((part) => part.type === "tool-call");
+ if (onlyToolCalls) {
+ standaloneToolCalls.push(...parts);
continue;
}
}
diff --git a/packages/shared/src/utils/normalized-io/core/normalize/message.ts b/packages/shared/src/utils/normalized-io/core/normalize/message.ts
index 83979b6d36..d4004d2bde 100644
--- a/packages/shared/src/utils/normalized-io/core/normalize/message.ts
+++ b/packages/shared/src/utils/normalized-io/core/normalize/message.ts
@@ -6,16 +6,12 @@ import type {
SiblingPartSlot,
} from "../../conventions/io-convention";
import { isMessageLike, isToolDefinitionMessage } from "../utils/format";
-import type {
- NormalizedMessage,
- NormalizedMessagePart,
- NormalizedMessageRole,
-} from "../../types";
+import type { NormalizedMessage, NormalizedMessagePart } from "../../types";
import { asRecord, isRecord, optionalString, parseArray } from "../utils/json";
import { normalizeFinishReason } from "./finish-reason";
import { normalizeMediaPartsFromString } from "./message-parts/media";
import { extractCitations } from "./message-parts/text";
-import { normalizePart, normalizePartList } from "./part";
+import { normalizePartValue, normalizePartList } from "./part";
import { coerceRole, normalizeRole } from "./role";
import type { ParserContext } from "../parser-context";
import { providersInOrder } from "../utils/providers";
@@ -24,7 +20,6 @@ import { providersInOrder } from "../utils/providers";
function normalizeMessageContent(
value: Record<string, unknown>,
nestedContent: Record<string, unknown> | undefined,
- role: NormalizedMessageRole,
parserContext: ParserContext,
): NormalizedMessagePart[] {
const rawParts = Array.isArray(value.parts)
@@ -42,14 +37,19 @@ function normalizeMessageContent(
const parsedContent = parseArray(value.content);
if (parsedContent?.length) {
const parsedParts = parsedContent.map((part) =>
- normalizePart(part, parserContext),
+ normalizePartValue(part, parserContext),
);
if (
parsedParts.every(
- (part) => part?.type === "tool-call" || part?.type === "tool-result",
+ (parts) =>
+ parts.length > 0 &&
+ parts.every(
+ (part) =>
+ part.type === "tool-call" || part.type === "tool-result",
+ ),
)
) {
- return parsedParts.filter((part) => part !== null);
+ return parsedParts.flat();
}
}
@@ -57,8 +57,7 @@ function normalizeMessageContent(
}
if (isRecord(value.content)) {
- const part = normalizePart(value.content, parserContext);
- return part ? [part] : [];
+ return normalizePartValue(value.content, parserContext);
}
return [];
@@ -78,7 +77,7 @@ function applySiblingFields(
parserContext: ParserContext,
): void {
const partContext: PartHandlerContext = {
- normalizePart: (part) => normalizePart(part, parserContext),
+ normalizePartValue: (part) => normalizePartValue(part, parserContext),
normalizePartList: (values) => normalizePartList(values, parserContext),
};
const contributions: SiblingPartContribution[] = [];
@@ -191,16 +190,19 @@ export function normalizeMessage(
// Standalone tool-call/result values (no message keys): normalize once,
// inspect the result, rather than shape-probing before normalizing.
- const directPart = !isMessageLike(value)
- ? normalizePart(value, parserContext)
- : null;
- if (
- directPart &&
- (directPart.type === "tool-call" || directPart.type === "tool-result")
- ) {
+ const directParts = !isMessageLike(value)
+ ? normalizePartValue(value, parserContext)
+ : [];
+ const onlyToolCalls =
+ directParts.length > 0 &&
+ directParts.every((part) => part.type === "tool-call");
+ const onlyToolResults =
+ directParts.length > 0 &&
+ directParts.every((part) => part.type === "tool-result");
+ if (onlyToolCalls || onlyToolResults) {
return {
- role: directPart.type === "tool-result" ? "tool" : "assistant",
- parts: [directPart],
+ role: onlyToolResults ? "tool" : "assistant",
+ parts: directParts,
source,
};
}
@@ -212,12 +214,7 @@ export function normalizeMessage(
(nestedContent ? normalizeRole(nestedContent) : undefined) ??
fallbackRole;
- const parts = normalizeMessageContent(
- value,
- nestedContent,
- role,
- parserContext,
- );
+ const parts = normalizeMessageContent(value, nestedContent, parserContext);
applySiblingFields(value, parts, parserContext);
role = coerceRole(role, value, parts);
@@ -225,8 +222,7 @@ export function normalizeMessage(
// records, {text} items): represent them instead of dropping — the trace
// view renders these as JSON too.
if (parts.length === 0 && !isMessageLike(value)) {
- const part = normalizePart(value, parserContext);
- if (part) parts.push(part);
+ parts.push(...directParts);
}
// Choice/candidate-level values are wired in by the accumulator collector.
diff --git a/packages/shared/src/utils/normalized-io/core/normalize/part.ts b/packages/shared/src/utils/normalized-io/core/normalize/part.ts
index 06e8a5edc7..1281ec76e6 100644
--- a/packages/shared/src/utils/normalized-io/core/normalize/part.ts
+++ b/packages/shared/src/utils/normalized-io/core/normalize/part.ts
@@ -42,7 +42,7 @@ const SHARED_TYPED_PART_HANDLERS: Readonly<Record<string, PartHandler>> = {
function createPartContext(parserContext?: ParserContext): PartHandlerContext {
return {
- normalizePart: (value) => normalizePart(value, parserContext),
+ normalizePartValue: (value) => normalizePartValue(value, parserContext),
normalizePartList: (values) => normalizePartList(values, parserContext),
};
}
@@ -59,56 +59,61 @@ export function normalizePartList(
continue;
}
- const part = normalizePart(value, parserContext);
- if (!part) continue;
-
- // Text parts frequently embed media reference tokens mid-string; split
- // them out. Refusals and annotated text stay intact.
- if (part.type === "text" && !part.providerMetadata && !part.refusal) {
- parts.push(...normalizeMediaPartsFromString(part.text));
- continue;
+ for (const part of normalizePartValue(value, parserContext)) {
+ // Text parts frequently embed media reference tokens mid-string; split
+ // them out. Refusals and annotated text stay intact.
+ if (part.type === "text" && !part.providerMetadata && !part.refusal) {
+ parts.push(...normalizeMediaPartsFromString(part.text));
+ continue;
+ }
+ parts.push(part);
}
- parts.push(part);
}
return parts;
}
+function asParts(
+ value: NormalizedMessagePart | NormalizedMessagePart[] | null,
+): NormalizedMessagePart[] {
+ if (value === null) return [];
+ return Array.isArray(value) ? value : [value];
+}
+
function normalizePartBase(
value: unknown,
parserContext?: ParserContext,
-): NormalizedMessagePart | null {
+): NormalizedMessagePart[] {
if (typeof value === "string") {
const mediaReference = parseMediaReference(value);
- if (mediaReference) return filePartFromMediaReference(mediaReference);
- return { type: "text", text: value };
+ if (mediaReference) return [filePartFromMediaReference(mediaReference)];
+ return [{ type: "text", text: value }];
}
- if (!isRecord(value)) return null;
+ if (!isRecord(value)) return [];
const partContext = createPartContext(parserContext);
const providers = providersInOrder(parserContext?.preferredProvider);
const type = typeof value.type === "string" ? value.type : undefined;
if (type) {
- const sharedHandler = ownLookup(SHARED_TYPED_PART_HANDLERS, type);
- if (sharedHandler) {
- const result = sharedHandler(value, partContext);
- if (result.matched) return result.value;
- }
-
for (const provider of providers) {
const handler = ownLookup(provider.typedParts, type);
if (!handler) continue;
const result = handler(value, partContext);
- if (result.matched) return result.value;
+ if (result.matched) return asParts(result.value);
+ }
+ const sharedHandler = ownLookup(SHARED_TYPED_PART_HANDLERS, type);
+ if (sharedHandler) {
+ const result = sharedHandler(value, partContext);
+ if (result.matched) return asParts(result.value);
}
} else {
for (const provider of providers) {
const result = provider.tryNormalizeUntypedPart?.(value, partContext);
- if (result?.matched) return result.value;
+ if (result?.matched) return asParts(result.value);
}
}
- return normalizeFallbackPart(value);
+ return asParts(normalizeFallbackPart(value));
}
const COMMON_CONSUMED_PART_KEYS = [
@@ -254,13 +259,21 @@ function withProviderMetadata<T extends NormalizedMessagePart>(
return providerMetadata ? ({ ...part, providerMetadata } as T) : part;
}
-export function normalizePart(
+export function normalizePartValue(
value: unknown,
parserContext?: ParserContext,
-): NormalizedMessagePart | null {
- const part = normalizePartBase(value, parserContext);
- if (!part) return null;
+): NormalizedMessagePart[] {
const record = asRecord(value);
+ return normalizePartBase(value, parserContext).map((entry) =>
+ finalizePart(entry, value, record),
+ );
+}
+
+function finalizePart(
+ part: NormalizedMessagePart,
+ value: unknown,
+ record: Record<string, unknown> | undefined,
+): NormalizedMessagePart {
const normalized = record ? withProviderMetadata(part, record) : part;
if (normalized.type === "tool-call") {