🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
feat(langchain): add serialized generation result fixture for LangChain messages normalization (#17589)
marliessophieede67ed4b68e
diff --git a/packages/shared/src/utils/normalized-io/conventions/io-convention.ts b/packages/shared/src/utils/normalized-io/conventions/io-convention.tsindex 4110e38780..b722a31211 100644--- a/packages/shared/src/utils/normalized-io/conventions/io-convention.ts+++ b/packages/shared/src/utils/normalized-io/conventions/io-convention.ts@@ -21,7 +21,7 @@ export const claimed = <T>(value: T | null): ConventionResult<T> => ({export const dropped = { matched: true, value: null } as const;export type PartHandlerContext = {- normalizePart(value: unknown): NormalizedMessagePart | null;+ normalizePartValue(value: unknown): NormalizedMessagePart[];normalizePartList(values: unknown[]): NormalizedMessagePart[];};@@ -89,7 +89,7 @@ export type ToolDefinitionCarrier = {export type PartHandler = (value: Record<string, unknown>,context: PartHandlerContext,-) => ConventionResult<NormalizedMessagePart>;+) => ConventionResult<NormalizedMessagePart | NormalizedMessagePart[]>;export interface IOConvention {readonly name: string;diff --git a/packages/shared/src/utils/normalized-io/conventions/providers/langchain/fixtures.ts b/packages/shared/src/utils/normalized-io/conventions/providers/langchain/fixtures.tsindex e8396107b8..622658b408 100644--- a/packages/shared/src/utils/normalized-io/conventions/providers/langchain/fixtures.ts+++ b/packages/shared/src/utils/normalized-io/conventions/providers/langchain/fixtures.ts@@ -1,5 +1,176 @@import type { NormalizedIOFixture } from "../fixture-types";+// Reconstructed from a user-provided trace excerpt, not a verbatim capture.+// https://reference.langchain.com/python/langchain-core/outputs/llm_result/LLMResult+export const langchainSerializedGenerationResultFixture = {+ name: "normalizes batched serialized LangChain messages and ChatGeneration output",+ spanIO: {+ input: {+ messages: [+ [+ {+ lc: 1,+ type: "constructor",+ id: ["langchain", "schema", "messages", "SystemMessage"],+ kwargs: {+ content: "* Never disclose private information *",+ type: "system",+ },+ },+ {+ lc: 1,+ type: "constructor",+ id: ["langchain", "schema", "messages", "SystemMessage"],+ kwargs: {+ content: "Answer questions about office hours.",+ type: "system",+ },+ },+ {+ lc: 1,+ type: "constructor",+ id: ["langchain", "schema", "messages", "HumanMessage"],+ kwargs: { content: "office hours", type: "human", id: "123" },+ },+ {+ lc: 1,+ type: "constructor",+ id: ["langchain", "schema", "messages", "AIMessage"],+ kwargs: {+ content: [{ type: "text", text: "This is my answer" }],+ response_metadata: {},+ },+ },+ {+ lc: 1,+ type: "constructor",+ id: ["langchain", "schema", "messages", "AIMessage"],+ kwargs: {+ content: [+ {+ id: "rs_099776",+ // Schema-derived reasoning fields, not captured model output:+ // https://github.com/openai/openai-python/blob/main/src/openai/types/responses/response_reasoning_item.py+ summary: [+ {+ type: "summary_text",+ text: "Checking whether current office hours are available.",+ },+ ],+ type: "reasoning",+ content: [+ {+ type: "reasoning_text",+ text: "The provided context does not contain current opening times.",+ },+ ],+ index: 0,+ encrypted_content: "encrypted",+ },+ {+ type: "text",+ text: "I don't have the latest opening times.",+ },+ ],+ response_metadata: {},+ },+ },+ ],+ ],+ },+ output: {+ generations: [+ [+ {+ text: "You've asked about our office hours.",+ generation_info: null,+ type: "ChatGeneration",+ message: {+ lc: 1,+ type: "constructor",+ id: ["langchain", "schema", "messages", "AIMessage"],+ kwargs: {+ content: [+ {+ type: "text",+ text: "You've asked about our office hours.",+ },+ ],+ response_metadata: {},+ },+ },+ },+ ],+ ],+ llm_output: null,+ run: null,+ type: "LLMResult",+ },+ metadata: undefined,+ },+ expected: {+ messages: [+ {+ source: "input",+ role: "system",+ parts: [+ { type: "text", text: "* Never disclose private information *" },+ ],+ },+ {+ source: "input",+ role: "system",+ parts: [{ type: "text", text: "Answer questions about office hours." }],+ },+ {+ source: "input",+ role: "user",+ id: "123",+ parts: [{ type: "text", text: "office hours" }],+ },+ {+ source: "input",+ role: "assistant",+ parts: [{ type: "text", text: "This is my answer" }],+ },+ {+ source: "input",+ role: "assistant",+ parts: [+ {+ type: "reasoning",+ content: {+ kind: "text",+ text: "Checking whether current office hours are available.",+ },+ providerMetadata: { id: "rs_099776", index: 0 },+ },+ {+ type: "reasoning",+ content: {+ kind: "text",+ text: "The provided context does not contain current opening times.",+ },+ providerMetadata: { id: "rs_099776", index: 0 },+ },+ {+ type: "reasoning",+ content: { kind: "encrypted", data: "encrypted" },+ providerMetadata: { id: "rs_099776", index: 0 },+ },+ { type: "text", text: "I don't have the latest opening times." },+ ],+ },+ {+ source: "output",+ role: "assistant",+ parts: [{ type: "text", text: "You've asked about our office hours." }],+ },+ ],+ toolDefinitions: [],+ },+} satisfies NormalizedIOFixture;+/*** LangChain serialization envelope: instrumentation that dumps LangChain* message objects (dumpd) wraps each message in constructor kwargs with thediff --git a/packages/shared/src/utils/normalized-io/conventions/providers/langchain/index.ts b/packages/shared/src/utils/normalized-io/conventions/providers/langchain/index.tsindex 0cd113c33b..002f5dc2ca 100644--- a/packages/shared/src/utils/normalized-io/conventions/providers/langchain/index.ts+++ b/packages/shared/src/utils/normalized-io/conventions/providers/langchain/index.ts@@ -15,6 +15,7 @@ import type {ConventionResult,IOConvention,MessageEnvelopeContext,+ MessageSource,PartHandlerContext,} from "../../io-convention";@@ -109,8 +110,32 @@ function langchainCollectSiblingParts(: [];}+function langchainMessages(+ root: Record<string, unknown>,+ kind: "input" | "output",+): MessageSource[] {+ if (kind !== "output" || !Array.isArray(root.generations)) return [];++ // LLMResult batches generations; ChatResult contains a flat list.+ const sources: MessageSource[] = [];+ for (const value of root.generations.flat(1)) {+ const generation = asRecord(value);+ if (!generation) return [];+ const message = asRecord(generation.message);+ if (!message && typeof generation.text !== "string") return [];+ sources.push({+ kind: "single",+ value: message ?? generation.text,+ fallbackRole: "assistant",+ finishReasonCarrier: asRecord(generation.generation_info),+ });+ }+ return sources;+}+export const langchainProvider = {name: "langchain",+ claimMessages: langchainMessages,// Serialized LangChain message classes carry their role as a type string.roleByMessageType: {human: "user",diff --git a/packages/shared/src/utils/normalized-io/conventions/providers/langchain/langchain.test.ts b/packages/shared/src/utils/normalized-io/conventions/providers/langchain/langchain.test.tsindex 073de15bf1..ca63c3b6d3 100644--- a/packages/shared/src/utils/normalized-io/conventions/providers/langchain/langchain.test.ts+++ b/packages/shared/src/utils/normalized-io/conventions/providers/langchain/langchain.test.ts@@ -3,6 +3,7 @@ import { describe, expect, it } from "vitest";import { normalizeSpanIO } from "../../../parser";import {capturedTraceFixtures,+ langchainSerializedGenerationResultFixture,langchainBatchedMessagesFixture,langchainDictToolMessageFixture,langchainMultiBatchMessagesFixture,@@ -13,6 +14,7 @@ import {describe("LangChain normalized I/O", () => {it.each([...capturedTraceFixtures,+ langchainSerializedGenerationResultFixture,langchainBatchedMessagesFixture,langchainMultiBatchMessagesFixture,langchainDictToolMessageFixture,diff --git a/packages/shared/src/utils/normalized-io/conventions/providers/openai/index.ts b/packages/shared/src/utils/normalized-io/conventions/providers/openai/index.tsindex 687eeaa032..01972eae99 100644--- a/packages/shared/src/utils/normalized-io/conventions/providers/openai/index.ts+++ b/packages/shared/src/utils/normalized-io/conventions/providers/openai/index.ts@@ -274,6 +274,49 @@ const normalizeOpenAiResponsesCustomToolCall: PartHandler = (value) =>const normalizeOpenAiReasoningText: PartHandler = (value) =>claimed(reasoningPart(value.text));+function isReasoningTextArray(+ value: unknown,+ expectedType: "summary_text" | "reasoning_text",+): boolean {+ if (!Array.isArray(value)) return false;++ return value.every((entry) => {+ const part = asRecord(entry);+ return part?.type === expectedType && typeof part.text === "string";+ });+}++const normalizeOpenAiReasoning: PartHandler = (value, context) => {+ if (!isReasoningTextArray(value.summary, "summary_text")) {+ return unmatched;+ }++ const hasInvalidContent =+ value.content !== undefined &&+ !isReasoningTextArray(value.content, "reasoning_text");++ if (hasInvalidContent) {+ return unmatched;+ }++ const encryptedContent = value.encrypted_content;+ const isValidEncryptedContent =+ encryptedContent === undefined ||+ encryptedContent === null ||+ typeof encryptedContent === "string";++ if (!isValidEncryptedContent) {+ return unmatched;+ }++ // Text-based reasoning belongs to the shared handler.+ if (value.text !== undefined) {+ return unmatched;+ }++ return claimed(openAiReasoningParts(value, [], context));+};+const OPENAI_PART_HANDLERS = {function: normalizeOpenAiFunctionCall,function_call: normalizeOpenAiFunctionCall,@@ -286,6 +329,7 @@ const OPENAI_PART_HANDLERS = {input_file: normalizeOpenAiInputFile,refusal: normalizeOpenAiRefusal,reasoning_text: normalizeOpenAiReasoningText,+ reasoning: normalizeOpenAiReasoning,summary_text: normalizeOpenAiReasoningText,mcp_call: normalizeOpenAiMcpCall,...Object.fromEntries(@@ -355,20 +399,7 @@ function openAiCollectSiblingParts(if (audioPart) parts.push(audioPart);if (value.type === "reasoning") {- const reasoningValues = (- baseParts.length === 0 ? [value.summary, value.content] : [value.summary]- )- .flatMap((entry) => (Array.isArray(entry) ? entry : [entry]))- .filter((entry) => entry !== undefined && entry !== null);- parts.push(...context.normalizePartList(reasoningValues));-- const encryptedContent = optionalString(value.encrypted_content);- if (encryptedContent) {- parts.push({- type: "reasoning",- content: { kind: "encrypted", data: encryptedContent },- });- }+ parts.push(...openAiReasoningParts(value, baseParts, context));}return parts.length > 0@@ -376,6 +407,29 @@ function openAiCollectSiblingParts(: [];}+function openAiReasoningParts(+ value: Record<string, unknown>,+ baseParts: readonly NormalizedMessagePart[],+ context: { normalizePartList(values: unknown[]): NormalizedMessagePart[] },+): NormalizedMessagePart[] {+ const parts: NormalizedMessagePart[] = [];+ const reasoningValues = (+ baseParts.length === 0 ? [value.summary, value.content] : [value.summary]+ )+ .flatMap((entry) => (Array.isArray(entry) ? entry : [entry]))+ .filter((entry) => entry !== undefined && entry !== null);+ parts.push(...context.normalizePartList(reasoningValues));++ const encryptedContent = optionalString(value.encrypted_content);+ if (encryptedContent) {+ parts.push({+ type: "reasoning",+ content: { kind: "encrypted", data: encryptedContent },+ });+ }+ return parts;+}+/*** OpenAI response envelopes: Chat Completions `choices[]` (finish reason* per choice) and Responses' `output[]` item array.diff --git a/packages/shared/src/utils/normalized-io/core/accumulator/collect.ts b/packages/shared/src/utils/normalized-io/core/accumulator/collect.tsindex 9ddae42010..2246bc04d1 100644--- a/packages/shared/src/utils/normalized-io/core/accumulator/collect.ts+++ b/packages/shared/src/utils/normalized-io/core/accumulator/collect.ts@@ -7,7 +7,7 @@ import { addMessage, addToolDefinitionValue } from "./helpers";import type { NormalizedIOAccumulator } from "./interface";import type { ParserContext } from "../parser-context";import {- normalizePart,+ normalizePartValue,normalizeMessage,normalizeFinishReason,} from "../normalize";@@ -143,9 +143,11 @@ function collectMessageSequence(if (record?.type === "mcp_list_tools") continue;if (record && !isMessageLike(record)) {- const part = normalizePart(record, parserContext);- if (part?.type === "tool-call") {- standaloneToolCalls.push(part);+ const parts = normalizePartValue(record, parserContext);+ const onlyToolCalls =+ parts.length > 0 && parts.every((part) => part.type === "tool-call");+ if (onlyToolCalls) {+ standaloneToolCalls.push(...parts);continue;}}diff --git a/packages/shared/src/utils/normalized-io/core/normalize/message.ts b/packages/shared/src/utils/normalized-io/core/normalize/message.tsindex 83979b6d36..d4004d2bde 100644--- a/packages/shared/src/utils/normalized-io/core/normalize/message.ts+++ b/packages/shared/src/utils/normalized-io/core/normalize/message.ts@@ -6,16 +6,12 @@ import type {SiblingPartSlot,} from "../../conventions/io-convention";import { isMessageLike, isToolDefinitionMessage } from "../utils/format";-import type {- NormalizedMessage,- NormalizedMessagePart,- NormalizedMessageRole,-} from "../../types";+import type { NormalizedMessage, NormalizedMessagePart } from "../../types";import { asRecord, isRecord, optionalString, parseArray } from "../utils/json";import { normalizeFinishReason } from "./finish-reason";import { normalizeMediaPartsFromString } from "./message-parts/media";import { extractCitations } from "./message-parts/text";-import { normalizePart, normalizePartList } from "./part";+import { normalizePartValue, normalizePartList } from "./part";import { coerceRole, normalizeRole } from "./role";import type { ParserContext } from "../parser-context";import { providersInOrder } from "../utils/providers";@@ -24,7 +20,6 @@ import { providersInOrder } from "../utils/providers";function normalizeMessageContent(value: Record<string, unknown>,nestedContent: Record<string, unknown> | undefined,- role: NormalizedMessageRole,parserContext: ParserContext,): NormalizedMessagePart[] {const rawParts = Array.isArray(value.parts)@@ -42,14 +37,19 @@ function normalizeMessageContent(const parsedContent = parseArray(value.content);if (parsedContent?.length) {const parsedParts = parsedContent.map((part) =>- normalizePart(part, parserContext),+ normalizePartValue(part, parserContext),);if (parsedParts.every(- (part) => part?.type === "tool-call" || part?.type === "tool-result",+ (parts) =>+ parts.length > 0 &&+ parts.every(+ (part) =>+ part.type === "tool-call" || part.type === "tool-result",+ ),)) {- return parsedParts.filter((part) => part !== null);+ return parsedParts.flat();}}@@ -57,8 +57,7 @@ function normalizeMessageContent(}if (isRecord(value.content)) {- const part = normalizePart(value.content, parserContext);- return part ? [part] : [];+ return normalizePartValue(value.content, parserContext);}return [];@@ -78,7 +77,7 @@ function applySiblingFields(parserContext: ParserContext,): void {const partContext: PartHandlerContext = {- normalizePart: (part) => normalizePart(part, parserContext),+ normalizePartValue: (part) => normalizePartValue(part, parserContext),normalizePartList: (values) => normalizePartList(values, parserContext),};const contributions: SiblingPartContribution[] = [];@@ -191,16 +190,19 @@ export function normalizeMessage(// Standalone tool-call/result values (no message keys): normalize once,// inspect the result, rather than shape-probing before normalizing.- const directPart = !isMessageLike(value)- ? normalizePart(value, parserContext)- : null;- if (- directPart &&- (directPart.type === "tool-call" || directPart.type === "tool-result")- ) {+ const directParts = !isMessageLike(value)+ ? normalizePartValue(value, parserContext)+ : [];+ const onlyToolCalls =+ directParts.length > 0 &&+ directParts.every((part) => part.type === "tool-call");+ const onlyToolResults =+ directParts.length > 0 &&+ directParts.every((part) => part.type === "tool-result");+ if (onlyToolCalls || onlyToolResults) {return {- role: directPart.type === "tool-result" ? "tool" : "assistant",- parts: [directPart],+ role: onlyToolResults ? "tool" : "assistant",+ parts: directParts,source,};}@@ -212,12 +214,7 @@ export function normalizeMessage((nestedContent ? normalizeRole(nestedContent) : undefined) ??fallbackRole;- const parts = normalizeMessageContent(- value,- nestedContent,- role,- parserContext,- );+ const parts = normalizeMessageContent(value, nestedContent, parserContext);applySiblingFields(value, parts, parserContext);role = coerceRole(role, value, parts);@@ -225,8 +222,7 @@ export function normalizeMessage(// records, {text} items): represent them instead of dropping — the trace// view renders these as JSON too.if (parts.length === 0 && !isMessageLike(value)) {- const part = normalizePart(value, parserContext);- if (part) parts.push(part);+ parts.push(...directParts);}// Choice/candidate-level values are wired in by the accumulator collector.diff --git a/packages/shared/src/utils/normalized-io/core/normalize/part.ts b/packages/shared/src/utils/normalized-io/core/normalize/part.tsindex 06e8a5edc7..1281ec76e6 100644--- a/packages/shared/src/utils/normalized-io/core/normalize/part.ts+++ b/packages/shared/src/utils/normalized-io/core/normalize/part.ts@@ -42,7 +42,7 @@ const SHARED_TYPED_PART_HANDLERS: Readonly<Record<string, PartHandler>> = {function createPartContext(parserContext?: ParserContext): PartHandlerContext {return {- normalizePart: (value) => normalizePart(value, parserContext),+ normalizePartValue: (value) => normalizePartValue(value, parserContext),normalizePartList: (values) => normalizePartList(values, parserContext),};}@@ -59,56 +59,61 @@ export function normalizePartList(continue;}- const part = normalizePart(value, parserContext);- if (!part) continue;-- // Text parts frequently embed media reference tokens mid-string; split- // them out. Refusals and annotated text stay intact.- if (part.type === "text" && !part.providerMetadata && !part.refusal) {- parts.push(...normalizeMediaPartsFromString(part.text));- continue;+ for (const part of normalizePartValue(value, parserContext)) {+ // Text parts frequently embed media reference tokens mid-string; split+ // them out. Refusals and annotated text stay intact.+ if (part.type === "text" && !part.providerMetadata && !part.refusal) {+ parts.push(...normalizeMediaPartsFromString(part.text));+ continue;+ }+ parts.push(part);}- parts.push(part);}return parts;}+function asParts(+ value: NormalizedMessagePart | NormalizedMessagePart[] | null,+): NormalizedMessagePart[] {+ if (value === null) return [];+ return Array.isArray(value) ? value : [value];+}+function normalizePartBase(value: unknown,parserContext?: ParserContext,-): NormalizedMessagePart | null {+): NormalizedMessagePart[] {if (typeof value === "string") {const mediaReference = parseMediaReference(value);- if (mediaReference) return filePartFromMediaReference(mediaReference);- return { type: "text", text: value };+ if (mediaReference) return [filePartFromMediaReference(mediaReference)];+ return [{ type: "text", text: value }];}- if (!isRecord(value)) return null;+ if (!isRecord(value)) return [];const partContext = createPartContext(parserContext);const providers = providersInOrder(parserContext?.preferredProvider);const type = typeof value.type === "string" ? value.type : undefined;if (type) {- const sharedHandler = ownLookup(SHARED_TYPED_PART_HANDLERS, type);- if (sharedHandler) {- const result = sharedHandler(value, partContext);- if (result.matched) return result.value;- }-for (const provider of providers) {const handler = ownLookup(provider.typedParts, type);if (!handler) continue;const result = handler(value, partContext);- if (result.matched) return result.value;+ if (result.matched) return asParts(result.value);+ }+ const sharedHandler = ownLookup(SHARED_TYPED_PART_HANDLERS, type);+ if (sharedHandler) {+ const result = sharedHandler(value, partContext);+ if (result.matched) return asParts(result.value);}} else {for (const provider of providers) {const result = provider.tryNormalizeUntypedPart?.(value, partContext);- if (result?.matched) return result.value;+ if (result?.matched) return asParts(result.value);}}- return normalizeFallbackPart(value);+ return asParts(normalizeFallbackPart(value));}const COMMON_CONSUMED_PART_KEYS = [@@ -254,13 +259,21 @@ function withProviderMetadata<T extends NormalizedMessagePart>(return providerMetadata ? ({ ...part, providerMetadata } as T) : part;}-export function normalizePart(+export function normalizePartValue(value: unknown,parserContext?: ParserContext,-): NormalizedMessagePart | null {- const part = normalizePartBase(value, parserContext);- if (!part) return null;+): NormalizedMessagePart[] {const record = asRecord(value);+ return normalizePartBase(value, parserContext).map((entry) =>+ finalizePart(entry, value, record),+ );+}++function finalizePart(+ part: NormalizedMessagePart,+ value: unknown,+ record: Record<string, unknown> | undefined,+): NormalizedMessagePart {const normalized = record ? withProviderMetadata(part, record) : part;if (normalized.type === "tool-call") {