🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
fix(parser): add fixture for normalizing batched messages and update tests (#17319)
marliessophie6d817b0691bc
diff --git a/packages/shared/src/utils/normalized-io/conventions/providers/langchain/fixtures.ts b/packages/shared/src/utils/normalized-io/conventions/providers/langchain/fixtures.tsindex f491844cc5..36c8521f2e 100644--- a/packages/shared/src/utils/normalized-io/conventions/providers/langchain/fixtures.ts+++ b/packages/shared/src/utils/normalized-io/conventions/providers/langchain/fixtures.ts@@ -121,6 +121,150 @@ export const langchainSerializedEnvelopeFixture = {},} satisfies NormalizedIOFixture;+/**+ * OpenInference LangChain instrumentation: chat-model runs record the+ * callback's `messages: List[List[BaseMessage]]` verbatim, so the message+ * list is nested one level under the batch dimension, with each message a+ * `dumpd()` constructor envelope and the tool declarations alongside under a+ * top-level `tools` array. The batch level must be flattened so the inner+ * messages normalize, and the tool declarations land as tool definitions.+ */+export const langchainBatchedMessagesFixture = {+ name: "normalizes OpenInference LangChain batched (list-of-lists) messages",+ spanIO: {+ input: {+ messages: [+ [+ {+ lc: 1,+ type: "constructor",+ id: ["langchain", "schema", "messages", "SystemMessage"],+ kwargs: {+ content: "You are a helpful support assistant.",+ type: "system",+ },+ },+ {+ lc: 1,+ type: "constructor",+ id: ["langchain", "schema", "messages", "SystemMessage"],+ kwargs: { content: "## Overview", type: "system" },+ },+ {+ lc: 1,+ type: "constructor",+ id: ["langchain", "schema", "messages", "HumanMessage"],+ kwargs: {+ content: "do you offer obseravility of the moon",+ type: "human",+ id: "73mm861f-fe05-4dd1-8c6f-b1321067a231",+ },+ },+ ],+ ],+ tools: [+ {+ type: "function",+ function: {+ name: "get_callback_details",+ description: "Initiates the callback flow for a customer.",+ parameters: {+ type: "object",+ properties: { phone: { type: "string" } },+ required: ["phone"],+ },+ },+ },+ ],+ },+ output: undefined,+ metadata: undefined,+ },+ expected: {+ messages: [+ {+ role: "system",+ parts: [{ type: "text", text: "You are a helpful support assistant." }],+ source: "input",+ },+ {+ role: "system",+ parts: [{ type: "text", text: "## Overview" }],+ source: "input",+ },+ {+ id: "73mm861f-fe05-4dd1-8c6f-b1321067a231",+ role: "user",+ parts: [+ { type: "text", text: "do you offer obseravility of the moon" },+ ],+ source: "input",+ },+ ],+ toolDefinitions: [+ {+ name: "get_callback_details",+ description: "Initiates the callback flow for a customer.",+ inputSchema: {+ type: "object",+ properties: { phone: { type: "string" } },+ required: ["phone"],+ },+ type: "function",+ },+ ],+ },+} satisfies NormalizedIOFixture;++/**+ * Multiple batches (`[[convA], [convB]]`) hardly occur in practice —+ * LangChain fans each prompt into its own run/span — but if one appears, the+ * flat transcript has no batch boundary, so conversations concatenate in+ * order. Locked as best-effort: visible beats the pre-flatten drop.+ */+export const langchainMultiBatchMessagesFixture = {+ name: "flattens multiple LangChain batches into one transcript",+ spanIO: {+ input: {+ messages: [+ [+ {+ lc: 1,+ type: "constructor",+ id: ["langchain", "schema", "messages", "HumanMessage"],+ kwargs: { content: "Question A", type: "human" },+ },+ ],+ [+ {+ lc: 1,+ type: "constructor",+ id: ["langchain", "schema", "messages", "HumanMessage"],+ kwargs: { content: "Question B", type: "human" },+ },+ ],+ ],+ },+ output: undefined,+ metadata: undefined,+ },+ expected: {+ messages: [+ {+ role: "user",+ parts: [{ type: "text", text: "Question A" }],+ source: "input",+ },+ {+ role: "user",+ parts: [{ type: "text", text: "Question B" }],+ source: "input",+ },+ ],+ toolDefinitions: [],+ },+} satisfies NormalizedIOFixture;+/*** LangChain dict serialization (`.dict()` / LangSmith-style): messages carry* `type: "tool"` instead of a `role` key. A ToolMessage dict with adiff --git a/packages/shared/src/utils/normalized-io/conventions/providers/langchain/langchain.test.ts b/packages/shared/src/utils/normalized-io/conventions/providers/langchain/langchain.test.tsindex fad6078e8e..a34a59031f 100644--- a/packages/shared/src/utils/normalized-io/conventions/providers/langchain/langchain.test.ts+++ b/packages/shared/src/utils/normalized-io/conventions/providers/langchain/langchain.test.ts@@ -2,13 +2,17 @@ import { describe, expect, it } from "vitest";import { normalizeSpanIO } from "../../../parser";import {+ langchainBatchedMessagesFixture,langchainDictToolMessageFixture,+ langchainMultiBatchMessagesFixture,langchainSerializedEnvelopeFixture,langgraphProductionShapeFixture,} from "./fixtures";describe("LangChain normalized I/O", () => {it.each([+ langchainBatchedMessagesFixture,+ langchainMultiBatchMessagesFixture,langchainDictToolMessageFixture,langchainSerializedEnvelopeFixture,langgraphProductionShapeFixture,diff --git a/packages/shared/src/utils/normalized-io/core/accumulator/collect.ts b/packages/shared/src/utils/normalized-io/core/accumulator/collect.tsindex f2d67d75f8..39e5d0cb3f 100644--- a/packages/shared/src/utils/normalized-io/core/accumulator/collect.ts+++ b/packages/shared/src/utils/normalized-io/core/accumulator/collect.ts@@ -121,6 +121,19 @@ function collectMessageSequence(};for (const value of values) {+ // Some providers return a list of messages, rather than a single message.+ if (Array.isArray(value)) {+ flushStandaloneToolCalls();+ collectMessageSequence(+ value,+ fallbackRole,+ parserContext,+ messages,+ accumulator,+ );+ continue;+ }+const record = asRecord(value);if (record) collectToolDefinitionsFromRecord(record, accumulator);