🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
fix(normalized-io): extract nested messages and Pydantic agent history (#18357)
marliessophie74f4757710ce
diff --git a/packages/shared/src/utils/normalized-io/conventions/providers/mixed/fixtures/index.ts b/packages/shared/src/utils/normalized-io/conventions/providers/mixed/fixtures/index.tsindex ffc3a2fabb..cc1e8b053b 100644--- a/packages/shared/src/utils/normalized-io/conventions/providers/mixed/fixtures/index.ts+++ b/packages/shared/src/utils/normalized-io/conventions/providers/mixed/fixtures/index.ts@@ -2,6 +2,7 @@ import type { NormalizedIOFixture } from "../../fixture-types";import { looseProviderMessageShapesFixture } from "./loose-provider-message-shapes";import { outputOnlyPlainTextFixture } from "./output-only-plain-text";import { outputOnlyStructuredMessageFixture } from "./output-only-structured-message";+import { paramsMessagesFixture } from "./params-messages";import { rawPassthroughToolCallsFixture } from "./raw-passthrough-tool-calls";// Verbatim stored observation IO from ChatML integration-example exports.@@ -278,6 +279,7 @@ export const mixedNormalizedIOFixtures = [...capturedTraceFixtures,looseProviderMessageShapesFixture,outputOnlyStructuredMessageFixture,+ paramsMessagesFixture,outputOnlyPlainTextFixture,rawPassthroughToolCallsFixture,];diff --git a/packages/shared/src/utils/normalized-io/conventions/providers/mixed/fixtures/params-messages.ts b/packages/shared/src/utils/normalized-io/conventions/providers/mixed/fixtures/params-messages.tsnew file mode 100644index 0000000000..337ef260ac--- /dev/null+++ b/packages/shared/src/utils/normalized-io/conventions/providers/mixed/fixtures/params-messages.ts@@ -0,0 +1,50 @@+import type { NormalizedIOFixture } from "../../fixture-types";++export const paramsMessagesFixture = {+ name: "normalizes messages nested in input params",+ spanIO: {+ input: {+ params: {+ messages: [{ role: "user", content: "What is 2 + 2?" }],+ model: "gpt-4.1-mini",+ temperature: 0,+ },+ skipDoneStreamingCallback: false,+ },+ output: {+ chatCompletion: { id: "completion-1" },+ openaiCompletionModel: { id: "model-1" },+ },+ metadata: undefined,+ },+ expected: {+ messages: [+ {+ role: "user",+ parts: [{ type: "text", text: "What is 2 + 2?" }],+ source: "input",+ },+ {+ role: "assistant",+ parts: [+ {+ type: "data",+ value: {+ chatCompletion: { id: "completion-1" },+ openaiCompletionModel: { id: "model-1" },+ },+ },+ ],+ source: "output",+ },+ ],+ toolDefinitions: [],+ additionalInput: {+ params: {+ model: "gpt-4.1-mini",+ temperature: 0,+ },+ skipDoneStreamingCallback: false,+ },+ },+} satisfies NormalizedIOFixture;diff --git a/packages/shared/src/utils/normalized-io/conventions/providers/mixed/mixed.test.ts b/packages/shared/src/utils/normalized-io/conventions/providers/mixed/mixed.test.tsindex 343ed622c8..f4f80b91e5 100644--- a/packages/shared/src/utils/normalized-io/conventions/providers/mixed/mixed.test.ts+++ b/packages/shared/src/utils/normalized-io/conventions/providers/mixed/mixed.test.ts@@ -10,4 +10,37 @@ describe("mixed normalized I/O", () => {span: spanIO,});});++ it("prefers root messages over nested params messages", () => {+ const params = { messages: [{ role: "user", content: "Nested" }] };+ const result = normalizeSpanIO({+ input: { messages: [{ role: "user", content: "Root" }], params },+ output: undefined,+ metadata: undefined,+ });+ expect(result.messages).toEqual([+ {+ role: "user",+ source: "input",+ parts: [{ type: "text", text: "Root" }],+ },+ ]);+ expect(result.additionalInput).toEqual({ params });+ });++ it("leaves unrelated params messages as raw data", () => {+ const input = { params: { messages: [{ status: "queued" }] } };+ const result = normalizeSpanIO({+ input,+ output: undefined,+ metadata: undefined,+ });+ expect(result.messages).toEqual([+ {+ role: "user",+ source: "input",+ parts: [{ type: "data", value: input }],+ },+ ]);+ });});diff --git a/packages/shared/src/utils/normalized-io/conventions/providers/pydantic-ai/fixtures.ts b/packages/shared/src/utils/normalized-io/conventions/providers/pydantic-ai/fixtures.tsindex c8916a4990..4f96dcb418 100644--- a/packages/shared/src/utils/normalized-io/conventions/providers/pydantic-ai/fixtures.ts+++ b/packages/shared/src/utils/normalized-io/conventions/providers/pydantic-ai/fixtures.ts@@ -1,5 +1,153 @@import type { NormalizedIOFixture } from "../fixture-types";+const structuredResult = { reasoning: "The answer is correct.", score: 1 };++export const agentRunFixture = {+ name: "extracts Pydantic agent history and model request tools",+ spanIO: {+ metadata: undefined,+ input: { args: ["Evaluate this answer."], kwargs: {} },+ output: {+ output: structuredResult,+ _state: {+ message_history: [+ {+ kind: "request",+ parts: [+ { part_kind: "system-prompt", content: "Evaluate carefully." },+ { part_kind: "user-prompt", content: "Evaluate this answer." },+ ],+ },+ {+ kind: "response",+ parts: [+ { part_kind: "text", content: "Checking the reference." },+ {+ part_kind: "tool-call",+ tool_name: "lookup_reference",+ tool_call_id: "call_1",+ args: '{"query":"answer"}',+ },+ ],+ },+ {+ kind: "request",+ parts: [+ {+ part_kind: "tool-return",+ tool_name: "lookup_reference",+ tool_call_id: "call_1",+ content: "Verified.",+ },+ ],+ },+ {+ kind: "response",+ parts: [+ {+ part_kind: "tool-call",+ tool_name: "final_result",+ tool_call_id: "call_2",+ args: structuredResult,+ },+ ],+ },+ ],+ last_model_request_parameters: {+ function_tools: [+ {+ name: "lookup_reference",+ parameters_json_schema: { type: "object" },+ },+ ],+ output_tools: [+ {+ name: "final_result",+ parameters_json_schema: { type: "object" },+ },+ ],+ builtin_tools: [+ { kind: "web_search", search_context_size: "medium" },+ { unsupported_builtin: true },+ ],+ },+ },+ },+ },+ expected: {+ messages: [+ {+ role: "user",+ source: "input",+ parts: [+ {+ type: "data",+ value: { args: ["Evaluate this answer."], kwargs: {} },+ },+ ],+ },+ {+ role: "system",+ source: "output",+ parts: [{ type: "text", text: "Evaluate carefully." }],+ },+ {+ role: "user",+ source: "output",+ parts: [{ type: "text", text: "Evaluate this answer." }],+ },+ {+ role: "assistant",+ source: "output",+ parts: [+ { type: "text", text: "Checking the reference." },+ {+ type: "tool-call",+ toolType: "tool-call",+ toolName: "lookup_reference",+ toolCallId: "call_1",+ input: { query: "answer" },+ },+ ],+ },+ {+ role: "tool",+ source: "output",+ parts: [+ {+ type: "tool-result",+ toolName: "lookup_reference",+ toolCallId: "call_1",+ output: { type: "text", value: "Verified." },+ },+ ],+ },+ {+ role: "assistant",+ source: "output",+ parts: [+ {+ type: "tool-call",+ toolType: "tool-call",+ toolName: "final_result",+ toolCallId: "call_2",+ input: structuredResult,+ },+ ],+ },+ ],+ toolDefinitions: [+ { name: "lookup_reference", inputSchema: { type: "object" } },+ { name: "final_result", inputSchema: { type: "object" } },+ {+ name: "web_search",+ type: "web_search",+ providerMetadata: { search_context_size: "medium" },+ },+ ],+ },+} satisfies NormalizedIOFixture;+export const pydanticAiProductionShapeFixture = {name: "normalizes an anonymized Pydantic AI span",otel: {diff --git a/packages/shared/src/utils/normalized-io/conventions/providers/pydantic-ai/index.ts b/packages/shared/src/utils/normalized-io/conventions/providers/pydantic-ai/index.tsindex 9170dda977..cff1fbdd94 100644--- a/packages/shared/src/utils/normalized-io/conventions/providers/pydantic-ai/index.ts+++ b/packages/shared/src/utils/normalized-io/conventions/providers/pydantic-ai/index.ts@@ -1,40 +1,151 @@import { claimed, unmatched } from "../..";-import { parseRecord } from "../../../core/utils/json";+import {+ asRecord,+ parseArray,+ parseRecord,+ recordKeyAsParsed,+} from "../../../core/utils/json";import {toolDefinition,toolDefinitionProviderMetadata,} from "../../../core/normalize/tool-definitions";import type {IOConvention,+ PartHandlerContext,+ MessageSource,ToolDefinitionCarrier,ToolDefinitionSource,} from "../../io-convention";-/**- * Pydantic AI convention. Currently tool-definition discovery only: Pydantic- * AI logs its request config as a `model_request_parameters` span attribute- * (possibly JSON-encoded) with tool declarations under `function_tools`.- */-function pydanticAiToolDefinitionSources(- carrier: ToolDefinitionCarrier,-): ToolDefinitionSource[] {- const modelRequestParameters = parseRecord(- carrier.metadataAttributes?.model_request_parameters,+type PydanticMessage = {+ kind: "request" | "response";+ parts: unknown[];+};++function isPydanticMessage(value: unknown): value is PydanticMessage {+ const message = asRecord(value);+ return (+ (message?.kind === "request" || message?.kind === "response") &&+ Array.isArray(message.parts));- const functionTools = modelRequestParameters?.function_tools;- if (functionTools === undefined) return [];+}++// tryUnwrapMessage returns one message; split mixed-role Pydantic request parts in the claim instead.+function pydanticAiMessage(message: PydanticMessage) {+ if (message.kind === "response") {+ return [{ role: "assistant", parts: message.parts }];+ }+ return message.parts.map((part) => {+ const kind = asRecord(part)?.part_kind;+ let role = "user";+ if (kind === "system-prompt") role = "system";+ if (kind === "tool-return") role = "tool";+ // Prompt content may itself contain several multimodal parts.+ if (kind === "system-prompt" || kind === "user-prompt") {+ return { role, content: asRecord(part)?.content };+ }+ return { role, parts: [part] };+ });+}++/** Extract native history without claiming unrelated application state. */+function pydanticAiMessages(+ root: Record<string, unknown>,+ kind: "input" | "output",+): MessageSource[] {+ const history = parseArray(parseRecord(root._state)?.message_history);+ if (!history?.length || !history.every(isPydanticMessage)) return [];+ recordKeyAsParsed(root, "_state", "message_history");return [{- sourceKey: "model_request_parameters.function_tools",- value: functionTools,- options: { allowToolMap: true },+ kind: "sequence",+ values: history.flatMap(pydanticAiMessage),+ fallbackRole: kind === "input" ? "user" : "assistant",},];}+function pydanticAiPart(+ part: Record<string, unknown>,+ context: PartHandlerContext,+) {+ const { part_kind, tool_name, tool_call_id, args, ...rest } = part;+ let value: Record<string, unknown>;+ switch (part_kind) {+ case "text":+ value = { ...rest, type: "text" };+ break;+ case "tool-call":+ value = {+ ...rest,+ type: "tool-call",+ toolName: tool_name,+ toolCallId: tool_call_id,+ input: args,+ };+ break;+ case "tool-return":+ value = {+ ...rest,+ type: "tool-result",+ toolName: tool_name,+ toolCallId: tool_call_id,+ output: part.content,+ };+ break;+ default:+ return unmatched;+ }+ return claimed(context.normalizePartValue(value)[0] ?? null);+}++/** Builtin tools use `kind` where the common tool parser expects `type`. */+function pydanticAiBuiltinTool(value: unknown): unknown {+ const tool = asRecord(value);+ if (!tool || typeof tool.kind !== "string") return value;+ const { kind, ...rest } = tool;+ return { ...rest, type: kind };+}++function pydanticAiToolDefinitionSources(+ carrier: ToolDefinitionCarrier,+): ToolDefinitionSource[] {+ const state = parseRecord(carrier.root?._state);+ const sources: ToolDefinitionSource[] = [];+ for (const [sourceKey, value] of [+ [+ "model_request_parameters",+ carrier.metadataAttributes?.model_request_parameters,+ ],+ [+ "_state.last_model_request_parameters",+ state?.last_model_request_parameters,+ ],+ ] as const) {+ const parameters = parseRecord(value);+ for (const key of ["function_tools", "output_tools", "builtin_tools"]) {+ if (parameters?.[key] === undefined) continue;+ sources.push({+ sourceKey: `${sourceKey}.${key}`,+ value:+ key === "builtin_tools"+ ? (parseArray(parameters[key]) ?? []).map(pydanticAiBuiltinTool)+ : parameters[key],+ options: {+ allowProviderToolWithoutName: key === "builtin_tools",+ allowToolMap: key !== "builtin_tools",+ },+ });+ }+ }+ return sources;+}+export const pydanticAiProvider = {name: "pydantic-ai",+ claimMessages: pydanticAiMessages,+ tryNormalizeUntypedPart: pydanticAiPart,// Pydantic AI tool declarations: { name, description,// parameters_json_schema }.tryNormalizeToolDefinition: (value: Record<string, unknown>) => {diff --git a/packages/shared/src/utils/normalized-io/conventions/providers/pydantic-ai/pydantic-ai.test.ts b/packages/shared/src/utils/normalized-io/conventions/providers/pydantic-ai/pydantic-ai.test.tsindex 73f763bbd7..5762591323 100644--- a/packages/shared/src/utils/normalized-io/conventions/providers/pydantic-ai/pydantic-ai.test.ts+++ b/packages/shared/src/utils/normalized-io/conventions/providers/pydantic-ai/pydantic-ai.test.ts@@ -3,17 +3,38 @@ import { describe, expect, it } from "vitest";import { normalizeSpanIO } from "../../../parser";import {capturedTraceFixtures,+ agentRunFixture,pydanticAiProductionShapeFixture,} from "./fixtures";describe("Pydantic AI normalized I/O", () => {- it.each([...capturedTraceFixtures, pydanticAiProductionShapeFixture])(- "$name",- ({ spanIO, expected }) => {- expect(normalizeSpanIO(spanIO)).toEqual({- ...expected,- span: spanIO,- });- },- );+ it("leaves unrelated message history as data", () => {+ const output = {+ _state: { message_history: [{ content: "An audit entry." }] },+ };+ expect(+ normalizeSpanIO({ input: undefined, output, metadata: undefined }),+ ).toEqual({+ messages: [+ {+ role: "assistant",+ source: "output",+ parts: [{ type: "data", value: output }],+ },+ ],+ toolDefinitions: [],+ span: { input: undefined, output, metadata: undefined },+ });+ });++ it.each([+ ...capturedTraceFixtures,+ pydanticAiProductionShapeFixture,+ agentRunFixture,+ ])("$name", ({ spanIO, expected }) => {+ expect(normalizeSpanIO(spanIO)).toEqual({+ ...expected,+ span: spanIO,+ });+ });});diff --git a/packages/shared/src/utils/normalized-io/core/accumulator/collect.ts b/packages/shared/src/utils/normalized-io/core/accumulator/collect.tsindex ef0b699c6c..38923d9d09 100644--- a/packages/shared/src/utils/normalized-io/core/accumulator/collect.ts+++ b/packages/shared/src/utils/normalized-io/core/accumulator/collect.ts@@ -249,6 +249,14 @@ function collectRecordMessages(source === "input" ? findSystemMessageSources(root, source) : [];const claimedSources = claimMessages(root, source, parserContext);const rootMessages = parseArray(root.messages);+ const paramsMessages =+ source === "input"+ ? parseArray(parseRecord(root.params)?.messages)+ : undefined;+ const hasMessageShapedParams = paramsMessages?.some((value) => {+ const record = asRecord(value);+ return record !== undefined && isMessageLike(record);+ });const messages: NormalizedMessage[] = [];@@ -265,6 +273,15 @@ function collectRecordMessages(messages,accumulator,);+ } else if (paramsMessages && hasMessageShapedParams) {+ recordKeyAsParsed(root, "params", "messages");+ collectMessageSequence(+ paramsMessages,+ fallbackRole,+ parserContext,+ messages,+ accumulator,+ );} else {const recordMessage = normalizeMessage(root, fallbackRole, parserContext);if (recordMessage) {diff --git a/packages/shared/src/utils/normalized-io/core/normalize/part.ts b/packages/shared/src/utils/normalized-io/core/normalize/part.tsindex 0dde412256..d3f4873ec1 100644--- a/packages/shared/src/utils/normalized-io/core/normalize/part.ts+++ b/packages/shared/src/utils/normalized-io/core/normalize/part.ts@@ -119,6 +119,7 @@ function normalizePartBase(const COMMON_CONSUMED_PART_KEYS = ["type",+ "part_kind", // Pydantic part discriminator"providerMetadata","providerOptions",] as const;@@ -156,8 +157,10 @@ const CONSUMED_PART_KEYS_BY_TYPE: Record<"id","call_id","toolCallId",+ "tool_call_id","name","toolName",+ "tool_name","input","arguments","args",@@ -174,6 +177,7 @@ const CONSUMED_PART_KEYS_BY_TYPE: Record<"id","call_id","toolCallId",+ "tool_call_id","tool_use_id","name","toolName",