Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

fix(normalized-io): extract nested messages and Pydantic agent history (#18357)

marliessophie74f4757710ce
diff --git a/packages/shared/src/utils/normalized-io/conventions/providers/mixed/fixtures/index.ts b/packages/shared/src/utils/normalized-io/conventions/providers/mixed/fixtures/index.ts
index ffc3a2fabb..cc1e8b053b 100644
--- a/packages/shared/src/utils/normalized-io/conventions/providers/mixed/fixtures/index.ts
+++ b/packages/shared/src/utils/normalized-io/conventions/providers/mixed/fixtures/index.ts
@@ -2,6 +2,7 @@ import type { NormalizedIOFixture } from "../../fixture-types";
import { looseProviderMessageShapesFixture } from "./loose-provider-message-shapes";
import { outputOnlyPlainTextFixture } from "./output-only-plain-text";
import { outputOnlyStructuredMessageFixture } from "./output-only-structured-message";
+import { paramsMessagesFixture } from "./params-messages";
import { rawPassthroughToolCallsFixture } from "./raw-passthrough-tool-calls";
// Verbatim stored observation IO from ChatML integration-example exports.
@@ -278,6 +279,7 @@ export const mixedNormalizedIOFixtures = [
...capturedTraceFixtures,
looseProviderMessageShapesFixture,
outputOnlyStructuredMessageFixture,
+ paramsMessagesFixture,
outputOnlyPlainTextFixture,
rawPassthroughToolCallsFixture,
];
diff --git a/packages/shared/src/utils/normalized-io/conventions/providers/mixed/fixtures/params-messages.ts b/packages/shared/src/utils/normalized-io/conventions/providers/mixed/fixtures/params-messages.ts
new file mode 100644
index 0000000000..337ef260ac
--- /dev/null
+++ b/packages/shared/src/utils/normalized-io/conventions/providers/mixed/fixtures/params-messages.ts
@@ -0,0 +1,50 @@
+import type { NormalizedIOFixture } from "../../fixture-types";
+
+export const paramsMessagesFixture = {
+ name: "normalizes messages nested in input params",
+ spanIO: {
+ input: {
+ params: {
+ messages: [{ role: "user", content: "What is 2 + 2?" }],
+ model: "gpt-4.1-mini",
+ temperature: 0,
+ },
+ skipDoneStreamingCallback: false,
+ },
+ output: {
+ chatCompletion: { id: "completion-1" },
+ openaiCompletionModel: { id: "model-1" },
+ },
+ metadata: undefined,
+ },
+ expected: {
+ messages: [
+ {
+ role: "user",
+ parts: [{ type: "text", text: "What is 2 + 2?" }],
+ source: "input",
+ },
+ {
+ role: "assistant",
+ parts: [
+ {
+ type: "data",
+ value: {
+ chatCompletion: { id: "completion-1" },
+ openaiCompletionModel: { id: "model-1" },
+ },
+ },
+ ],
+ source: "output",
+ },
+ ],
+ toolDefinitions: [],
+ additionalInput: {
+ params: {
+ model: "gpt-4.1-mini",
+ temperature: 0,
+ },
+ skipDoneStreamingCallback: false,
+ },
+ },
+} satisfies NormalizedIOFixture;
diff --git a/packages/shared/src/utils/normalized-io/conventions/providers/mixed/mixed.test.ts b/packages/shared/src/utils/normalized-io/conventions/providers/mixed/mixed.test.ts
index 343ed622c8..f4f80b91e5 100644
--- a/packages/shared/src/utils/normalized-io/conventions/providers/mixed/mixed.test.ts
+++ b/packages/shared/src/utils/normalized-io/conventions/providers/mixed/mixed.test.ts
@@ -10,4 +10,37 @@ describe("mixed normalized I/O", () => {
span: spanIO,
});
});
+
+ it("prefers root messages over nested params messages", () => {
+ const params = { messages: [{ role: "user", content: "Nested" }] };
+ const result = normalizeSpanIO({
+ input: { messages: [{ role: "user", content: "Root" }], params },
+ output: undefined,
+ metadata: undefined,
+ });
+ expect(result.messages).toEqual([
+ {
+ role: "user",
+ source: "input",
+ parts: [{ type: "text", text: "Root" }],
+ },
+ ]);
+ expect(result.additionalInput).toEqual({ params });
+ });
+
+ it("leaves unrelated params messages as raw data", () => {
+ const input = { params: { messages: [{ status: "queued" }] } };
+ const result = normalizeSpanIO({
+ input,
+ output: undefined,
+ metadata: undefined,
+ });
+ expect(result.messages).toEqual([
+ {
+ role: "user",
+ source: "input",
+ parts: [{ type: "data", value: input }],
+ },
+ ]);
+ });
});
diff --git a/packages/shared/src/utils/normalized-io/conventions/providers/pydantic-ai/fixtures.ts b/packages/shared/src/utils/normalized-io/conventions/providers/pydantic-ai/fixtures.ts
index c8916a4990..4f96dcb418 100644
--- a/packages/shared/src/utils/normalized-io/conventions/providers/pydantic-ai/fixtures.ts
+++ b/packages/shared/src/utils/normalized-io/conventions/providers/pydantic-ai/fixtures.ts
@@ -1,5 +1,153 @@
import type { NormalizedIOFixture } from "../fixture-types";
+const structuredResult = { reasoning: "The answer is correct.", score: 1 };
+
+export const agentRunFixture = {
+ name: "extracts Pydantic agent history and model request tools",
+ spanIO: {
+ metadata: undefined,
+ input: { args: ["Evaluate this answer."], kwargs: {} },
+ output: {
+ output: structuredResult,
+ _state: {
+ message_history: [
+ {
+ kind: "request",
+ parts: [
+ { part_kind: "system-prompt", content: "Evaluate carefully." },
+ { part_kind: "user-prompt", content: "Evaluate this answer." },
+ ],
+ },
+ {
+ kind: "response",
+ parts: [
+ { part_kind: "text", content: "Checking the reference." },
+ {
+ part_kind: "tool-call",
+ tool_name: "lookup_reference",
+ tool_call_id: "call_1",
+ args: '{"query":"answer"}',
+ },
+ ],
+ },
+ {
+ kind: "request",
+ parts: [
+ {
+ part_kind: "tool-return",
+ tool_name: "lookup_reference",
+ tool_call_id: "call_1",
+ content: "Verified.",
+ },
+ ],
+ },
+ {
+ kind: "response",
+ parts: [
+ {
+ part_kind: "tool-call",
+ tool_name: "final_result",
+ tool_call_id: "call_2",
+ args: structuredResult,
+ },
+ ],
+ },
+ ],
+ last_model_request_parameters: {
+ function_tools: [
+ {
+ name: "lookup_reference",
+ parameters_json_schema: { type: "object" },
+ },
+ ],
+ output_tools: [
+ {
+ name: "final_result",
+ parameters_json_schema: { type: "object" },
+ },
+ ],
+ builtin_tools: [
+ { kind: "web_search", search_context_size: "medium" },
+ { unsupported_builtin: true },
+ ],
+ },
+ },
+ },
+ },
+ expected: {
+ messages: [
+ {
+ role: "user",
+ source: "input",
+ parts: [
+ {
+ type: "data",
+ value: { args: ["Evaluate this answer."], kwargs: {} },
+ },
+ ],
+ },
+ {
+ role: "system",
+ source: "output",
+ parts: [{ type: "text", text: "Evaluate carefully." }],
+ },
+ {
+ role: "user",
+ source: "output",
+ parts: [{ type: "text", text: "Evaluate this answer." }],
+ },
+ {
+ role: "assistant",
+ source: "output",
+ parts: [
+ { type: "text", text: "Checking the reference." },
+ {
+ type: "tool-call",
+ toolType: "tool-call",
+ toolName: "lookup_reference",
+ toolCallId: "call_1",
+ input: { query: "answer" },
+ },
+ ],
+ },
+ {
+ role: "tool",
+ source: "output",
+ parts: [
+ {
+ type: "tool-result",
+ toolName: "lookup_reference",
+ toolCallId: "call_1",
+ output: { type: "text", value: "Verified." },
+ },
+ ],
+ },
+ {
+ role: "assistant",
+ source: "output",
+ parts: [
+ {
+ type: "tool-call",
+ toolType: "tool-call",
+ toolName: "final_result",
+ toolCallId: "call_2",
+ input: structuredResult,
+ },
+ ],
+ },
+ ],
+ toolDefinitions: [
+ { name: "lookup_reference", inputSchema: { type: "object" } },
+ { name: "final_result", inputSchema: { type: "object" } },
+ {
+ name: "web_search",
+ type: "web_search",
+ providerMetadata: { search_context_size: "medium" },
+ },
+ ],
+ },
+} satisfies NormalizedIOFixture;
+
export const pydanticAiProductionShapeFixture = {
name: "normalizes an anonymized Pydantic AI span",
otel: {
diff --git a/packages/shared/src/utils/normalized-io/conventions/providers/pydantic-ai/index.ts b/packages/shared/src/utils/normalized-io/conventions/providers/pydantic-ai/index.ts
index 9170dda977..cff1fbdd94 100644
--- a/packages/shared/src/utils/normalized-io/conventions/providers/pydantic-ai/index.ts
+++ b/packages/shared/src/utils/normalized-io/conventions/providers/pydantic-ai/index.ts
@@ -1,40 +1,151 @@
import { claimed, unmatched } from "../..";
-import { parseRecord } from "../../../core/utils/json";
+import {
+ asRecord,
+ parseArray,
+ parseRecord,
+ recordKeyAsParsed,
+} from "../../../core/utils/json";
import {
toolDefinition,
toolDefinitionProviderMetadata,
} from "../../../core/normalize/tool-definitions";
import type {
IOConvention,
+ PartHandlerContext,
+ MessageSource,
ToolDefinitionCarrier,
ToolDefinitionSource,
} from "../../io-convention";
-/**
- * Pydantic AI convention. Currently tool-definition discovery only: Pydantic
- * AI logs its request config as a `model_request_parameters` span attribute
- * (possibly JSON-encoded) with tool declarations under `function_tools`.
- */
-function pydanticAiToolDefinitionSources(
- carrier: ToolDefinitionCarrier,
-): ToolDefinitionSource[] {
- const modelRequestParameters = parseRecord(
- carrier.metadataAttributes?.model_request_parameters,
+type PydanticMessage = {
+ kind: "request" | "response";
+ parts: unknown[];
+};
+
+function isPydanticMessage(value: unknown): value is PydanticMessage {
+ const message = asRecord(value);
+ return (
+ (message?.kind === "request" || message?.kind === "response") &&
+ Array.isArray(message.parts)
);
- const functionTools = modelRequestParameters?.function_tools;
- if (functionTools === undefined) return [];
+}
+
+// tryUnwrapMessage returns one message; split mixed-role Pydantic request parts in the claim instead.
+function pydanticAiMessage(message: PydanticMessage) {
+ if (message.kind === "response") {
+ return [{ role: "assistant", parts: message.parts }];
+ }
+ return message.parts.map((part) => {
+ const kind = asRecord(part)?.part_kind;
+ let role = "user";
+ if (kind === "system-prompt") role = "system";
+ if (kind === "tool-return") role = "tool";
+ // Prompt content may itself contain several multimodal parts.
+ if (kind === "system-prompt" || kind === "user-prompt") {
+ return { role, content: asRecord(part)?.content };
+ }
+ return { role, parts: [part] };
+ });
+}
+
+/** Extract native history without claiming unrelated application state. */
+function pydanticAiMessages(
+ root: Record<string, unknown>,
+ kind: "input" | "output",
+): MessageSource[] {
+ const history = parseArray(parseRecord(root._state)?.message_history);
+ if (!history?.length || !history.every(isPydanticMessage)) return [];
+ recordKeyAsParsed(root, "_state", "message_history");
return [
{
- sourceKey: "model_request_parameters.function_tools",
- value: functionTools,
- options: { allowToolMap: true },
+ kind: "sequence",
+ values: history.flatMap(pydanticAiMessage),
+ fallbackRole: kind === "input" ? "user" : "assistant",
},
];
}
+function pydanticAiPart(
+ part: Record<string, unknown>,
+ context: PartHandlerContext,
+) {
+ const { part_kind, tool_name, tool_call_id, args, ...rest } = part;
+ let value: Record<string, unknown>;
+ switch (part_kind) {
+ case "text":
+ value = { ...rest, type: "text" };
+ break;
+ case "tool-call":
+ value = {
+ ...rest,
+ type: "tool-call",
+ toolName: tool_name,
+ toolCallId: tool_call_id,
+ input: args,
+ };
+ break;
+ case "tool-return":
+ value = {
+ ...rest,
+ type: "tool-result",
+ toolName: tool_name,
+ toolCallId: tool_call_id,
+ output: part.content,
+ };
+ break;
+ default:
+ return unmatched;
+ }
+ return claimed(context.normalizePartValue(value)[0] ?? null);
+}
+
+/** Builtin tools use `kind` where the common tool parser expects `type`. */
+function pydanticAiBuiltinTool(value: unknown): unknown {
+ const tool = asRecord(value);
+ if (!tool || typeof tool.kind !== "string") return value;
+ const { kind, ...rest } = tool;
+ return { ...rest, type: kind };
+}
+
+function pydanticAiToolDefinitionSources(
+ carrier: ToolDefinitionCarrier,
+): ToolDefinitionSource[] {
+ const state = parseRecord(carrier.root?._state);
+ const sources: ToolDefinitionSource[] = [];
+ for (const [sourceKey, value] of [
+ [
+ "model_request_parameters",
+ carrier.metadataAttributes?.model_request_parameters,
+ ],
+ [
+ "_state.last_model_request_parameters",
+ state?.last_model_request_parameters,
+ ],
+ ] as const) {
+ const parameters = parseRecord(value);
+ for (const key of ["function_tools", "output_tools", "builtin_tools"]) {
+ if (parameters?.[key] === undefined) continue;
+ sources.push({
+ sourceKey: `${sourceKey}.${key}`,
+ value:
+ key === "builtin_tools"
+ ? (parseArray(parameters[key]) ?? []).map(pydanticAiBuiltinTool)
+ : parameters[key],
+ options: {
+ allowProviderToolWithoutName: key === "builtin_tools",
+ allowToolMap: key !== "builtin_tools",
+ },
+ });
+ }
+ }
+ return sources;
+}
+
export const pydanticAiProvider = {
name: "pydantic-ai",
+ claimMessages: pydanticAiMessages,
+ tryNormalizeUntypedPart: pydanticAiPart,
// Pydantic AI tool declarations: { name, description,
// parameters_json_schema }.
tryNormalizeToolDefinition: (value: Record<string, unknown>) => {
diff --git a/packages/shared/src/utils/normalized-io/conventions/providers/pydantic-ai/pydantic-ai.test.ts b/packages/shared/src/utils/normalized-io/conventions/providers/pydantic-ai/pydantic-ai.test.ts
index 73f763bbd7..5762591323 100644
--- a/packages/shared/src/utils/normalized-io/conventions/providers/pydantic-ai/pydantic-ai.test.ts
+++ b/packages/shared/src/utils/normalized-io/conventions/providers/pydantic-ai/pydantic-ai.test.ts
@@ -3,17 +3,38 @@ import { describe, expect, it } from "vitest";
import { normalizeSpanIO } from "../../../parser";
import {
capturedTraceFixtures,
+ agentRunFixture,
pydanticAiProductionShapeFixture,
} from "./fixtures";
describe("Pydantic AI normalized I/O", () => {
- it.each([...capturedTraceFixtures, pydanticAiProductionShapeFixture])(
- "$name",
- ({ spanIO, expected }) => {
- expect(normalizeSpanIO(spanIO)).toEqual({
- ...expected,
- span: spanIO,
- });
- },
- );
+ it("leaves unrelated message history as data", () => {
+ const output = {
+ _state: { message_history: [{ content: "An audit entry." }] },
+ };
+ expect(
+ normalizeSpanIO({ input: undefined, output, metadata: undefined }),
+ ).toEqual({
+ messages: [
+ {
+ role: "assistant",
+ source: "output",
+ parts: [{ type: "data", value: output }],
+ },
+ ],
+ toolDefinitions: [],
+ span: { input: undefined, output, metadata: undefined },
+ });
+ });
+
+ it.each([
+ ...capturedTraceFixtures,
+ pydanticAiProductionShapeFixture,
+ agentRunFixture,
+ ])("$name", ({ spanIO, expected }) => {
+ expect(normalizeSpanIO(spanIO)).toEqual({
+ ...expected,
+ span: spanIO,
+ });
+ });
});
diff --git a/packages/shared/src/utils/normalized-io/core/accumulator/collect.ts b/packages/shared/src/utils/normalized-io/core/accumulator/collect.ts
index ef0b699c6c..38923d9d09 100644
--- a/packages/shared/src/utils/normalized-io/core/accumulator/collect.ts
+++ b/packages/shared/src/utils/normalized-io/core/accumulator/collect.ts
@@ -249,6 +249,14 @@ function collectRecordMessages(
source === "input" ? findSystemMessageSources(root, source) : [];
const claimedSources = claimMessages(root, source, parserContext);
const rootMessages = parseArray(root.messages);
+ const paramsMessages =
+ source === "input"
+ ? parseArray(parseRecord(root.params)?.messages)
+ : undefined;
+ const hasMessageShapedParams = paramsMessages?.some((value) => {
+ const record = asRecord(value);
+ return record !== undefined && isMessageLike(record);
+ });
const messages: NormalizedMessage[] = [];
@@ -265,6 +273,15 @@ function collectRecordMessages(
messages,
accumulator,
);
+ } else if (paramsMessages && hasMessageShapedParams) {
+ recordKeyAsParsed(root, "params", "messages");
+ collectMessageSequence(
+ paramsMessages,
+ fallbackRole,
+ parserContext,
+ messages,
+ accumulator,
+ );
} else {
const recordMessage = normalizeMessage(root, fallbackRole, parserContext);
if (recordMessage) {
diff --git a/packages/shared/src/utils/normalized-io/core/normalize/part.ts b/packages/shared/src/utils/normalized-io/core/normalize/part.ts
index 0dde412256..d3f4873ec1 100644
--- a/packages/shared/src/utils/normalized-io/core/normalize/part.ts
+++ b/packages/shared/src/utils/normalized-io/core/normalize/part.ts
@@ -119,6 +119,7 @@ function normalizePartBase(
const COMMON_CONSUMED_PART_KEYS = [
"type",
+ "part_kind", // Pydantic part discriminator
"providerMetadata",
"providerOptions",
] as const;
@@ -156,8 +157,10 @@ const CONSUMED_PART_KEYS_BY_TYPE: Record<
"id",
"call_id",
"toolCallId",
+ "tool_call_id",
"name",
"toolName",
+ "tool_name",
"input",
"arguments",
"args",
@@ -174,6 +177,7 @@ const CONSUMED_PART_KEYS_BY_TYPE: Record<
"id",
"call_id",
"toolCallId",
+ "tool_call_id",
"tool_use_id",
"name",
"toolName",