The Open Framework for Building Agents
test(eve): reproduce unmarked input message omission on model spans (#4205)
roboeve[bot]4c6fc60a8f3e
diff --git a/.changeset/count-omitted-input-messages.md b/.changeset/count-omitted-input-messages.mdnew file mode 100644index 000000000..cadf1a5b2--- /dev/null+++ b/.changeset/count-omitted-input-messages.md@@ -0,0 +1,5 @@+---+"eve": patch+---++Record `agent.input.messages.omitted` on model spans when the oldest prompt messages are dropped to fit the 32 KB content cap, so a long conversation no longer looks like it starts partway through.diff --git a/docs/reference/cli.md b/docs/reference/cli.mdindex 5ed67c1fa..060e45458 100644--- a/docs/reference/cli.md+++ b/docs/reference/cli.md@@ -403,7 +403,7 @@ Each root agent turn starts a new trace. With local caller trace context, that tEvery span carries a real duration. A turn's root `invoke_agent` span is written when the turn settles, so a running turn shows only its steps.-Model, `execute_tool`, and memory spans retain their content by default. Set `EVE_TRACES_CONTENT=off` to omit system prompts, prompt messages, and response text for models; call arguments and results for tools; and recalled memory records. Each captured value is capped at 32 KB.+Model, `execute_tool`, and memory spans retain their content by default. Set `EVE_TRACES_CONTENT=off` to omit system prompts, prompt messages, and response text for models; call arguments and results for tools; and recalled memory records. Each captured value is capped at 32 KB. When a model call's prompt messages exceed the cap, the oldest are dropped first and `agent.input.messages.omitted` records how many.Step spans carry token counts under `agent.usage.*`, and cost when Vercel AI Gateway served the call. Model spans also expose `gen_ai.usage.*` token counters. The CLI sums step-level counters only, so model and delegated-call totals are not counted twice.diff --git a/packages/eve/src/tracing/eve/agent-otel-provider.test.ts b/packages/eve/src/tracing/eve/agent-otel-provider.test.tsindex d408c4fa7..8923c0b87 100644--- a/packages/eve/src/tracing/eve/agent-otel-provider.test.ts+++ b/packages/eve/src/tracing/eve/agent-otel-provider.test.ts@@ -2839,7 +2839,7 @@ describe("createAgentOtelInstrumentation", () => {);});- it("caps full model input while keeping valid message JSON", async () => {+ it("caps full model input, keeps valid message JSON, and counts omitted messages", async () => {const runtime = createRuntime();const manyMessages = Array.from({ length: 200 }, (_, index) => ({content: `message ${index} ${"x".repeat(200)}`,@@ -2906,6 +2906,9 @@ describe("createAgentOtelInstrumentation", () => {expect(parsed.length).toBeGreaterThan(1);expect(JSON.stringify(parsed)).toContain("message 199");expect(JSON.stringify(parsed)).not.toContain("message 0 ");+ expect(model.attributes["agent.input.messages.omitted"]).toBe(+ manyMessages.length - parsed.length,+ );expect(model.attributes["agent.input.messages.delta"]).toBeUndefined();});diff --git a/packages/tracing/src/adapters/serialization.test.ts b/packages/tracing/src/adapters/serialization.test.tsindex 448baee35..22fffb0d1 100644--- a/packages/tracing/src/adapters/serialization.test.ts+++ b/packages/tracing/src/adapters/serialization.test.ts@@ -45,7 +45,7 @@ describe("GenAI message attributes", () => {{ type: "file", mediaType: "application/octet-stream", filename: "test.bin", data },],},- ]);+ ])?.json;} finally {spy.mockRestore();}@@ -65,7 +65,7 @@ describe("GenAI message attributes", () => {kind: "execution.continuation",role: "user",},- ]),+ ])?.json,).toBe('[{"kind":"user","parts":[{"content":"A real user message.","type":"text"}],"role":"user"},{"kind":"execution.continuation","parts":[{"content":"Continue the interrupted turn.","type":"text"}],"role":"user"}]',);@@ -74,7 +74,7 @@ describe("GenAI message attributes", () => {it.each(FRAMEWORK_MESSAGE_KINDS)("preserves %s in the GenAI input attribute", (kind) => {const attribute = genAiInputMessagesAttribute([{ content: "Framework message.", kind, role: "user" },- ]);+ ])?.json;expect(JSON.parse(attribute!)).toEqual([{@@ -86,7 +86,7 @@ describe("GenAI message attributes", () => {});it.each([- ["only", [{ content: "x".repeat(CONTENT_ATTRIBUTE_LIMIT + 1), role: "user" }], undefined],+ ["only", [{ content: "x".repeat(CONTENT_ATTRIBUTE_LIMIT + 1), role: "user" }], undefined, 0],["newest",[@@ -98,21 +98,26 @@ describe("GenAI message attributes", () => {},],"context.state",+ 1,],- ] as const)("keeps a truncated %s message when it alone exceeds the cap", (_, messages, kind) => {- const attribute = genAiInputMessagesAttribute(messages);+ ] as const)(+ "keeps a truncated %s message when it alone exceeds the cap",+ (_, messages, kind, omitted) => {+ const result = genAiInputMessagesAttribute(messages);- expect(attribute).toBeDefined();- expect(attribute!.length).toBeLessThanOrEqual(CONTENT_ATTRIBUTE_LIMIT);- const parsed = JSON.parse(attribute!) as Array<Record<string, unknown>>;- expect(parsed).toHaveLength(1);- expect(parsed[0]).toMatchObject({- parts: [{ content: expect.stringMatching(/… \[truncated\]$/u), type: "text" }],- role: "user",- });- expect(parsed[0]?.kind).toBe(kind);- expect(attribute).not.toContain("older message");- });+ expect(result).toBeDefined();+ expect(result!.omitted).toBe(omitted);+ expect(result!.json.length).toBeLessThanOrEqual(CONTENT_ATTRIBUTE_LIMIT);+ const parsed = JSON.parse(result!.json) as Array<Record<string, unknown>>;+ expect(parsed).toHaveLength(1);+ expect(parsed[0]).toMatchObject({+ parts: [{ content: expect.stringMatching(/… \[truncated\]$/u), type: "text" }],+ role: "user",+ });+ expect(parsed[0]?.kind).toBe(kind);+ expect(result!.json).not.toContain("older message");+ },+ );it("formats model input, output, and system instructions for inspectors", () => {expect(@@ -129,7 +134,7 @@ describe("GenAI message attributes", () => {],role: "assistant",},- ]),+ ])?.json,).toBe('[{"kind":"user","parts":[{"content":"hello","type":"text"}],"role":"user"},{"parts":[{"arguments":{"message":"echo"},"id":"call-1","name":"delegate","type":"tool_call"}],"role":"assistant"}]',);@@ -183,7 +188,7 @@ describe("GenAI message attributes", () => {],role: "assistant",},- ]);+ ])?.json;expect(attribute).toBeDefined();expect(attribute!.length).toBeLessThanOrEqual(CONTENT_ATTRIBUTE_LIMIT);diff --git a/packages/tracing/src/adapters/serialization.ts b/packages/tracing/src/adapters/serialization.tsindex b04878da1..c199d9dd3 100644--- a/packages/tracing/src/adapters/serialization.ts+++ b/packages/tracing/src/adapters/serialization.ts@@ -8,7 +8,7 @@/** Content attributes are capped so a giant payload cannot bloat a span segment. */export const CONTENT_ATTRIBUTE_LIMIT = 32 * 1024;-import type { ContentSerializer } from "../core/types.js";+import type { ContentSerializer, InputMessagesContent } from "../core/types.js";export const aiSdkContentSerializer: ContentSerializer = {json: contentAttribute,@@ -74,7 +74,7 @@ export function contentAttribute(value: unknown): string | undefined {export function genAiInputMessagesAttribute(messages: unknown,messageKind?: (value: unknown) => string | undefined,-): string | undefined {+): InputMessagesContent | undefined {if (!Array.isArray(messages)) return undefined;const formatted = messages.flatMap((message) => {if (!isRecord(message) || message.role === "system" || typeof message.role !== "string") {@@ -89,9 +89,11 @@ export function genAiInputMessagesAttribute(});for (let start = 0; start < formatted.length; start += 1) {const json = semanticJsonAttribute(formatted.slice(start));- if (json !== undefined) return json;+ if (json !== undefined) return { json, omitted: start };}- return truncateSingleSemanticMessage(formatted) ?? semanticJsonAttribute([]);+ const newest = truncateSingleSemanticMessage(formatted);+ if (newest !== undefined) return { json: newest, omitted: formatted.length - 1 };+ return { json: "[]", omitted: formatted.length };}/** Serializes the system prompt using the OpenTelemetry GenAI instruction schema. */diff --git a/packages/tracing/src/core/snapshot.ts b/packages/tracing/src/core/snapshot.tsindex c17e514ea..750d6468e 100644--- a/packages/tracing/src/core/snapshot.ts+++ b/packages/tracing/src/core/snapshot.ts@@ -12,8 +12,11 @@ export function boundedSerializer(onError?: import("./types.js").TraceErrorHandler,): ContentSerializer {function invoke(method: keyof ContentSerializer, args: unknown[]): string | undefined {+ return bounded(() => Reflect.apply(serializer[method], serializer, args) as string | undefined);+ }+ function bounded(serialize: () => string | undefined): string | undefined {try {- const value = Reflect.apply(serializer[method], serializer, args) as string | undefined;+ const value = serialize();return value === undefined || new TextEncoder().encode(value).length > SERIALIZED_BYTES? undefined: value;@@ -27,7 +30,15 @@ export function boundedSerializer(return {json: (value) => invoke("json", [value]),text: (value) => invoke("text", [value]),- inputMessages: (value) => invoke("inputMessages", [value]),+ inputMessages: (value) => {+ let omitted = 0;+ const json = bounded(() => {+ const content = serializer.inputMessages(value);+ omitted = content?.omitted ?? 0;+ return content?.json;+ });+ return json === undefined ? undefined : { json, omitted };+ },instructions: (value) => invoke("instructions", [value]),toolDefinitions: (value) => invoke("toolDefinitions", [value]),outputMessages: (value, reason) => invoke("outputMessages", [value, reason]),diff --git a/packages/tracing/src/core/span-kinds.ts b/packages/tracing/src/core/span-kinds.tsindex e27bce3a2..b62cfef2f 100644--- a/packages/tracing/src/core/span-kinds.ts+++ b/packages/tracing/src/core/span-kinds.ts@@ -153,6 +153,10 @@ const kinds: { [K in ScopeData["type"]]: Kind<K> } = {model: {parents: ["step"],prepare({ options }, context) {+ const inputMessages =+ context.capture.recordInputs && options.messages !== undefined+ ? context.serializer.inputMessages(options.messages)+ : undefined;return {name: modelName(options.modelId),kind: "CLIENT",@@ -166,7 +170,8 @@ const kinds: { [K in ScopeData["type"]]: Kind<K> } = {...runtimeContextAttributes(options.runtimeContext),...(context.capture.recordInputs && options.messages !== undefined? {- "gen_ai.input.messages": context.serializer.inputMessages(options.messages),+ "gen_ai.input.messages": inputMessages?.json,+ "agent.input.messages.omitted": inputMessages?.omitted || undefined,"gen_ai.system_instructions": context.serializer.instructions(options.instructions),}: undefined),diff --git a/packages/tracing/src/core/types.ts b/packages/tracing/src/core/types.tsindex 6fa1f7fd1..8049dd0ce 100644--- a/packages/tracing/src/core/types.ts+++ b/packages/tracing/src/core/types.ts@@ -147,10 +147,15 @@ export interface Usage {/** `subagent-call` and `remote-agent-call` delegate to another agent. */export type ToolKind = string;+/** Capped GenAI input messages and how many of the oldest were dropped to fit. */+export interface InputMessagesContent {+ readonly json: string;+ readonly omitted: number;+}export interface ContentSerializer {json(value: unknown): string | undefined;text(value: string): string | undefined;- inputMessages(value: unknown): string | undefined;+ inputMessages(value: unknown): InputMessagesContent | undefined;instructions(value: unknown): string | undefined;toolDefinitions(value: unknown): string | undefined;outputMessages(value: readonly ContentPart[], finishReason: string): string | undefined;