Skip to content

armgit / eve

Public Imported from GitHub
0

The Open Framework for Building Agents

test(eve): reproduce unmarked input message omission on model spans (#4205)

roboeve[bot]4c6fc60a8f3e
diff --git a/.changeset/count-omitted-input-messages.md b/.changeset/count-omitted-input-messages.md
new file mode 100644
index 000000000..cadf1a5b2
--- /dev/null
+++ b/.changeset/count-omitted-input-messages.md
@@ -0,0 +1,5 @@
+---
+"eve": patch
+---
+
+Record `agent.input.messages.omitted` on model spans when the oldest prompt messages are dropped to fit the 32 KB content cap, so a long conversation no longer looks like it starts partway through.
diff --git a/docs/reference/cli.md b/docs/reference/cli.md
index 5ed67c1fa..060e45458 100644
--- a/docs/reference/cli.md
+++ b/docs/reference/cli.md
@@ -403,7 +403,7 @@ Each root agent turn starts a new trace. With local caller trace context, that t
Every span carries a real duration. A turn's root `invoke_agent` span is written when the turn settles, so a running turn shows only its steps.
-Model, `execute_tool`, and memory spans retain their content by default. Set `EVE_TRACES_CONTENT=off` to omit system prompts, prompt messages, and response text for models; call arguments and results for tools; and recalled memory records. Each captured value is capped at 32 KB.
+Model, `execute_tool`, and memory spans retain their content by default. Set `EVE_TRACES_CONTENT=off` to omit system prompts, prompt messages, and response text for models; call arguments and results for tools; and recalled memory records. Each captured value is capped at 32 KB. When a model call's prompt messages exceed the cap, the oldest are dropped first and `agent.input.messages.omitted` records how many.
Step spans carry token counts under `agent.usage.*`, and cost when Vercel AI Gateway served the call. Model spans also expose `gen_ai.usage.*` token counters. The CLI sums step-level counters only, so model and delegated-call totals are not counted twice.
diff --git a/packages/eve/src/tracing/eve/agent-otel-provider.test.ts b/packages/eve/src/tracing/eve/agent-otel-provider.test.ts
index d408c4fa7..8923c0b87 100644
--- a/packages/eve/src/tracing/eve/agent-otel-provider.test.ts
+++ b/packages/eve/src/tracing/eve/agent-otel-provider.test.ts
@@ -2839,7 +2839,7 @@ describe("createAgentOtelInstrumentation", () => {
);
});
- it("caps full model input while keeping valid message JSON", async () => {
+ it("caps full model input, keeps valid message JSON, and counts omitted messages", async () => {
const runtime = createRuntime();
const manyMessages = Array.from({ length: 200 }, (_, index) => ({
content: `message ${index} ${"x".repeat(200)}`,
@@ -2906,6 +2906,9 @@ describe("createAgentOtelInstrumentation", () => {
expect(parsed.length).toBeGreaterThan(1);
expect(JSON.stringify(parsed)).toContain("message 199");
expect(JSON.stringify(parsed)).not.toContain("message 0 ");
+ expect(model.attributes["agent.input.messages.omitted"]).toBe(
+ manyMessages.length - parsed.length,
+ );
expect(model.attributes["agent.input.messages.delta"]).toBeUndefined();
});
diff --git a/packages/tracing/src/adapters/serialization.test.ts b/packages/tracing/src/adapters/serialization.test.ts
index 448baee35..22fffb0d1 100644
--- a/packages/tracing/src/adapters/serialization.test.ts
+++ b/packages/tracing/src/adapters/serialization.test.ts
@@ -45,7 +45,7 @@ describe("GenAI message attributes", () => {
{ type: "file", mediaType: "application/octet-stream", filename: "test.bin", data },
],
},
- ]);
+ ])?.json;
} finally {
spy.mockRestore();
}
@@ -65,7 +65,7 @@ describe("GenAI message attributes", () => {
kind: "execution.continuation",
role: "user",
},
- ]),
+ ])?.json,
).toBe(
'[{"kind":"user","parts":[{"content":"A real user message.","type":"text"}],"role":"user"},{"kind":"execution.continuation","parts":[{"content":"Continue the interrupted turn.","type":"text"}],"role":"user"}]',
);
@@ -74,7 +74,7 @@ describe("GenAI message attributes", () => {
it.each(FRAMEWORK_MESSAGE_KINDS)("preserves %s in the GenAI input attribute", (kind) => {
const attribute = genAiInputMessagesAttribute([
{ content: "Framework message.", kind, role: "user" },
- ]);
+ ])?.json;
expect(JSON.parse(attribute!)).toEqual([
{
@@ -86,7 +86,7 @@ describe("GenAI message attributes", () => {
});
it.each([
- ["only", [{ content: "x".repeat(CONTENT_ATTRIBUTE_LIMIT + 1), role: "user" }], undefined],
+ ["only", [{ content: "x".repeat(CONTENT_ATTRIBUTE_LIMIT + 1), role: "user" }], undefined, 0],
[
"newest",
[
@@ -98,21 +98,26 @@ describe("GenAI message attributes", () => {
},
],
"context.state",
+ 1,
],
- ] as const)("keeps a truncated %s message when it alone exceeds the cap", (_, messages, kind) => {
- const attribute = genAiInputMessagesAttribute(messages);
+ ] as const)(
+ "keeps a truncated %s message when it alone exceeds the cap",
+ (_, messages, kind, omitted) => {
+ const result = genAiInputMessagesAttribute(messages);
- expect(attribute).toBeDefined();
- expect(attribute!.length).toBeLessThanOrEqual(CONTENT_ATTRIBUTE_LIMIT);
- const parsed = JSON.parse(attribute!) as Array<Record<string, unknown>>;
- expect(parsed).toHaveLength(1);
- expect(parsed[0]).toMatchObject({
- parts: [{ content: expect.stringMatching(/… \[truncated\]$/u), type: "text" }],
- role: "user",
- });
- expect(parsed[0]?.kind).toBe(kind);
- expect(attribute).not.toContain("older message");
- });
+ expect(result).toBeDefined();
+ expect(result!.omitted).toBe(omitted);
+ expect(result!.json.length).toBeLessThanOrEqual(CONTENT_ATTRIBUTE_LIMIT);
+ const parsed = JSON.parse(result!.json) as Array<Record<string, unknown>>;
+ expect(parsed).toHaveLength(1);
+ expect(parsed[0]).toMatchObject({
+ parts: [{ content: expect.stringMatching(/… \[truncated\]$/u), type: "text" }],
+ role: "user",
+ });
+ expect(parsed[0]?.kind).toBe(kind);
+ expect(result!.json).not.toContain("older message");
+ },
+ );
it("formats model input, output, and system instructions for inspectors", () => {
expect(
@@ -129,7 +134,7 @@ describe("GenAI message attributes", () => {
],
role: "assistant",
},
- ]),
+ ])?.json,
).toBe(
'[{"kind":"user","parts":[{"content":"hello","type":"text"}],"role":"user"},{"parts":[{"arguments":{"message":"echo"},"id":"call-1","name":"delegate","type":"tool_call"}],"role":"assistant"}]',
);
@@ -183,7 +188,7 @@ describe("GenAI message attributes", () => {
],
role: "assistant",
},
- ]);
+ ])?.json;
expect(attribute).toBeDefined();
expect(attribute!.length).toBeLessThanOrEqual(CONTENT_ATTRIBUTE_LIMIT);
diff --git a/packages/tracing/src/adapters/serialization.ts b/packages/tracing/src/adapters/serialization.ts
index b04878da1..c199d9dd3 100644
--- a/packages/tracing/src/adapters/serialization.ts
+++ b/packages/tracing/src/adapters/serialization.ts
@@ -8,7 +8,7 @@
/** Content attributes are capped so a giant payload cannot bloat a span segment. */
export const CONTENT_ATTRIBUTE_LIMIT = 32 * 1024;
-import type { ContentSerializer } from "../core/types.js";
+import type { ContentSerializer, InputMessagesContent } from "../core/types.js";
export const aiSdkContentSerializer: ContentSerializer = {
json: contentAttribute,
@@ -74,7 +74,7 @@ export function contentAttribute(value: unknown): string | undefined {
export function genAiInputMessagesAttribute(
messages: unknown,
messageKind?: (value: unknown) => string | undefined,
-): string | undefined {
+): InputMessagesContent | undefined {
if (!Array.isArray(messages)) return undefined;
const formatted = messages.flatMap((message) => {
if (!isRecord(message) || message.role === "system" || typeof message.role !== "string") {
@@ -89,9 +89,11 @@ export function genAiInputMessagesAttribute(
});
for (let start = 0; start < formatted.length; start += 1) {
const json = semanticJsonAttribute(formatted.slice(start));
- if (json !== undefined) return json;
+ if (json !== undefined) return { json, omitted: start };
}
- return truncateSingleSemanticMessage(formatted) ?? semanticJsonAttribute([]);
+ const newest = truncateSingleSemanticMessage(formatted);
+ if (newest !== undefined) return { json: newest, omitted: formatted.length - 1 };
+ return { json: "[]", omitted: formatted.length };
}
/** Serializes the system prompt using the OpenTelemetry GenAI instruction schema. */
diff --git a/packages/tracing/src/core/snapshot.ts b/packages/tracing/src/core/snapshot.ts
index c17e514ea..750d6468e 100644
--- a/packages/tracing/src/core/snapshot.ts
+++ b/packages/tracing/src/core/snapshot.ts
@@ -12,8 +12,11 @@ export function boundedSerializer(
onError?: import("./types.js").TraceErrorHandler,
): ContentSerializer {
function invoke(method: keyof ContentSerializer, args: unknown[]): string | undefined {
+ return bounded(() => Reflect.apply(serializer[method], serializer, args) as string | undefined);
+ }
+ function bounded(serialize: () => string | undefined): string | undefined {
try {
- const value = Reflect.apply(serializer[method], serializer, args) as string | undefined;
+ const value = serialize();
return value === undefined || new TextEncoder().encode(value).length > SERIALIZED_BYTES
? undefined
: value;
@@ -27,7 +30,15 @@ export function boundedSerializer(
return {
json: (value) => invoke("json", [value]),
text: (value) => invoke("text", [value]),
- inputMessages: (value) => invoke("inputMessages", [value]),
+ inputMessages: (value) => {
+ let omitted = 0;
+ const json = bounded(() => {
+ const content = serializer.inputMessages(value);
+ omitted = content?.omitted ?? 0;
+ return content?.json;
+ });
+ return json === undefined ? undefined : { json, omitted };
+ },
instructions: (value) => invoke("instructions", [value]),
toolDefinitions: (value) => invoke("toolDefinitions", [value]),
outputMessages: (value, reason) => invoke("outputMessages", [value, reason]),
diff --git a/packages/tracing/src/core/span-kinds.ts b/packages/tracing/src/core/span-kinds.ts
index e27bce3a2..b62cfef2f 100644
--- a/packages/tracing/src/core/span-kinds.ts
+++ b/packages/tracing/src/core/span-kinds.ts
@@ -153,6 +153,10 @@ const kinds: { [K in ScopeData["type"]]: Kind<K> } = {
model: {
parents: ["step"],
prepare({ options }, context) {
+ const inputMessages =
+ context.capture.recordInputs && options.messages !== undefined
+ ? context.serializer.inputMessages(options.messages)
+ : undefined;
return {
name: modelName(options.modelId),
kind: "CLIENT",
@@ -166,7 +170,8 @@ const kinds: { [K in ScopeData["type"]]: Kind<K> } = {
...runtimeContextAttributes(options.runtimeContext),
...(context.capture.recordInputs && options.messages !== undefined
? {
- "gen_ai.input.messages": context.serializer.inputMessages(options.messages),
+ "gen_ai.input.messages": inputMessages?.json,
+ "agent.input.messages.omitted": inputMessages?.omitted || undefined,
"gen_ai.system_instructions": context.serializer.instructions(options.instructions),
}
: undefined),
diff --git a/packages/tracing/src/core/types.ts b/packages/tracing/src/core/types.ts
index 6fa1f7fd1..8049dd0ce 100644
--- a/packages/tracing/src/core/types.ts
+++ b/packages/tracing/src/core/types.ts
@@ -147,10 +147,15 @@ export interface Usage {
/** `subagent-call` and `remote-agent-call` delegate to another agent. */
export type ToolKind = string;
+/** Capped GenAI input messages and how many of the oldest were dropped to fit. */
+export interface InputMessagesContent {
+ readonly json: string;
+ readonly omitted: number;
+}
export interface ContentSerializer {
json(value: unknown): string | undefined;
text(value: string): string | undefined;
- inputMessages(value: unknown): string | undefined;
+ inputMessages(value: unknown): InputMessagesContent | undefined;
instructions(value: unknown): string | undefined;
toolDefinitions(value: unknown): string | undefined;
outputMessages(value: readonly ContentPart[], finishReason: string): string | undefined;