The Open Framework for Building Agents
feat(eve): capture model and tool content on local trace spans (#1443)
Chad Hietalabf01952a6146
diff --git a/.changeset/local-span-content.md b/.changeset/local-span-content.mdnew file mode 100644index 000000000..848016477--- /dev/null+++ b/.changeset/local-span-content.md@@ -0,0 +1,5 @@+---+"eve": patch+---++Local trace spans now capture model and tool payloads: the system prompt, prompt messages, and response text/reasoning/tool calls on model spans, and call arguments/results on tool spans, each capped at 32 KB with provider transport metadata stripped. Set `EVE_TRACES_CONTENT=off` to keep payloads out of the spool.diff --git a/docs/guides/instrumentation.md b/docs/guides/instrumentation.mdindex 78305a9df..d62f6b2bc 100644--- a/docs/guides/instrumentation.md+++ b/docs/guides/instrumentation.md@@ -13,7 +13,7 @@ When no authored `instrumentation.ts` exists, `eve dev` records agent, AI SDK, aThe directory is an immutable OTLP/JSON spool and remains available after `eve dev` exits, subject to the [retention policy](#local-trace-retention) below. Inspection tools may build a query index from these segments, but the index is derived and can be rebuilt without changing the captured trace data.-Use `eve traces ls` to list captured traces and `eve traces <trace>` to inspect a session's span tree.+Use `eve traces ls` to list captured traces and `eve traces <trace>` to inspect a session's span tree. Model and tool-call spans also carry the inputs and outputs (system prompt, prompt messages, and response text for models; call arguments and results for tools, each capped at 32 KB) — set `EVE_TRACES_CONTENT=off` to keep payloads out of the spool.When a model call is served by Vercel AI Gateway, its `agent.step` span also carries the cost the gateway reported: `gen_ai.usage.cost` (raw inference, USD), `gen_ai.usage.gateway_cost` (with the gateway surcharge), `gen_ai.usage.input_cost` / `gen_ai.usage.output_cost` (the split), and `gen_ai.generation.id` for reconciliation with the gateway dashboard. These attributes only exist for gateway-served calls — other providers emit nothing. Cost per turn is the sum across the turn's step spans.@@ -34,12 +34,13 @@ Whatever survives is then evicted oldest-first while the store exceeds `EVE_TRACOpen sessions are tracked per dev worker, so a session waiting across a restart, or a trace a draining worker is still writing, is not covered by the first rule. The five-minute write-recency rule is what protects those, and it applies even when you set `EVE_TRACES_MAX_AGE_MS=0`.-| Variable | Default | Effect |-| ---------------------------- | -------------------- | --------------------------------------------- |-| `EVE_TRACES` | on | `off` stops writing traces and stops sweeping |-| `EVE_TRACES_MAX_AGE_MS` | `604800000` (7d) | Age after which a trace may be evicted |-| `EVE_TRACES_MAX_TOTAL_BYTES` | `536870912` (512 MB) | Size budget for the whole store |-| `EVE_TRACES_RETAIN_COUNT` | `20` | Newest traces kept regardless of age or size |+| Variable | Default | Effect |+| ---------------------------- | -------------------- | ------------------------------------------------------------------------------------------- |+| `EVE_TRACES` | on | `off` stops writing traces and stops sweeping |+| `EVE_TRACES_CONTENT` | on | `off` stops capturing model prompt/response and tool input/output attributes on local spans |+| `EVE_TRACES_MAX_AGE_MS` | `604800000` (7d) | Age after which a trace may be evicted |+| `EVE_TRACES_MAX_TOTAL_BYTES` | `536870912` (512 MB) | Size budget for the whole store |+| `EVE_TRACES_RETAIN_COUNT` | `20` | Newest traces kept regardless of age or size |Set these in `.env.local`, which `eve dev` loads automatically. Each bound accepts `off` to disable it individually — note that `EVE_TRACES_RETAIN_COUNT=off` removes the keep-newest guarantee rather than retaining everything.diff --git a/packages/eve/src/harness/agent-otel-content.ts b/packages/eve/src/harness/agent-otel-content.tsnew file mode 100644index 000000000..17cc53458--- /dev/null+++ b/packages/eve/src/harness/agent-otel-content.ts@@ -0,0 +1,121 @@+/**+ * Serializes model and tool payloads into span content attributes for the+ * local trace viewer: prompt messages, the system prompt, responses, and+ * tool arguments/results. Everything is capped so a giant payload cannot+ * bloat a span segment past the reader's per-file limit, and message+ * payloads stay valid JSON through every truncation path.+ */++/** Content attributes are capped so a giant payload cannot bloat a span segment. */+export const CONTENT_ATTRIBUTE_LIMIT = 32 * 1024;++/** Provider transport noise stripped from messages (not tool data). */+const CONTENT_NOISE_KEYS = new Set(["providerOptions", "providerMetadata"]);++/** Marker prepended when oldest messages are dropped to fit the content cap. */+const TRUNCATED_MESSAGES_KEY = "eve.truncated";++/**+ * Serializes one payload value. `strip` removes {@link CONTENT_NOISE_KEYS};+ * pass `false` for tool data, where those keys may be legitimate domain+ * fields the user authored or the tool returned.+ */+export function contentAttribute(value: unknown, strip = true): string | undefined {+ if (value === undefined) return undefined;+ const prepared = strip ? stripContentNoise(value, 0) : value;+ let json: string | undefined;+ try {+ json = JSON.stringify(prepared);+ } catch {+ return undefined;+ }+ if (json === undefined) return undefined;+ return textContentAttribute(json);+}++/**+ * Serializes the prompt messages, keeping the result parseable: oldest+ * messages drop first behind an omission marker, and a single message over+ * the cap is truncated at the text level rather than the JSON level.+ */+export function messagesContentAttribute(messages: unknown): string | undefined {+ if (!Array.isArray(messages)) return contentAttribute(messages);+ const stripped = stripContentNoise(messages, 0) as unknown[];+ const full = stringifyContent(stripped);+ if (full !== undefined && full.length <= CONTENT_ATTRIBUTE_LIMIT) return full;+ for (let omitted = 1; omitted < stripped.length; omitted += 1) {+ const json = stringifyContent([+ { [TRUNCATED_MESSAGES_KEY]: { omittedMessages: omitted } },+ ...stripped.slice(omitted),+ ]);+ if (json !== undefined && json.length <= CONTENT_ATTRIBUTE_LIMIT) return json;+ }+ return truncateSingleMessage(stripped);+}++/** Normalizes the AI SDK's `instructions` prompt to plain text for `ai.prompt.system`. */+export function systemPromptAttribute(instructions: unknown): string | undefined {+ if (typeof instructions === "string") return textContentAttribute(instructions);+ if (!isRecord(instructions) && !Array.isArray(instructions)) return undefined;+ const messages = Array.isArray(instructions) ? instructions : [instructions];+ const texts: string[] = [];+ for (const message of messages) {+ if (!isRecord(message)) continue;+ if (typeof message.content === "string") texts.push(message.content);+ else if (Array.isArray(message.content))+ for (const part of message.content)+ if (isRecord(part) && part.type === "text" && typeof part.text === "string")+ texts.push(part.text);+ }+ const joined = texts.join("\n\n").trim();+ return joined.length === 0 ? undefined : textContentAttribute(joined);+}++/** Caps plain text, marking the cut. */+export function textContentAttribute(text: string): string | undefined {+ if (text.length === 0) return undefined;+ return text.length <= CONTENT_ATTRIBUTE_LIMIT+ ? text+ : `${text.slice(0, CONTENT_ATTRIBUTE_LIMIT)}… [truncated]`;+}++function stripContentNoise(value: unknown, depth: number): unknown {+ if (depth > 32 || value === null || typeof value !== "object") return value;+ if (Array.isArray(value)) return value.map((e) => stripContentNoise(e, depth + 1));+ const out: Record<string, unknown> = {};+ for (const [k, v] of Object.entries(value))+ if (!CONTENT_NOISE_KEYS.has(k)) out[k] = stripContentNoise(v, depth + 1);+ return out;+}++function truncateSingleMessage(messages: unknown[]): string | undefined {+ if (messages.length === 0) return undefined;+ const last = messages[messages.length - 1];+ if (!isRecord(last)) return undefined;+ const marker = { [TRUNCATED_MESSAGES_KEY]: { omittedMessages: messages.length - 1 } };+ const role = typeof last.role === "string" ? last.role : "user";+ let text = "";+ if (typeof last.content === "string") text = last.content;+ else if (Array.isArray(last.content))+ for (const part of last.content)+ if (isRecord(part) && part.type === "text" && typeof part.text === "string")+ text += (text ? "\n" : "") + part.text;+ for (let len = Math.min(text.length, CONTENT_ATTRIBUTE_LIMIT); len > 0; len -= 256) {+ const cut = len >= text.length ? text : `${text.slice(0, len)}… [truncated]`;+ const json = stringifyContent([marker, { role, content: cut }]);+ if (json !== undefined && json.length <= CONTENT_ATTRIBUTE_LIMIT) return json;+ }+ return stringifyContent([marker, { role, content: "" }]);+}++function stringifyContent(value: unknown): string | undefined {+ try {+ return JSON.stringify(value);+ } catch {+ return undefined;+ }+}++function isRecord(value: unknown): value is Record<string, unknown> {+ return typeof value === "object" && value !== null && !Array.isArray(value);+}diff --git a/packages/eve/src/harness/agent-otel-provider.test.ts b/packages/eve/src/harness/agent-otel-provider.test.tsindex d3e3689a2..be93e5b95 100644--- a/packages/eve/src/harness/agent-otel-provider.test.ts+++ b/packages/eve/src/harness/agent-otel-provider.test.ts@@ -79,13 +79,28 @@ async function emitAttempt(input: {]);await Reflect.apply(bridge.onStepStart!, bridge, [{ callId: "call-1", stepNumber: 0 }]);await Reflect.apply(bridge.onLanguageModelCallStart!, bridge, [- { callId: "call-1", messages: [], modelId: "claude-test", provider: "anthropic" },+ {+ callId: "call-1",+ instructions: "You are a weather assistant (system prompt).",+ messages: [+ {+ content: "real user text",+ providerOptions: { anthropic: { signature: "sig-blob" } },+ role: "user",+ },+ ],+ modelId: "claude-test",+ provider: "anthropic",+ },]);await bridge.executeLanguageModelCall!({ callId: "call-1", execute: async () => undefined });await Reflect.apply(bridge.onLanguageModelCallEnd!, bridge, [{callId: "call-1",- content: [],+ content: [+ { type: "reasoning", text: "thinking about weather" },+ { type: "text", text: "Checking the weather." },+ ],finishReason: "tool-calls",performance: { responseTimeMs: 10 },responseId: "response-1",@@ -231,7 +246,134 @@ describe("createAgentOtelInstrumentation", () => {"agent.framework.name": "eve","agent.root.session.id": "session-1",});+ });++ it("captures model and tool inputs/outputs on the operation spans", async () => {+ const runtime = createRuntime();+ await emitAttempt({+ hooks: runtime.hooks,+ runInContext: runtime.runInContext,+ sessionId: "session-1",+ turnId: "turn-1",+ turnSequence: 0,+ });+ await runtime.provider.forceFlush();++ const spans = runtime.exporter.getFinishedSpans();+ const model = byName(spans, "ai.streamText.doStream")[0]!;+ const tool = byName(spans, "ai.toolCall")[0]!;+ // Provider transport noise (signatures et al.) is stripped at capture time.+ expect(model.attributes["ai.prompt.messages"]).toBe(+ '[{"content":"real user text","role":"user"}]',+ );+ expect(model.attributes["ai.prompt.system"]).toBe(+ "You are a weather assistant (system prompt).",+ );+ expect(model.attributes["ai.response.finish_reason"]).toBe("tool-calls");+ expect(model.attributes["ai.response.reasoning"]).toBe("thinking about weather");+ expect(model.attributes["ai.response.text"]).toBe("Checking the weather.");+ expect(tool.attributes["gen_ai.tool.call.arguments"]).toBe('{"secret":"value"}');+ expect(tool.attributes["gen_ai.tool.call.result"]).toBe('{"temperature":72}');+ // Structural spans stay structural: content lives only on the operation spans.+ const structural = byName(spans, "agent.action")[0]!;+ expect(JSON.stringify(structural.attributes)).not.toContain("secret");+ expect(JSON.stringify(structural.attributes)).not.toContain("temperature");+ });++ it("truncates long conversations from the front, keeping valid JSON and recent messages", async () => {+ const runtime = createRuntime();+ const manyMessages = Array.from({ length: 200 }, (_, index) => ({+ content: `message ${index} ${"x".repeat(200)}`,+ role: index % 2 === 0 ? "user" : "assistant",+ }));+ const scope: InstrumentationAttemptScope = {+ attemptId: "session-1:turn-1:0:0",+ attemptIndex: 0,+ functionId: "weather",+ sessionId: "session-1",+ stepIndex: 0,+ turnId: "turn-1",+ };+ await runtime.hooks.publish({+ agentName: "weather",+ channelKind: "http",+ rootSessionId: "session-1",+ sessionId: "session-1",+ type: "session.started",+ });+ await runtime.hooks.publish({+ rootSessionId: "session-1",+ sequence: 0,+ sessionId: "session-1",+ turnId: "turn-1",+ type: "turn.started",+ });+ const bridge = createAiSdkHookBridge(scope, runtime.hooks, runtime.runInContext);+ Reflect.apply(bridge.onStart!, bridge, [+ {+ callId: "call-1",+ messages: manyMessages,+ modelId: "claude-test",+ operationId: "ai.streamText",+ provider: "anthropic",+ },+ ]);+ await Reflect.apply(bridge.onStepStart!, bridge, [{ callId: "call-1", stepNumber: 0 }]);+ await Reflect.apply(bridge.onLanguageModelCallStart!, bridge, [+ { callId: "call-1", messages: manyMessages, modelId: "claude-test", provider: "anthropic" },+ ]);+ await bridge.executeLanguageModelCall!({ callId: "call-1", execute: async () => undefined });+ await Reflect.apply(bridge.onLanguageModelCallEnd!, bridge, [+ {+ callId: "call-1",+ content: [],+ finishReason: "stop",+ performance: { responseTimeMs: 10 },+ responseId: "response-1",+ usage: { inputTokens: 10, outputTokens: 5 },+ },+ ]);+ await runtime.provider.forceFlush();++ const model = byName(runtime.exporter.getFinishedSpans(), "ai.streamText.doStream")[0]!;+ const raw = model.attributes["ai.prompt.messages"];+ expect(typeof raw).toBe("string");+ expect((raw as string).length).toBeLessThanOrEqual(32 * 1024);+ const parsed = JSON.parse(raw as string) as Array<Record<string, unknown>>;+ const marker = parsed[0]!["eve.truncated"] as { omittedMessages: number };+ expect(marker.omittedMessages).toBeGreaterThan(0);+ expect(marker.omittedMessages).toBeLessThan(200);+ expect(JSON.stringify(parsed)).toContain("message 199");+ expect(JSON.stringify(parsed)).not.toContain("message 0 ");+ });++ it("captures nothing when content capture is off", async () => {+ const exporter = new InMemorySpanExporter();+ const provider = new BasicTracerProvider({+ spanProcessors: [new SimpleSpanProcessor(exporter)],+ });+ const agentOtel = createAgentOtelInstrumentation({+ captureContent: false,+ frameworkVersion: "test",+ stateStore: new InMemoryAgentTraceStateStore(),+ tracer: provider.getTracer("eve.agent"),+ });+ const hooks = createInstrumentationHooks([agentOtel.hook]);+ await emitAttempt({+ hooks,+ runInContext: agentOtel.runInContext,+ sessionId: "session-1",+ turnId: "turn-1",+ turnSequence: 0,+ });+ await provider.forceFlush();++ const spans = exporter.getFinishedSpans();+ expect(JSON.stringify(spans.map((span) => span.attributes))).not.toContain("secret");+ expect(JSON.stringify(spans.map((span) => span.attributes))).not.toContain("temperature");expect(JSON.stringify(spans.map((span) => span.attributes))).not.toContain("private");+ expect(JSON.stringify(spans.map((span) => span.attributes))).not.toContain("real user text");+ expect(JSON.stringify(spans.map((span) => span.attributes))).not.toContain("system prompt");});it("writes gateway cost attributes on the step span when the gateway reports them", async () => {diff --git a/packages/eve/src/harness/agent-otel-provider.ts b/packages/eve/src/harness/agent-otel-provider.tsindex eb2058887..e758dac86 100644--- a/packages/eve/src/harness/agent-otel-provider.ts+++ b/packages/eve/src/harness/agent-otel-provider.ts@@ -9,6 +9,12 @@ import {trace,} from "#compiled/@opentelemetry/api/index.js";+import {+ contentAttribute,+ messagesContentAttribute,+ systemPromptAttribute,+ textContentAttribute,+} from "#harness/agent-otel-content.js";import type {InstrumentationAttemptMetadataEvent,InstrumentationAttemptScope,@@ -110,6 +116,12 @@ export class InMemoryAgentTraceStateStore implements AgentTraceStateStore {}export interface AgentOtelInstrumentationInput {+ /**+ * Capture model prompts/responses and tool call inputs/outputs as span+ * attributes. Content stays on the local machine — this provider only+ * wires the dev-time local spool — but can be turned off per project.+ */+ readonly captureContent?: boolean;readonly frameworkVersion: string;readonly stateStore: AgentTraceStateStore;readonly tracer: Tracer;@@ -125,6 +137,7 @@ export interface AgentOtelInstrumentation {export function createAgentOtelInstrumentation(input: AgentOtelInstrumentationInput,): AgentOtelInstrumentation {+ const captureContent = input.captureContent ?? true;const executionContexts = new WeakMap<InstrumentationAttemptScope,{ readonly models: Map<string, Context>; readonly tools: Map<string, Context> }@@ -302,6 +315,12 @@ export function createAgentOtelInstrumentation(},attempt.operation.context,);+ if (captureContent) {+ const messages = messagesContentAttribute(event.source.messages);+ if (messages !== undefined) span.setAttribute("ai.prompt.messages", messages);+ const system = systemPromptAttribute(event.source.instructions);+ if (system !== undefined) span.setAttribute("ai.prompt.system", system);+ }const state = { context: trace.setSpan(attempt.operation.context, span), span };getExecutionContexts(event.scope).models.set(event.id, state.context);return state;@@ -316,6 +335,31 @@ export function createAgentOtelInstrumentation(setUsage(state.span, event.source.usage);const attempt = steps.get(event.scope);if (attempt !== undefined) setUsage(attempt.step.span, event.source.usage);+ if (captureContent) {+ state.span.setAttribute("ai.response.finish_reason", event.source.finishReason);+ const reasoning = textContentAttribute(+ event.source.content+ .filter((part) => part.type === "reasoning")+ .map((part) => part.text)+ .filter((part) => part.trim().length > 0)+ .join("\n"),+ );+ if (reasoning !== undefined) state.span.setAttribute("ai.response.reasoning", reasoning);+ const text = textContentAttribute(+ event.source.content+ .filter((part) => part.type === "text")+ .map((part) => part.text)+ .join(""),+ );+ if (text !== undefined) state.span.setAttribute("ai.response.text", text);+ const toolCalls = event.source.content+ .filter((part) => part.type === "tool-call")+ .map((part) => ({ input: part.input, toolName: part.toolName }));+ if (toolCalls.length > 0) {+ const json = contentAttribute(toolCalls, false);+ if (json !== undefined) state.span.setAttribute("ai.response.tool_calls", json);+ }+ }}state.span.end();};@@ -355,6 +399,10 @@ export function createAgentOtelInstrumentation(},actionContext,);+ if (captureContent) {+ const args = contentAttribute(event.source.toolCall.input, false);+ if (args !== undefined) toolSpan.setAttribute("gen_ai.tool.call.arguments", args);+ }const state: ToolSpanState = {context: trace.setSpan(actionContext, toolSpan),span: actionSpan,@@ -373,6 +421,9 @@ export function createAgentOtelInstrumentation(} else if (event.source.toolOutput.type !== "tool-result") {recordError(state.toolSpan, event.source.toolOutput.error);recordError(state.span, event.source.toolOutput.error);+ } else if (captureContent) {+ const result = contentAttribute(event.source.toolOutput.output, false);+ if (result !== undefined) state.toolSpan.setAttribute("gen_ai.tool.call.result", result);}state.toolSpan.end();state.span.end();@@ -625,7 +676,5 @@ function recordError(span: Span, error: unknown): void {if (error instanceof Error) {span.recordException(error);span.setStatus({ code: SpanStatusCode.ERROR, message: error.message });- } else {- span.setStatus({ code: SpanStatusCode.ERROR });- }+ } else span.setStatus({ code: SpanStatusCode.ERROR });}diff --git a/packages/eve/src/harness/local-instrumentation-runtime.ts b/packages/eve/src/harness/local-instrumentation-runtime.tsindex 63d889e26..ce59ad1db 100644--- a/packages/eve/src/harness/local-instrumentation-runtime.ts+++ b/packages/eve/src/harness/local-instrumentation-runtime.ts@@ -49,6 +49,7 @@ export function installLocalInstrumentationRuntime(input: {throw new Error("eve could not register OpenTelemetry because another runtime already exists.");}const agentOtel = createAgentOtelInstrumentation({+ captureContent: process.env.EVE_TRACES_CONTENT !== "off",frameworkVersion: input.frameworkVersion,stateStore: new ContextAgentTraceStateStore(),tracer: trace.getTracer("eve.agent", input.frameworkVersion),