The Open Framework for Building Agents
feat(eve): show provider-executed tool calls in the trace conversation view (#1445)
Chad Hietala680db59940f6
diff --git a/.changeset/provider-tool-cards.md b/.changeset/provider-tool-cards.mdnew file mode 100644index 000000000..acdad9154--- /dev/null+++ b/.changeset/provider-tool-cards.md@@ -0,0 +1,5 @@+---+"eve": patch+---++Provider-executed tool calls (like a gateway's `web_search`) now show up in local traces: their calls and results are captured on the model span and the `/traces` viewer renders them as tool cards, with oversized outputs truncated to stay valid JSON.diff --git a/packages/eve/src/cli/dev/tui/traces/trace-conversation.test.ts b/packages/eve/src/cli/dev/tui/traces/trace-conversation.test.tsindex d53e9039c..9b9cbb246 100644--- a/packages/eve/src/cli/dev/tui/traces/trace-conversation.test.ts+++ b/packages/eve/src/cli/dev/tui/traces/trace-conversation.test.ts@@ -83,6 +83,41 @@ describe("buildConversationItems", () => {expect(items[2]?.result).toBe('{"temperatureF":72}');});+ it("renders provider-executed tool results as tool cards after the assistant", () => {+ // Provider-executed tools (e.g. web_search) never get an ai.toolCall+ // span; their outcomes live on the model span's tool_results attribute.+ const spans = weatherTurn().map((s) =>+ s.name === "ai.streamText.doStream"+ ? {+ ...s,+ attributes: {+ ...s.attributes,+ "ai.response.tool_results": JSON.stringify([+ {+ input: { query: "weather sf" },+ output: { results: ["sunny"] },+ toolName: "web_search",+ },+ { error: "quota exceeded", input: { query: "again" }, toolName: "web_search" },+ ]),+ },+ }+ : s,+ );+ const items = buildConversationItems(trace(spans));+ expect(items.map((item) => item.kind)).toEqual(["user", "assistant", "tool", "tool", "tool"]);+ const search = items[2]!;+ expect(search.name).toBe("web_search");+ expect(search.args).toBe('{"query":"weather sf"}');+ expect(search.result).toBe('{"results":["sunny"]}');+ expect(search.error).toBe(false);+ expect(search.durationMs).toBe(0);+ const failed = items[3]!;+ expect(failed.error).toBe(true);+ expect(failed.result).toBe("quota exceeded");+ expect(items[4]?.name).toBe("get_weather");+ });+it("finds turns parented to a session window span", () => {// Post-windowing capture: turns are children of the `agent.session` root,// so turn discovery must go by name, not root position.diff --git a/packages/eve/src/cli/dev/tui/traces/trace-conversation.ts b/packages/eve/src/cli/dev/tui/traces/trace-conversation.tsindex 13692eb3f..56d4fb031 100644--- a/packages/eve/src/cli/dev/tui/traces/trace-conversation.ts+++ b/packages/eve/src/cli/dev/tui/traces/trace-conversation.ts@@ -162,6 +162,27 @@ export function buildConversationItems(trace: LocalTrace): ConversationItem[] {},order: span.startTimeNs,});+ // Provider-executed tools (e.g. web_search) run inside the model+ // call and never get an ai.toolCall span; their results only exist+ // on the model span. Emit a card per result right after the+ // assistant card (same order, stable sort keeps emission order).+ for (const result of parseProviderToolResults(+ stringAttribute(span, "ai.response.tool_results"),+ )) {+ entries.push({+ item: {+ kind: "tool",+ args: result.args,+ durationMs: 0,+ error: result.error,+ name: stripTerminalControls(result.name),+ result: result.result,+ span,+ subagent,+ },+ order: span.startTimeNs,+ });+ }} else if (span.name === "ai.toolCall") {entries.push({item: {@@ -550,6 +571,37 @@ function parseToolCallNames(raw: string | undefined): readonly string[] | undefi}}+interface ProviderToolResult {+ readonly args: string | undefined;+ readonly error: boolean;+ readonly name: string;+ readonly result: string | undefined;+}++/** Parses the `ai.response.tool_results` JSON array of provider-executed tool outcomes. */+function parseProviderToolResults(raw: string | undefined): readonly ProviderToolResult[] {+ if (raw === undefined) return [];+ try {+ const parsed: unknown = JSON.parse(raw);+ if (!Array.isArray(parsed)) return [];+ // Capture-side truncation may collapse input/output to capped text, so+ // both structured values and pre-flattened strings appear here.+ const asText = (value: unknown): string | undefined =>+ typeof value === "string" ? value : value === undefined ? undefined : JSON.stringify(value);+ return parsed.filter(isRecord).map((entry) => {+ const failed = "error" in entry;+ return {+ args: asText(entry.input),+ error: failed,+ name: typeof entry.toolName === "string" ? entry.toolName : "tool",+ result: asText(failed ? entry.error : entry.output),+ };+ });+ } catch {+ return [];+ }+}+function stringAttribute(span: LocalTraceSpan, key: string): string | undefined {const value = span.attributes[key];return typeof value === "string" ? value : undefined;diff --git a/packages/eve/src/cli/dev/tui/traces/trace-view.ts b/packages/eve/src/cli/dev/tui/traces/trace-view.tsindex 5511b562f..eb2c98038 100644--- a/packages/eve/src/cli/dev/tui/traces/trace-view.ts+++ b/packages/eve/src/cli/dev/tui/traces/trace-view.ts@@ -136,8 +136,10 @@ function withViewerBase(row: string, width: number, theme: Theme): string {const CONVERSATION_CONTENT_KEYS: ReadonlySet<string> = new Set(["ai.prompt.messages","ai.prompt.system",+ "ai.response.reasoning","ai.response.text","ai.response.tool_calls",+ "ai.response.tool_results","gen_ai.tool.call.arguments","gen_ai.tool.call.result",]);diff --git a/packages/eve/src/cli/dev/tui/traces/trace-viewer-state.ts b/packages/eve/src/cli/dev/tui/traces/trace-viewer-state.tsindex 686d31353..31ec86972 100644--- a/packages/eve/src/cli/dev/tui/traces/trace-viewer-state.ts+++ b/packages/eve/src/cli/dev/tui/traces/trace-viewer-state.ts@@ -165,9 +165,13 @@ export function applyLoadedTrace(if (selectedSpanId === undefined) {selectedRow = state.selectedRow;} else {- selectedRow = conversationItems.findIndex(- (item) => item.span.spanId === selectedSpanId && item.kind === selectedItemKind,- );+ // Several cards can share one span (a model span also carries its+ // provider-executed tool cards), so match by occurrence, not first hit.+ const matches = (item: ConversationItem): boolean =>+ item.span.spanId === selectedSpanId && item.kind === selectedItemKind;+ const occurrence = state.conversationItems.slice(0, state.selectedRow).filter(matches).length;+ const indices = conversationItems.flatMap((item, index) => (matches(item) ? [index] : []));+ selectedRow = indices[Math.min(occurrence, indices.length - 1)] ?? -1;}if (selectedRow === -1) {selectedRow = Math.min(state.selectedRow, conversationItems.length - 1);diff --git a/packages/eve/src/harness/agent-otel-content.test.ts b/packages/eve/src/harness/agent-otel-content.test.tsnew file mode 100644index 000000000..b49595c72--- /dev/null+++ b/packages/eve/src/harness/agent-otel-content.test.ts@@ -0,0 +1,65 @@+import { describe, expect, it } from "vitest";++import {+ CONTENT_ATTRIBUTE_LIMIT,+ toolResultsContentAttribute,+} from "#harness/agent-otel-content.js";++describe("toolResultsContentAttribute", () => {+ it("returns the full payload when it fits", () => {+ const json = toolResultsContentAttribute([+ { input: { query: "weather" }, output: { results: ["sunny"] }, toolName: "web_search" },+ ]);+ expect(json).toBe(+ '[{"input":{"query":"weather"},"output":{"results":["sunny"]},"toolName":"web_search"}]',+ );+ });++ it("returns undefined for no results", () => {+ expect(toolResultsContentAttribute([])).toBeUndefined();+ });++ it("keeps oversized payloads valid JSON by capping entry text", () => {+ const json = toolResultsContentAttribute([+ {+ input: { query: "weather" },+ output: { excerpts: "x".repeat(CONTENT_ATTRIBUTE_LIMIT * 2) },+ toolName: "web_search",+ },+ ]);+ expect(json).toBeDefined();+ expect(json!.length).toBeLessThanOrEqual(CONTENT_ATTRIBUTE_LIMIT);+ const parsed = JSON.parse(json!) as Array<Record<string, unknown>>;+ expect(parsed[0]!.toolName).toBe("web_search");+ expect(parsed[0]!.input).toBe('{"query":"weather"}');+ expect(parsed[0]!.output).toContain("… [truncated]");+ });++ it("preserves the error key through truncation", () => {+ const json = toolResultsContentAttribute([+ {+ error: `quota exceeded ${"y".repeat(CONTENT_ATTRIBUTE_LIMIT * 2)}`,+ input: { query: "again" },+ toolName: "web_search",+ },+ ]);+ const parsed = JSON.parse(json!) as Array<Record<string, unknown>>;+ expect(parsed[0]!.error).toContain("quota exceeded");+ expect(parsed[0]!.error).toContain("… [truncated]");+ expect(parsed[0]!.output).toBeUndefined();+ });++ it("splits the budget across many oversized entries", () => {+ const entries = Array.from({ length: 8 }, (_, index) => ({+ input: { index },+ output: "z".repeat(CONTENT_ATTRIBUTE_LIMIT),+ toolName: `tool_${index}`,+ }));+ const json = toolResultsContentAttribute(entries);+ expect(json).toBeDefined();+ expect(json!.length).toBeLessThanOrEqual(CONTENT_ATTRIBUTE_LIMIT);+ const parsed = JSON.parse(json!) as Array<Record<string, unknown>>;+ expect(parsed).toHaveLength(8);+ expect(parsed[7]!.toolName).toBe("tool_7");+ });+});diff --git a/packages/eve/src/harness/agent-otel-content.ts b/packages/eve/src/harness/agent-otel-content.tsindex 17cc53458..12559c47e 100644--- a/packages/eve/src/harness/agent-otel-content.ts+++ b/packages/eve/src/harness/agent-otel-content.ts@@ -53,6 +53,36 @@ export function messagesContentAttribute(messages: unknown): string | undefinedreturn truncateSingleMessage(stripped);}+/**+ * Serializes provider-executed tool results, keeping the attribute valid+ * JSON under the cap: when the full payload is too big, each entry's input+ * and output collapse to capped text at progressively smaller budgets+ * instead of cutting the JSON mid-string.+ */+export function toolResultsContentAttribute(+ results: readonly Record<string, unknown>[],+): string | undefined {+ if (results.length === 0) return undefined;+ const full = stringifyContent(results);+ if (full !== undefined && full.length <= CONTENT_ATTRIBUTE_LIMIT) return full;+ for (let cap = CONTENT_ATTRIBUTE_LIMIT; cap >= 0; cap = cap >= 256 ? Math.floor(cap / 2) : -1) {+ const json = stringifyContent(results.map((entry) => cappedToolResult(entry, cap)));+ if (json !== undefined && json.length <= CONTENT_ATTRIBUTE_LIMIT) return json;+ }+ return undefined;+}++function cappedToolResult(entry: Record<string, unknown>, cap: number): Record<string, unknown> {+ const out: Record<string, unknown> = { toolName: entry.toolName };+ for (const key of ["input", "output", "error"]) {+ if (!(key in entry)) continue;+ const value = entry[key];+ const text = typeof value === "string" ? value : (stringifyContent(value) ?? "");+ out[key] = text.length <= cap ? text : `${text.slice(0, cap)}… [truncated]`;+ }+ return out;+}+/** Normalizes the AI SDK's `instructions` prompt to plain text for `ai.prompt.system`. */export function systemPromptAttribute(instructions: unknown): string | undefined {if (typeof instructions === "string") return textContentAttribute(instructions);diff --git a/packages/eve/src/harness/agent-otel-provider.test.ts b/packages/eve/src/harness/agent-otel-provider.test.tsindex be93e5b95..9cd394feb 100644--- a/packages/eve/src/harness/agent-otel-provider.test.ts+++ b/packages/eve/src/harness/agent-otel-provider.test.ts@@ -100,6 +100,21 @@ async function emitAttempt(input: {content: [{ type: "reasoning", text: "thinking about weather" },{ type: "text", text: "Checking the weather." },+ {+ input: { query: "weather today" },+ providerExecuted: true,+ toolCallId: "search-1",+ toolName: "web_search",+ type: "tool-call",+ },+ {+ input: { query: "weather today" },+ output: { results: ["sunny"] },+ providerExecuted: true,+ toolCallId: "search-1",+ toolName: "web_search",+ type: "tool-result",+ },],finishReason: "tool-calls",performance: { responseTimeMs: 10 },@@ -272,6 +287,14 @@ describe("createAgentOtelInstrumentation", () => {expect(model.attributes["ai.response.finish_reason"]).toBe("tool-calls");expect(model.attributes["ai.response.reasoning"]).toBe("thinking about weather");expect(model.attributes["ai.response.text"]).toBe("Checking the weather.");+ // Provider-executed tools never reach the tool loop; their calls and+ // results are captured off the model response content.+ expect(model.attributes["ai.response.tool_calls"]).toBe(+ '[{"input":{"query":"weather today"},"toolName":"web_search"}]',+ );+ expect(model.attributes["ai.response.tool_results"]).toBe(+ '[{"input":{"query":"weather today"},"output":{"results":["sunny"]},"toolName":"web_search"}]',+ );expect(tool.attributes["gen_ai.tool.call.arguments"]).toBe('{"secret":"value"}');expect(tool.attributes["gen_ai.tool.call.result"]).toBe('{"temperature":72}');// Structural spans stay structural: content lives only on the operation spans.diff --git a/packages/eve/src/harness/agent-otel-provider.ts b/packages/eve/src/harness/agent-otel-provider.tsindex e758dac86..06d9b32dc 100644--- a/packages/eve/src/harness/agent-otel-provider.ts+++ b/packages/eve/src/harness/agent-otel-provider.ts@@ -14,6 +14,7 @@ import {messagesContentAttribute,systemPromptAttribute,textContentAttribute,+ toolResultsContentAttribute,} from "#harness/agent-otel-content.js";import type {InstrumentationAttemptMetadataEvent,@@ -359,6 +360,20 @@ export function createAgentOtelInstrumentation(const json = contentAttribute(toolCalls, false);if (json !== undefined) state.span.setAttribute("ai.response.tool_calls", json);}+ // Provider-executed tools (e.g. web_search) run inside the model call,+ // never reach eve's tool loop, and so never get an ai.toolCall span.+ // Their results only exist as content parts on the model response.+ const toolResults = event.source.content+ .filter((part) => part.type === "tool-result" || part.type === "tool-error")+ .map((part) =>+ part.type === "tool-result"+ ? { input: part.input, output: part.output, toolName: part.toolName }+ : { error: errorText(part.error), input: part.input, toolName: part.toolName },+ );+ if (toolResults.length > 0) {+ const json = toolResultsContentAttribute(toolResults);+ if (json !== undefined) state.span.setAttribute("ai.response.tool_results", json);+ }}}state.span.end();@@ -672,6 +687,10 @@ function setUsage(}}+function errorText(error: unknown): unknown {+ return error instanceof Error ? error.message : error;+}+function recordError(span: Span, error: unknown): void {if (error instanceof Error) {span.recordException(error);