Skip to content

armgit / eve

Public Imported from GitHub
0

The Open Framework for Building Agents

feat(eve): show provider-executed tool calls in the trace conversation view (#1445)

Chad Hietala680db59940f6
diff --git a/.changeset/provider-tool-cards.md b/.changeset/provider-tool-cards.md
new file mode 100644
index 000000000..acdad9154
--- /dev/null
+++ b/.changeset/provider-tool-cards.md
@@ -0,0 +1,5 @@
+---
+"eve": patch
+---
+
+Provider-executed tool calls (like a gateway's `web_search`) now show up in local traces: their calls and results are captured on the model span and the `/traces` viewer renders them as tool cards, with oversized outputs truncated to stay valid JSON.
diff --git a/packages/eve/src/cli/dev/tui/traces/trace-conversation.test.ts b/packages/eve/src/cli/dev/tui/traces/trace-conversation.test.ts
index d53e9039c..9b9cbb246 100644
--- a/packages/eve/src/cli/dev/tui/traces/trace-conversation.test.ts
+++ b/packages/eve/src/cli/dev/tui/traces/trace-conversation.test.ts
@@ -83,6 +83,41 @@ describe("buildConversationItems", () => {
expect(items[2]?.result).toBe('{"temperatureF":72}');
});
+ it("renders provider-executed tool results as tool cards after the assistant", () => {
+ // Provider-executed tools (e.g. web_search) never get an ai.toolCall
+ // span; their outcomes live on the model span's tool_results attribute.
+ const spans = weatherTurn().map((s) =>
+ s.name === "ai.streamText.doStream"
+ ? {
+ ...s,
+ attributes: {
+ ...s.attributes,
+ "ai.response.tool_results": JSON.stringify([
+ {
+ input: { query: "weather sf" },
+ output: { results: ["sunny"] },
+ toolName: "web_search",
+ },
+ { error: "quota exceeded", input: { query: "again" }, toolName: "web_search" },
+ ]),
+ },
+ }
+ : s,
+ );
+ const items = buildConversationItems(trace(spans));
+ expect(items.map((item) => item.kind)).toEqual(["user", "assistant", "tool", "tool", "tool"]);
+ const search = items[2]!;
+ expect(search.name).toBe("web_search");
+ expect(search.args).toBe('{"query":"weather sf"}');
+ expect(search.result).toBe('{"results":["sunny"]}');
+ expect(search.error).toBe(false);
+ expect(search.durationMs).toBe(0);
+ const failed = items[3]!;
+ expect(failed.error).toBe(true);
+ expect(failed.result).toBe("quota exceeded");
+ expect(items[4]?.name).toBe("get_weather");
+ });
+
it("finds turns parented to a session window span", () => {
// Post-windowing capture: turns are children of the `agent.session` root,
// so turn discovery must go by name, not root position.
diff --git a/packages/eve/src/cli/dev/tui/traces/trace-conversation.ts b/packages/eve/src/cli/dev/tui/traces/trace-conversation.ts
index 13692eb3f..56d4fb031 100644
--- a/packages/eve/src/cli/dev/tui/traces/trace-conversation.ts
+++ b/packages/eve/src/cli/dev/tui/traces/trace-conversation.ts
@@ -162,6 +162,27 @@ export function buildConversationItems(trace: LocalTrace): ConversationItem[] {
},
order: span.startTimeNs,
});
+ // Provider-executed tools (e.g. web_search) run inside the model
+ // call and never get an ai.toolCall span; their results only exist
+ // on the model span. Emit a card per result right after the
+ // assistant card (same order, stable sort keeps emission order).
+ for (const result of parseProviderToolResults(
+ stringAttribute(span, "ai.response.tool_results"),
+ )) {
+ entries.push({
+ item: {
+ kind: "tool",
+ args: result.args,
+ durationMs: 0,
+ error: result.error,
+ name: stripTerminalControls(result.name),
+ result: result.result,
+ span,
+ subagent,
+ },
+ order: span.startTimeNs,
+ });
+ }
} else if (span.name === "ai.toolCall") {
entries.push({
item: {
@@ -550,6 +571,37 @@ function parseToolCallNames(raw: string | undefined): readonly string[] | undefi
}
}
+interface ProviderToolResult {
+ readonly args: string | undefined;
+ readonly error: boolean;
+ readonly name: string;
+ readonly result: string | undefined;
+}
+
+/** Parses the `ai.response.tool_results` JSON array of provider-executed tool outcomes. */
+function parseProviderToolResults(raw: string | undefined): readonly ProviderToolResult[] {
+ if (raw === undefined) return [];
+ try {
+ const parsed: unknown = JSON.parse(raw);
+ if (!Array.isArray(parsed)) return [];
+ // Capture-side truncation may collapse input/output to capped text, so
+ // both structured values and pre-flattened strings appear here.
+ const asText = (value: unknown): string | undefined =>
+ typeof value === "string" ? value : value === undefined ? undefined : JSON.stringify(value);
+ return parsed.filter(isRecord).map((entry) => {
+ const failed = "error" in entry;
+ return {
+ args: asText(entry.input),
+ error: failed,
+ name: typeof entry.toolName === "string" ? entry.toolName : "tool",
+ result: asText(failed ? entry.error : entry.output),
+ };
+ });
+ } catch {
+ return [];
+ }
+}
+
function stringAttribute(span: LocalTraceSpan, key: string): string | undefined {
const value = span.attributes[key];
return typeof value === "string" ? value : undefined;
diff --git a/packages/eve/src/cli/dev/tui/traces/trace-view.ts b/packages/eve/src/cli/dev/tui/traces/trace-view.ts
index 5511b562f..eb2c98038 100644
--- a/packages/eve/src/cli/dev/tui/traces/trace-view.ts
+++ b/packages/eve/src/cli/dev/tui/traces/trace-view.ts
@@ -136,8 +136,10 @@ function withViewerBase(row: string, width: number, theme: Theme): string {
const CONVERSATION_CONTENT_KEYS: ReadonlySet<string> = new Set([
"ai.prompt.messages",
"ai.prompt.system",
+ "ai.response.reasoning",
"ai.response.text",
"ai.response.tool_calls",
+ "ai.response.tool_results",
"gen_ai.tool.call.arguments",
"gen_ai.tool.call.result",
]);
diff --git a/packages/eve/src/cli/dev/tui/traces/trace-viewer-state.ts b/packages/eve/src/cli/dev/tui/traces/trace-viewer-state.ts
index 686d31353..31ec86972 100644
--- a/packages/eve/src/cli/dev/tui/traces/trace-viewer-state.ts
+++ b/packages/eve/src/cli/dev/tui/traces/trace-viewer-state.ts
@@ -165,9 +165,13 @@ export function applyLoadedTrace(
if (selectedSpanId === undefined) {
selectedRow = state.selectedRow;
} else {
- selectedRow = conversationItems.findIndex(
- (item) => item.span.spanId === selectedSpanId && item.kind === selectedItemKind,
- );
+ // Several cards can share one span (a model span also carries its
+ // provider-executed tool cards), so match by occurrence, not first hit.
+ const matches = (item: ConversationItem): boolean =>
+ item.span.spanId === selectedSpanId && item.kind === selectedItemKind;
+ const occurrence = state.conversationItems.slice(0, state.selectedRow).filter(matches).length;
+ const indices = conversationItems.flatMap((item, index) => (matches(item) ? [index] : []));
+ selectedRow = indices[Math.min(occurrence, indices.length - 1)] ?? -1;
}
if (selectedRow === -1) {
selectedRow = Math.min(state.selectedRow, conversationItems.length - 1);
diff --git a/packages/eve/src/harness/agent-otel-content.test.ts b/packages/eve/src/harness/agent-otel-content.test.ts
new file mode 100644
index 000000000..b49595c72
--- /dev/null
+++ b/packages/eve/src/harness/agent-otel-content.test.ts
@@ -0,0 +1,65 @@
+import { describe, expect, it } from "vitest";
+
+import {
+ CONTENT_ATTRIBUTE_LIMIT,
+ toolResultsContentAttribute,
+} from "#harness/agent-otel-content.js";
+
+describe("toolResultsContentAttribute", () => {
+ it("returns the full payload when it fits", () => {
+ const json = toolResultsContentAttribute([
+ { input: { query: "weather" }, output: { results: ["sunny"] }, toolName: "web_search" },
+ ]);
+ expect(json).toBe(
+ '[{"input":{"query":"weather"},"output":{"results":["sunny"]},"toolName":"web_search"}]',
+ );
+ });
+
+ it("returns undefined for no results", () => {
+ expect(toolResultsContentAttribute([])).toBeUndefined();
+ });
+
+ it("keeps oversized payloads valid JSON by capping entry text", () => {
+ const json = toolResultsContentAttribute([
+ {
+ input: { query: "weather" },
+ output: { excerpts: "x".repeat(CONTENT_ATTRIBUTE_LIMIT * 2) },
+ toolName: "web_search",
+ },
+ ]);
+ expect(json).toBeDefined();
+ expect(json!.length).toBeLessThanOrEqual(CONTENT_ATTRIBUTE_LIMIT);
+ const parsed = JSON.parse(json!) as Array<Record<string, unknown>>;
+ expect(parsed[0]!.toolName).toBe("web_search");
+ expect(parsed[0]!.input).toBe('{"query":"weather"}');
+ expect(parsed[0]!.output).toContain("… [truncated]");
+ });
+
+ it("preserves the error key through truncation", () => {
+ const json = toolResultsContentAttribute([
+ {
+ error: `quota exceeded ${"y".repeat(CONTENT_ATTRIBUTE_LIMIT * 2)}`,
+ input: { query: "again" },
+ toolName: "web_search",
+ },
+ ]);
+ const parsed = JSON.parse(json!) as Array<Record<string, unknown>>;
+ expect(parsed[0]!.error).toContain("quota exceeded");
+ expect(parsed[0]!.error).toContain("… [truncated]");
+ expect(parsed[0]!.output).toBeUndefined();
+ });
+
+ it("splits the budget across many oversized entries", () => {
+ const entries = Array.from({ length: 8 }, (_, index) => ({
+ input: { index },
+ output: "z".repeat(CONTENT_ATTRIBUTE_LIMIT),
+ toolName: `tool_${index}`,
+ }));
+ const json = toolResultsContentAttribute(entries);
+ expect(json).toBeDefined();
+ expect(json!.length).toBeLessThanOrEqual(CONTENT_ATTRIBUTE_LIMIT);
+ const parsed = JSON.parse(json!) as Array<Record<string, unknown>>;
+ expect(parsed).toHaveLength(8);
+ expect(parsed[7]!.toolName).toBe("tool_7");
+ });
+});
diff --git a/packages/eve/src/harness/agent-otel-content.ts b/packages/eve/src/harness/agent-otel-content.ts
index 17cc53458..12559c47e 100644
--- a/packages/eve/src/harness/agent-otel-content.ts
+++ b/packages/eve/src/harness/agent-otel-content.ts
@@ -53,6 +53,36 @@ export function messagesContentAttribute(messages: unknown): string | undefined
return truncateSingleMessage(stripped);
}
+/**
+ * Serializes provider-executed tool results, keeping the attribute valid
+ * JSON under the cap: when the full payload is too big, each entry's input
+ * and output collapse to capped text at progressively smaller budgets
+ * instead of cutting the JSON mid-string.
+ */
+export function toolResultsContentAttribute(
+ results: readonly Record<string, unknown>[],
+): string | undefined {
+ if (results.length === 0) return undefined;
+ const full = stringifyContent(results);
+ if (full !== undefined && full.length <= CONTENT_ATTRIBUTE_LIMIT) return full;
+ for (let cap = CONTENT_ATTRIBUTE_LIMIT; cap >= 0; cap = cap >= 256 ? Math.floor(cap / 2) : -1) {
+ const json = stringifyContent(results.map((entry) => cappedToolResult(entry, cap)));
+ if (json !== undefined && json.length <= CONTENT_ATTRIBUTE_LIMIT) return json;
+ }
+ return undefined;
+}
+
+function cappedToolResult(entry: Record<string, unknown>, cap: number): Record<string, unknown> {
+ const out: Record<string, unknown> = { toolName: entry.toolName };
+ for (const key of ["input", "output", "error"]) {
+ if (!(key in entry)) continue;
+ const value = entry[key];
+ const text = typeof value === "string" ? value : (stringifyContent(value) ?? "");
+ out[key] = text.length <= cap ? text : `${text.slice(0, cap)}… [truncated]`;
+ }
+ return out;
+}
+
/** Normalizes the AI SDK's `instructions` prompt to plain text for `ai.prompt.system`. */
export function systemPromptAttribute(instructions: unknown): string | undefined {
if (typeof instructions === "string") return textContentAttribute(instructions);
diff --git a/packages/eve/src/harness/agent-otel-provider.test.ts b/packages/eve/src/harness/agent-otel-provider.test.ts
index be93e5b95..9cd394feb 100644
--- a/packages/eve/src/harness/agent-otel-provider.test.ts
+++ b/packages/eve/src/harness/agent-otel-provider.test.ts
@@ -100,6 +100,21 @@ async function emitAttempt(input: {
content: [
{ type: "reasoning", text: "thinking about weather" },
{ type: "text", text: "Checking the weather." },
+ {
+ input: { query: "weather today" },
+ providerExecuted: true,
+ toolCallId: "search-1",
+ toolName: "web_search",
+ type: "tool-call",
+ },
+ {
+ input: { query: "weather today" },
+ output: { results: ["sunny"] },
+ providerExecuted: true,
+ toolCallId: "search-1",
+ toolName: "web_search",
+ type: "tool-result",
+ },
],
finishReason: "tool-calls",
performance: { responseTimeMs: 10 },
@@ -272,6 +287,14 @@ describe("createAgentOtelInstrumentation", () => {
expect(model.attributes["ai.response.finish_reason"]).toBe("tool-calls");
expect(model.attributes["ai.response.reasoning"]).toBe("thinking about weather");
expect(model.attributes["ai.response.text"]).toBe("Checking the weather.");
+ // Provider-executed tools never reach the tool loop; their calls and
+ // results are captured off the model response content.
+ expect(model.attributes["ai.response.tool_calls"]).toBe(
+ '[{"input":{"query":"weather today"},"toolName":"web_search"}]',
+ );
+ expect(model.attributes["ai.response.tool_results"]).toBe(
+ '[{"input":{"query":"weather today"},"output":{"results":["sunny"]},"toolName":"web_search"}]',
+ );
expect(tool.attributes["gen_ai.tool.call.arguments"]).toBe('{"secret":"value"}');
expect(tool.attributes["gen_ai.tool.call.result"]).toBe('{"temperature":72}');
// Structural spans stay structural: content lives only on the operation spans.
diff --git a/packages/eve/src/harness/agent-otel-provider.ts b/packages/eve/src/harness/agent-otel-provider.ts
index e758dac86..06d9b32dc 100644
--- a/packages/eve/src/harness/agent-otel-provider.ts
+++ b/packages/eve/src/harness/agent-otel-provider.ts
@@ -14,6 +14,7 @@ import {
messagesContentAttribute,
systemPromptAttribute,
textContentAttribute,
+ toolResultsContentAttribute,
} from "#harness/agent-otel-content.js";
import type {
InstrumentationAttemptMetadataEvent,
@@ -359,6 +360,20 @@ export function createAgentOtelInstrumentation(
const json = contentAttribute(toolCalls, false);
if (json !== undefined) state.span.setAttribute("ai.response.tool_calls", json);
}
+ // Provider-executed tools (e.g. web_search) run inside the model call,
+ // never reach eve's tool loop, and so never get an ai.toolCall span.
+ // Their results only exist as content parts on the model response.
+ const toolResults = event.source.content
+ .filter((part) => part.type === "tool-result" || part.type === "tool-error")
+ .map((part) =>
+ part.type === "tool-result"
+ ? { input: part.input, output: part.output, toolName: part.toolName }
+ : { error: errorText(part.error), input: part.input, toolName: part.toolName },
+ );
+ if (toolResults.length > 0) {
+ const json = toolResultsContentAttribute(toolResults);
+ if (json !== undefined) state.span.setAttribute("ai.response.tool_results", json);
+ }
}
}
state.span.end();
@@ -672,6 +687,10 @@ function setUsage(
}
}
+function errorText(error: unknown): unknown {
+ return error instanceof Error ? error.message : error;
+}
+
function recordError(span: Span, error: unknown): void {
if (error instanceof Error) {
span.recordException(error);