Skip to content

armgit / eve

Public Imported from GitHub
0

The Open Framework for Building Agents

feat(selfmod): add bounded local trace analysis (#3370)

prhad8ddf0d7d8f2
diff --git a/.changeset/quick-traces-list.md b/.changeset/quick-traces-list.md
new file mode 100644
index 000000000..794db0374
--- /dev/null
+++ b/.changeset/quick-traces-list.md
@@ -0,0 +1,5 @@
+---
+"eve": patch
+---
+
+Add bounded, conversation-scoped trace analysis to self-modification, with filtered search, compact timelines, and batched span payload inspection.
diff --git a/docs/reference/cli.md b/docs/reference/cli.md
index f053ae1c8..0aaf62f9e 100644
--- a/docs/reference/cli.md
+++ b/docs/reference/cli.md
@@ -311,7 +311,7 @@ eve traces --json # dump the full trace as JSON
Reads the immutable OTLP/JSON segments under `.eve/traces/v1`, so `eve dev` need not be running. Accepts a full trace id, a `gen_ai.conversation.id`, or an unambiguous prefix of either. Malformed segments are skipped without hiding valid spans from the same trace.
-Span rows carry inline metrics when the span recorded them — `↑input`/`↓output` token counts, gateway cost, and the tool name for `execute_tool` spans — and the header aggregates models, token totals, cost, and error count across the trace's step spans. `--verbose` expands each span under its tree row: status (with the error message on failures), timing, ids, every attribute (prompts, responses, and tool payloads as transcripts or pretty-printed JSON), and every span event with its offset from span start. `--json` prints the same records as JSON, one object per selected trace.
+Span rows carry inline metrics when the span recorded them — `↑input`/`↓output` token counts, gateway cost, and the tool name for `execute_tool` spans. The header lists models across the trace, sums token usage and cost from step spans, and counts all error-bearing spans. `--verbose` expands each span under its tree row: status (with the error message on failures), timing, ids, every attribute (prompts, responses, and tool payloads as transcripts or pretty-printed JSON), and every span event with its offset from span start. `--json` prints the same records as JSON, one object per selected trace.
Every subagent activation starts its own trace. The first child's `invoke_agent` root links to the dispatching caller with `eve.link.type=agent.dispatch`; remote agents preserve that caller in W3C `tracestate` even when HTTP `traceparent` advances through platform ingress. Later turns also start fresh traces without repeating the initial caller link. All related sessions retain the same `gen_ai.conversation.id`, and `agent.subagent.name` labels the child invocation.
diff --git a/packages/eve/src/cli/commands/trace-detail.test.ts b/packages/eve/src/cli/commands/trace-detail.test.ts
index 37b0ff69e..25f5807a3 100644
--- a/packages/eve/src/cli/commands/trace-detail.test.ts
+++ b/packages/eve/src/cli/commands/trace-detail.test.ts
@@ -7,7 +7,6 @@ import {
formatTokenSummary,
renderSpanDetailTree,
spanMetricChips,
- summarizeLocalTrace,
} from "./trace-detail.js";
function span(overrides: Partial<LocalTraceSpan> = {}): LocalTraceSpan {
@@ -51,74 +50,13 @@ describe("spanMetricChips", () => {
});
});
-describe("summarizeLocalTrace", () => {
- it("sums usage over agent.step spans only, avoiding double counts", () => {
- const summary = summarizeLocalTrace([
- span({
- attributes: {
- "agent.model.id": "gpt-5",
- "agent.usage.input_tokens": 1000,
- "agent.usage.output_tokens": 100,
- "agent.usage.cache_read_tokens": 800,
- "gen_ai.usage.cost": 0.01,
- },
- }),
- // Same usage repeated on the model span must not double-count.
- span({
- name: "ai.streamText.doStream",
- attributes: {
- "agent.usage.input_tokens": 1000,
- "agent.usage.output_tokens": 100,
- "gen_ai.request.model": "gpt-5",
- },
- }),
- span({
- attributes: {
- "agent.model.id": "claude-sonnet-4",
- "agent.usage.input_tokens": 500,
- "agent.usage.output_tokens": 50,
- },
- }),
- ]);
-
- expect(summary.inputTokens).toBe(1500);
- expect(summary.outputTokens).toBe(150);
- expect(summary.cacheReadTokens).toBe(800);
- expect(summary.costUsd).toBeCloseTo(0.01);
- expect(summary.models).toEqual(["gpt-5", "claude-sonnet-4"]);
- expect(summary.errorCount).toBe(0);
- });
-
- it("reads legacy GenAI cache details from persisted step spans", () => {
- const summary = summarizeLocalTrace([
- span({
- attributes: {
- "gen_ai.usage.cache_creation.input_tokens": 20,
- "gen_ai.usage.cache_read.input_tokens": 40,
- },
- }),
- ]);
-
- expect(summary.cacheReadTokens).toBe(40);
- expect(summary.cacheWriteTokens).toBe(20);
- });
-
- it("reports errors and leaves cost undefined when unreported", () => {
- const summary = summarizeLocalTrace([span({ statusCode: 2 }), span()]);
- expect(summary.errorCount).toBe(1);
- expect(summary.costUsd).toBeUndefined();
- });
-});
-
describe("formatTokenSummary / formatCostUsd", () => {
it("formats the header tokens row with cache parts when present", () => {
expect(
formatTokenSummary({
cacheReadTokens: 1100,
cacheWriteTokens: 0,
- errorCount: 0,
inputTokens: 1200,
- models: [],
outputTokens: 340,
}),
).toBe("↑1.2K in · ↓340 out · 1.1K cached");
diff --git a/packages/eve/src/cli/commands/trace-detail.ts b/packages/eve/src/cli/commands/trace-detail.ts
index 6df900612..32d78eb93 100644
--- a/packages/eve/src/cli/commands/trace-detail.ts
+++ b/packages/eve/src/cli/commands/trace-detail.ts
@@ -1,9 +1,7 @@
/**
- * Span-level detail rendering and trace-level usage aggregation for
- * `eve traces`. Kept separate from the command module: the tree renderer
- * stays compact while these helpers answer "what does this span carry" —
- * inline metric chips for tree rows, the `--verbose` per-span block, and
- * the usage/cost totals summarized in the trace header.
+ * Span-level detail rendering for `eve traces`. Kept separate from the command
+ * module: the tree renderer stays compact while these helpers answer "what
+ * does this span carry" — inline metric chips and the `--verbose` per-span block.
*/
import { formatCompactTokenCount } from "#cli/dev/tui/stream-format.js";
@@ -11,20 +9,7 @@ import { formatAttributeContent } from "#cli/dev/tui/traces/trace-content.js";
import { formatElapsed } from "#cli/format-elapsed.js";
import { sanitizeForTerminal } from "#cli/ui/output.js";
import type { LocalTraceSpan } from "#tracing/local-trace-reader.js";
-import { AGENT_USAGE_ATTRIBUTES } from "#tracing/agent-span-contract.js";
-
-/** Usage and cost totals aggregated over a trace's `agent.step` spans. */
-export interface LocalTraceSummary {
- readonly cacheReadTokens: number;
- readonly cacheWriteTokens: number;
- /** Total gateway cost in USD; undefined when no span reported cost. */
- readonly costUsd?: number;
- readonly errorCount: number;
- readonly inputTokens: number;
- /** Distinct model ids seen on any span, first-seen order. */
- readonly models: readonly string[];
- readonly outputTokens: number;
-}
+import { localTraceSpanCostUsd, type LocalTraceSummary } from "#tracing/local-trace-summary.js";
/**
* Compact metrics for one tree row: token chips (`↑1.4K`/`↓213`), cost
@@ -39,58 +24,18 @@ export function spanMetricChips(span: LocalTraceSpan): string[] {
const output = numberAttribute(span, "agent.usage.output_tokens");
if (input !== undefined) chips.push(`↑${formatCompactTokenCount(input)}`);
if (output !== undefined) chips.push(`↓${formatCompactTokenCount(output)}`);
- const cost = spanCostUsd(span);
+ const cost = localTraceSpanCostUsd(span);
if (cost !== undefined) chips.push(formatCostUsd(cost));
return chips;
}
-/**
- * Aggregates usage, cost, models, and errors across one trace. Only
- * `agent.step` spans contribute usage: model spans carry the same counters
- * and a subagent's totals already appear as its own step spans in the same
- * trace, so summing anything else would double-count.
- */
-export function summarizeLocalTrace(spans: readonly LocalTraceSpan[]): LocalTraceSummary {
- const models: string[] = [];
- let cacheReadTokens = 0;
- let cacheWriteTokens = 0;
- let costUsd: number | undefined;
- let errorCount = 0;
- let inputTokens = 0;
- let outputTokens = 0;
- for (const span of spans) {
- const model =
- stringAttribute(span, "agent.model.id") ?? stringAttribute(span, "gen_ai.request.model");
- if (model !== undefined && !models.includes(model)) models.push(model);
- if (span.statusCode === 2) errorCount += 1;
- if (span.name !== "agent.step") continue;
- inputTokens += numberAttribute(span, "agent.usage.input_tokens") ?? 0;
- outputTokens += numberAttribute(span, "agent.usage.output_tokens") ?? 0;
- cacheReadTokens +=
- numberAttribute(span, AGENT_USAGE_ATTRIBUTES.cacheReadTokens) ??
- numberAttribute(span, "gen_ai.usage.cache_read.input_tokens") ??
- 0;
- cacheWriteTokens +=
- numberAttribute(span, AGENT_USAGE_ATTRIBUTES.cacheWriteTokens) ??
- numberAttribute(span, "gen_ai.usage.cache_write.input_tokens") ??
- numberAttribute(span, "gen_ai.usage.cache_creation.input_tokens") ??
- 0;
- const cost = spanCostUsd(span);
- if (cost !== undefined) costUsd = (costUsd ?? 0) + cost;
- }
- return {
- cacheReadTokens,
- cacheWriteTokens,
- costUsd,
- errorCount,
- inputTokens,
- models,
- outputTokens,
- };
-}
-
/** One-line `Tokens` header value: `↑1.2K in · ↓340 out · 1.1K cached`. */
-export function formatTokenSummary(summary: LocalTraceSummary): string {
+export function formatTokenSummary(
+ summary: Pick<
+ LocalTraceSummary,
+ "inputTokens" | "outputTokens" | "cacheReadTokens" | "cacheWriteTokens"
+ >,
+): string {
const parts = [
`↑${formatCompactTokenCount(summary.inputTokens)} in`,
`↓${formatCompactTokenCount(summary.outputTokens)} out`,
@@ -217,12 +162,6 @@ function spanDetailEntries(span: LocalTraceSpan, width: number): SpanDetailEntry
return entries;
}
-function spanCostUsd(span: LocalTraceSpan): number | undefined {
- return (
- numberAttribute(span, "gen_ai.usage.gateway_cost") ?? numberAttribute(span, "gen_ai.usage.cost")
- );
-}
-
function spanKind(kind: number): string {
switch (kind) {
case 2:
@@ -248,11 +187,6 @@ function numberAttribute(span: LocalTraceSpan, key: string): number | undefined
return undefined;
}
-function stringAttribute(span: LocalTraceSpan, key: string): string | undefined {
- const value = span.attributes[key];
- return typeof value === "string" && value.length > 0 ? value : undefined;
-}
-
function durationMs(start: bigint, end: bigint): number {
return Number(end - start) / 1_000_000;
}
diff --git a/packages/eve/src/cli/commands/trace.ts b/packages/eve/src/cli/commands/trace.ts
index afaf6872b..1d9f092ae 100644
--- a/packages/eve/src/cli/commands/trace.ts
+++ b/packages/eve/src/cli/commands/trace.ts
@@ -5,8 +5,8 @@ import {
formatTokenSummary,
renderSpanDetailTree,
spanMetricChips,
- summarizeLocalTrace,
} from "#cli/commands/trace-detail.js";
+import { summarizeLocalTrace } from "#tracing/local-trace-summary.js";
import { formatElapsed } from "#cli/format-elapsed.js";
import { createCliTheme, renderCliSection, sanitizeForTerminal } from "#cli/ui/output.js";
import type { LocalTrace, LocalTraceSpan } from "#tracing/local-trace-reader.js";
@@ -164,17 +164,17 @@ export async function runTraceShowCommand(
}
function traceHeaderRows(trace: LocalTrace): { label: string; value: string }[] {
- const summary = summarizeLocalTrace(trace.spans);
+ const summary = summarizeLocalTrace(trace.traceId, trace.spans);
return [
{ label: "Trace ID", value: trace.traceId },
{ label: "Conversation ID", value: trace.conversationId ?? "unknown" },
{ label: "Agent", value: trace.agentName ?? "unknown" },
- { label: "Started", value: toDate(trace.startTimeNs).toISOString() },
+ { label: "Started", value: summary.startedAt },
{
label: "Duration",
- value: formatElapsed(durationMs(trace.startTimeNs, trace.endTimeNs)),
+ value: formatElapsed(summary.durationMs),
},
- { label: "Spans", value: String(trace.spans.length) },
+ { label: "Spans", value: String(summary.spanCount) },
...(summary.models.length === 0 ? [] : [{ label: "Models", value: summary.models.join(", ") }]),
...(summary.inputTokens === 0 && summary.outputTokens === 0
? []
@@ -182,12 +182,12 @@ function traceHeaderRows(trace: LocalTrace): { label: string; value: string }[]
...(summary.costUsd === undefined
? []
: [{ label: "Cost", value: formatCostUsd(summary.costUsd) }]),
- ...(summary.errorCount === 0
+ ...(summary.errorSpanCount === 0
? []
: [
{
label: "Errors",
- value: `${summary.errorCount} span${summary.errorCount === 1 ? "" : "s"}`,
+ value: `${summary.errorSpanCount} span${summary.errorSpanCount === 1 ? "" : "s"}`,
},
]),
];
diff --git a/packages/eve/src/cli/dev/tui/traces/trace-conversation.test.ts b/packages/eve/src/cli/dev/tui/traces/trace-conversation.test.ts
index 13f49151b..42f749a42 100644
--- a/packages/eve/src/cli/dev/tui/traces/trace-conversation.test.ts
+++ b/packages/eve/src/cli/dev/tui/traces/trace-conversation.test.ts
@@ -436,11 +436,12 @@ describe("renderConversationItem", () => {
expect(lines[1]).not.toMatch(/\buser\b/);
});
- it("shows gateway cost right-aligned when the step span carries it", () => {
+ it.each([
+ [{ "gen_ai.usage.cost": 0.0031 }, 0.0031],
+ [{ "gen_ai.usage.cost": 0.5, "gen_ai.usage.gateway_cost": 0.0123 }, 0.0123],
+ ])("shows the same cost and precision as the trace tree for %o", (attributes, cost) => {
const turn = span("a".repeat(16), "agent.turn", 0, 0, undefined, {});
- const step = span("b".repeat(16), "agent.step", 10, 5000, turn.spanId, {
- "gen_ai.usage.cost": 0.0031,
- });
+ const step = span("b".repeat(16), "agent.step", 10, 5000, turn.spanId, attributes);
const model = span("c".repeat(16), "ai.streamText.doStream", 20, 2000, step.spanId, {
"gen_ai.request.model": "claude-test",
"gen_ai.input.messages": JSON.stringify([
@@ -452,9 +453,9 @@ describe("renderConversationItem", () => {
const assistant = buildConversationItems(trace([turn, step, model])).find(
(item) => item.kind === "assistant",
)!;
- expect(assistant.costUsd).toBe(0.0031);
+ expect(assistant.costUsd).toBe(cost);
const lines = renderConversationItem(assistant, 80, THEME, false, false).map(stripAnsi);
- expect(lines[1]!.trimEnd()).toMatch(/\$0\.0031$/);
+ expect(lines[1]!.trimEnd().endsWith(`$${cost.toFixed(4)}`)).toBe(true);
});
it("renders tool calls as their own cards with args and result", () => {
diff --git a/packages/eve/src/cli/dev/tui/traces/trace-conversation.ts b/packages/eve/src/cli/dev/tui/traces/trace-conversation.ts
index 960ce6398..22c3e2777 100644
--- a/packages/eve/src/cli/dev/tui/traces/trace-conversation.ts
+++ b/packages/eve/src/cli/dev/tui/traces/trace-conversation.ts
@@ -5,11 +5,13 @@
* cards directly; model response spans provide assistant cards.
*/
+import { formatCostUsd } from "#cli/commands/trace-detail.js";
import { formatElapsed } from "#cli/format-elapsed.js";
import { clipVisible, stripTerminalControls, visibleLength } from "#cli/ui/terminal-text.js";
import type { LocalTrace, LocalTraceSpan } from "#tracing/local-trace-reader.js";
import { compareLocalTraceSpans, isAgentTurnSpan } from "#tracing/local-trace-reader.js";
import { agentTurnIdentity } from "#tracing/agent-span-contract.js";
+import { localTraceSpanCostUsd } from "#tracing/local-trace-summary.js";
import { formatCompactTokenCount } from "../stream-format.js";
import type { Theme } from "../theme.js";
@@ -443,16 +445,10 @@ function assistantMetrics(
parts.push(`${glyph.arrowUp}${formatCompactTokenCount(item.inputTokens)}`);
if (item.outputTokens !== undefined)
parts.push(`${glyph.arrowDown}${formatCompactTokenCount(item.outputTokens)}`);
- if (item.costUsd !== undefined) parts.push(formatCost(item.costUsd));
+ if (item.costUsd !== undefined) parts.push(formatCostUsd(item.costUsd));
return parts.length === 0 ? "" : colors.dim(parts.join(" · "));
}
-/** Formats a USD cost with enough precision for typical AI inference prices. */
-function formatCost(usd: number): string {
- if (usd >= 0.01) return `$${usd.toFixed(2)}`;
- return `$${usd.toFixed(4)}`;
-}
-
/** Reads the gateway cost from the model span's ancestor step span. */
function stepCostUsd(
modelSpan: LocalTraceSpan,
@@ -464,7 +460,7 @@ function stepCostUsd(
while (span.parentSpanId !== undefined) {
span = byId.get(span.parentSpanId);
if (span === undefined) return undefined;
- if (span.name === "agent.step") return numberAttribute(span, "gen_ai.usage.cost");
+ if (span.name === "agent.step") return localTraceSpanCostUsd(span);
}
return undefined;
}
diff --git a/packages/eve/src/self-modification/extension/local-only.ts b/packages/eve/src/self-modification/extension/local-only.ts
new file mode 100644
index 000000000..a6ea2b5b5
--- /dev/null
+++ b/packages/eve/src/self-modification/extension/local-only.ts
@@ -0,0 +1,23 @@
+import { defineDynamic } from "#dynamic/definition.js";
+
+import { resolveSelfModificationConfig, type ResolvedSelfModificationConfig } from "../config.js";
+import { resolveSelfModificationMode } from "../mode.js";
+import selfModification from "./extension.js";
+
+/** Returns a definition only when self-modification is running locally. */
+export function resolveLocalOnly<T>(
+ config: ResolvedSelfModificationConfig,
+ definition: T,
+): T | null {
+ return resolveSelfModificationMode(config) === "local" ? definition : null;
+}
+
+/** Creates a dynamic definition that is absent from deployed self-modification. */
+export function defineLocalOnlyDynamic<T>(definition: T) {
+ return defineDynamic({
+ events: {
+ "session.started": () =>
+ resolveLocalOnly(resolveSelfModificationConfig(selfModification.config), definition),
+ },
+ });
+}
diff --git a/packages/eve/src/self-modification/extension/subagents/agent/instructions.test.ts b/packages/eve/src/self-modification/extension/subagents/agent/instructions.test.ts
deleted file mode 100644
index 8b8d7a088..000000000
--- a/packages/eve/src/self-modification/extension/subagents/agent/instructions.test.ts
+++ /dev/null
@@ -1,25 +0,0 @@
-import { describe, expect, it } from "vitest";
-
-import instructions from "./instructions.js";
-
-describe("self-modification subagent instructions", () => {
- it("handles a bundled mount that is not visible until the next turn", () => {
- const originalEveDev = process.env.EVE_DEV;
- process.env.EVE_DEV = "1";
- try {
- const resolved = instructions.events["session.started"]?.({ data: {} }, {} as never);
-
- expect(resolved).toMatchObject({
- markdown: expect.stringContaining(
- "first call registry_add with the exact address eve/self-modification",
- ),
- });
- expect(resolved).toMatchObject({
- markdown: expect.stringContaining("Check /source/extensions/self-modification.ts"),
- });
- } finally {
- if (originalEveDev === undefined) delete process.env.EVE_DEV;
- else process.env.EVE_DEV = originalEveDev;
- }
- });
-});
diff --git a/packages/eve/src/self-modification/extension/subagents/agent/instructions.ts b/packages/eve/src/self-modification/extension/subagents/agent/instructions.ts
index 57a5806aa..070c0c60d 100644
--- a/packages/eve/src/self-modification/extension/subagents/agent/instructions.ts
+++ b/packages/eve/src/self-modification/extension/subagents/agent/instructions.ts
@@ -60,8 +60,7 @@ const localGuidance = `## Local environment
The registry_add tool will complete installation for items that need no setup. In the local dev TUI, a \`needs-terminal\` result from the tool call automatically opens the existing setup panel for the user to complete setup there. In headless development, if a \`needs-terminal\` result includes \`nextCommand\`, present that exact value as the only shell command in your response. Never infer, construct, or rewrite a command: installing an item uses \`eve add <item>\`; \`eve registry add\` configures registry namespace mappings and does not install items.
-Local eve dev logs are available read-only at /logs.
-Local trace segments are mounted read-only at /traces when available. Inspect other traces only when the user asks about another session or broader behavior.
+Local eve dev logs are mounted read-only at /logs. Local trace segments are mounted read-only at /traces when available. For latency, failure, token, or behavior analysis, load and follow the \`trace_analysis\` skill. Trace searches are scoped to the invoking conversation and exclude the current investigation by default.
The application package.json is not mounted. Do not search outside /source for application files. You cannot run host binaries such as git, node, pnpm, or tsc. Use existing imports and registry_add for supported registry installations.`;
@@ -73,38 +72,19 @@ The configured target branch is checked out as a disposable workspace under /wor
Complete all edits and registry installations before publication, and call publish by itself. Before publication, review and summarize the complete intended scope. Call publish once with a concise title and summary. A successful result is only a draft pull request. Return its URL and changed paths, and state that merge and deployment have not occurred.`;
-const packagedSubagentGuidance = `## Configure this subagent
+const packagedSubagentGuidance = `## Changing this subagent
-Configure this self-modification subagent's model, reasoning, and policy only through its authored mount. Check /source/extensions/self-modification.ts and /source/extensions/self-modification/extension.ts; modify whichever exists. If neither file exists, this subagent is the bundled eve development default and is using the default settings: first call registry_add with the exact address eve/self-modification to scaffold the authored mount. This known scaffold does not require search_registry.
+When the user explicitly asks to change the self-modification subagent's model, reasoning, or policy, make edits only through its authored mount. For such requests, check /source/extensions/self-modification.ts and /source/extensions/self-modification/extension.ts; modify whichever exists. If neither file exists, this subagent is the bundled eve development default and is using the default settings: first call registry_add with the exact address eve/self-modification to scaffold the authored mount. This known scaffold does not require search_registry.
After registry_add reports successful installation, try to read /source/extensions/self-modification.ts and /source/extensions/self-modification/extension.ts. If either is available, modify it. If neither is yet available, you may need to wait for the next turn, by asking the requester to confirm.`;
-function readTrace(
- event: unknown,
-): { readonly traceFlags: number; readonly traceId: string } | undefined {
- return (
- event as {
- readonly data?: {
- readonly trace?: { readonly traceFlags: number; readonly traceId: string };
- };
- }
- ).data?.trace;
-}
-
-function localTraceGuidance(event: unknown): string {
- const trace = readTrace(event);
- if (trace === undefined) return "";
-
- return `The invoking trace has ID ${trace.traceId}. ${(trace.traceFlags & 1) === 1 ? "If local segments were captured," : "This trace was not sampled, so local segments may be absent. If any are present,"} inspect them at /traces/${trace.traceId}.`;
-}
-
function renderInstructions(sections: readonly string[]): string {
return sections.filter((section) => section.length > 0).join("\n\n");
}
export default defineDynamic({
events: {
- "session.started": (event) => {
+ "session.started": () => {
const mode = resolveSelfModificationMode(
resolveSelfModificationConfig(selfModification.config),
);
@@ -120,7 +100,6 @@ export default defineDynamic({
registryWorkflow,
documentationGuidance,
mode === "local" ? localGuidance : deployedGuidance,
- mode === "local" ? localTraceGuidance(event) : "",
workingGuidance,
mode === "local" ? localReportingGuidance : deployedReportingGuidance,
]),
diff --git a/packages/eve/src/self-modification/extension/subagents/agent/skills/trace_analysis.ts b/packages/eve/src/self-modification/extension/subagents/agent/skills/trace_analysis.ts
new file mode 100644
index 000000000..8de03a65f
--- /dev/null
+++ b/packages/eve/src/self-modification/extension/subagents/agent/skills/trace_analysis.ts
@@ -0,0 +1,34 @@
+import { defineSkill } from "eve/skills";
+
+import type { ResolvedSelfModificationConfig } from "../../../../config.js";
+import { defineLocalOnlyDynamic, resolveLocalOnly } from "../../../local-only.js";
+
+const traceAnalysisSkill = defineSkill({
+ description:
+ "Analyze local eve traces for latency, token usage, failures, repeated tool calls, and discovery inefficiency with a bounded, conversation-scoped investigation.",
+ markdown: `# Trace analysis
+
+Use this workflow when the user asks to review, diagnose, or optimize behavior from local traces.
+
+## Keep the investigation bounded
+
+1. Call \`search_traces\` once with \`limit\` no greater than 5. It is already scoped to the invoking conversation, including its child-agent traces, and excludes the current investigation by default.
+2. Rank findings across correctness, latency, token usage, and tool-call efficiency. Follow the user's stated priority; do not assume that "improve" means only fixing errors.
+3. Call \`inspect_trace\` only for the one or two traces needed to explain a high-impact signal. Set \`limit\` to the smallest useful bound, normally 20–40. Its timeline includes bounded argument previews and \`availableFields\` for each span.
+4. Use \`inspect_trace_spans\` only with span ids returned by a timeline. Include only the required payload fields and batch related spans in one call. An empty \`availableFields\` list means none are recorded, not that the call failed.
+5. Use the read-only \`/logs\` mount only to corroborate a specific runtime error that the structural trace cannot explain. Search a literal diagnostic fragment with bounded output before reading a file. With no observed failure, skip broad log searches and speculative web searches for possible failures.
+6. Inspect \`/source\` or \`/eve-docs\` only when evidence points to a persistent authored change or an unresolved API question.
+
+Prefer three analysis rounds or fewer. Do not read raw trace segments, search the whole filesystem, or enumerate broad documentation when the bounded tools answer the question.
+
+Search and inspection share the same summary. Its durationMs is elapsed time; inspection's modelWorkMs and toolWorkMs sum operation time and may exceed elapsed time when calls overlap. errorSpanCount counts error-bearing spans, not independent failures or final activation outcome; failures ranking uses this count. Token totals sum step usage without counting repeated model counters.
+
+For latency, distinguish tool execution time from model round-trip time. Repeated tool counts are leads, not proof that calls are independent; inspect the compact timeline before recommending batching. Report the highest-impact opportunities first.
+`,
+});
+
+export function resolveTraceAnalysisSkill(config: ResolvedSelfModificationConfig) {
+ return resolveLocalOnly(config, traceAnalysisSkill);
+}
+
+export default defineLocalOnlyDynamic(traceAnalysisSkill);
diff --git a/packages/eve/src/self-modification/extension/subagents/agent/tools/inspect_trace.ts b/packages/eve/src/self-modification/extension/subagents/agent/tools/inspect_trace.ts
new file mode 100644
index 000000000..1a0135b8d
--- /dev/null
+++ b/packages/eve/src/self-modification/extension/subagents/agent/tools/inspect_trace.ts
@@ -0,0 +1,128 @@
+import { defineTool } from "eve/tools";
+
+import { analyzeLocalTrace } from "#tracing/local-trace-analysis.js";
+
+import type { ResolvedSelfModificationConfig } from "../../../../config.js";
+import { defineLocalOnlyDynamic, resolveLocalOnly } from "../../../local-only.js";
+import {
+ boundedText,
+ detailFields,
+ readTraceSources,
+ TRACE_ID,
+} from "../../../trace-inspection.js";
+
+const MAX_RECORDS = 200;
+const MAX_OFFSET = 100_000;
+const DEFAULT_RECORDS = 40;
+const MAX_ARGUMENT_PREVIEW = 200;
+
+const inputSchema = {
+ type: "object",
+ additionalProperties: false,
+ properties: {
+ limit: { type: "integer", minimum: 1, maximum: MAX_RECORDS },
+ offset: {
+ type: "integer",
+ minimum: 0,
+ maximum: MAX_OFFSET,
+ description: "Zero-based record offset for paging through larger traces.",
+ },
+ traceId: { type: "string", pattern: "^[0-9a-f]{32}$" },
+ },
+ required: ["traceId"],
+} as const;
+
+const inspectTraceTool = defineTool({
+ description:
+ "Inspect one trace as a bounded structural timeline. Use offset to page through larger traces. Tool records include a short argument preview and availableFields; use inspect_trace_spans only for selected raw tool payloads.",
+ inputSchema,
+ outputSchema: { type: "object", additionalProperties: true },
+ async execute(input, ctx) {
+ const { limit, offset, traceId } = parseInput(input);
+ const sources = await readTraceSources(traceId, ctx);
+ const analysis = analyzeLocalTrace(
+ traceId,
+ sources.map(({ span }) => span),
+ );
+ const byId = new Map(sources.map((source) => [source.span.spanId, source]));
+ const records = analysis.records.slice(offset, offset + limit).map((record) => {
+ const source = byId.get(record.spanId);
+ const fields = source === undefined ? undefined : detailFields(source);
+ const availableFields =
+ fields === undefined
+ ? []
+ : (["arguments", "result", "error"] as const).filter(
+ (field) => typeof fields[field] === "string",
+ );
+ const argumentsValue = fields?.arguments;
+ return {
+ ...boundedRecord(record),
+ availableFields,
+ ...(record.category === "tool" && typeof argumentsValue === "string"
+ ? { argumentsPreview: boundedText(argumentsValue, MAX_ARGUMENT_PREVIEW) }
+ : {}),
+ };
+ });
+ return {
+ traceId,
+ offset,
+ timeline: records,
+ summary: analysis.summary,
+ modelWorkMs: analysis.modelWorkMs,
+ toolWorkMs: analysis.toolWorkMs,
+ groups: analysis.groups,
+ total: analysis.records.length,
+ hasMore: offset + records.length < analysis.records.length,
+ truncated: offset + records.length < analysis.records.length,
+ };
+ },
+});
+
+export function resolveInspectTraceTool(config: ResolvedSelfModificationConfig) {
+ return resolveLocalOnly(config, inspectTraceTool);
+}
+
+export default defineLocalOnlyDynamic(inspectTraceTool);
+
+function parseInput(value: unknown): {
+ readonly limit: number;
+ readonly offset: number;
+ readonly traceId: string;
+} {
+ if (typeof value !== "object" || value === null || Array.isArray(value)) {
+ throw new Error("Trace inspection input must be an object.");
+ }
+ const input = value as Record<string, unknown>;
+ if (typeof input.traceId !== "string" || !TRACE_ID.test(input.traceId)) {
+ throw new Error("traceId must be a lowercase 32-character hex id.");
+ }
+ if (
+ input.limit !== undefined &&
+ (typeof input.limit !== "number" ||
+ !Number.isInteger(input.limit) ||
+ input.limit < 1 ||
+ input.limit > MAX_RECORDS)
+ ) {
+ throw new Error(`limit must be an integer between 1 and ${MAX_RECORDS}.`);
+ }
+ if (
+ input.offset !== undefined &&
+ (typeof input.offset !== "number" ||
+ !Number.isInteger(input.offset) ||
+ input.offset < 0 ||
+ input.offset > MAX_OFFSET)
+ ) {
+ throw new Error(`offset must be an integer between 0 and ${MAX_OFFSET}.`);
+ }
+ return {
+ limit: (input.limit as number | undefined) ?? DEFAULT_RECORDS,
+ offset: (input.offset as number | undefined) ?? 0,
+ traceId: input.traceId,
+ };
+}
+
+function boundedRecord(
+ record: ReturnType<typeof analyzeLocalTrace>["records"][number],
+): ReturnType<typeof analyzeLocalTrace>["records"][number] {
+ return record.error === undefined ? record : { ...record, error: boundedText(record.error, 500) };
+}
diff --git a/packages/eve/src/self-modification/extension/subagents/agent/tools/inspect_trace_spans.ts b/packages/eve/src/self-modification/extension/subagents/agent/tools/inspect_trace_spans.ts
new file mode 100644
index 000000000..761a95d93
--- /dev/null
+++ b/packages/eve/src/self-modification/extension/subagents/agent/tools/inspect_trace_spans.ts
@@ -0,0 +1,114 @@
+import { defineTool } from "eve/tools";
+
+import type { ResolvedSelfModificationConfig } from "../../../../config.js";
+import { defineLocalOnlyDynamic, resolveLocalOnly } from "../../../local-only.js";
+import { detailFields, readTraceSources, SPAN_ID, TRACE_ID } from "../../../trace-inspection.js";
+
+const FIELDS = ["arguments", "result", "error"] as const;
+type Field = (typeof FIELDS)[number];
+const MAX_SPANS = 10;
+const MAX_FIELD_BYTES = 16 * 1024;
+const MAX_TOTAL_FIELD_BYTES = 32 * 1024;
+
+const inputSchema = {
+ type: "object",
+ additionalProperties: false,
+ properties: {
+ include: {
+ type: "array",
+ minItems: 1,
+ maxItems: FIELDS.length,
+ items: { enum: FIELDS, type: "string" },
+ },
+ spanIds: {
+ type: "array",
+ minItems: 1,
+ maxItems: MAX_SPANS,
+ items: { type: "string", pattern: "^[0-9a-f]{16}$" },
+ },
+ traceId: { type: "string", pattern: "^[0-9a-f]{32}$" },
+ },
+ required: ["traceId", "spanIds", "include"],
+} as const;
+
+const inspectTraceSpansTool = defineTool({
+ description:
+ "Return bounded arguments, results, or errors for selected spans in one trace. First use inspect_trace to choose spanIds and see availableFields.",
+ inputSchema,
+ outputSchema: { type: "object", additionalProperties: true },
+ async execute(input, ctx) {
+ const { include, spanIds, traceId } = parseInput(input);
+ const sources = new Map(
+ (await readTraceSources(traceId, ctx)).map((source) => [source.span.spanId, source]),
+ );
+ let remainingBytes = MAX_TOTAL_FIELD_BYTES;
+ let truncated = false;
+ const spans = spanIds.map((spanId) => {
+ const source = sources.get(spanId);
+ if (source === undefined) return { spanId, found: false };
+ const values = detailFields(source);
+ const availableFields = FIELDS.filter((field) => typeof values[field] === "string");
+ const fields: Record<string, string> = {};
+ let spanTruncated = false;
+ for (const field of include) {
+ const value = values[field];
+ if (typeof value !== "string") continue;
+ const maximum = Math.min(MAX_FIELD_BYTES, remainingBytes);
+ fields[field] = value.slice(0, maximum);
+ remainingBytes -= fields[field].length;
+ if (value.length > maximum) spanTruncated = true;
+ }
+ truncated ||= spanTruncated;
+ return {
+ spanId,
+ found: true,
+ availableFields,
+ fields,
+ rawPath: `/traces/${traceId}/segments/${source.segmentFile}`,
+ truncated: spanTruncated,
+ };
+ });
+ return { traceId, spans, truncated };
+ },
+});
+
+export function resolveInspectTraceSpansTool(config: ResolvedSelfModificationConfig) {
+ return resolveLocalOnly(config, inspectTraceSpansTool);
+}
+
+export default defineLocalOnlyDynamic(inspectTraceSpansTool);
+
+function parseInput(value: unknown): {
+ readonly include: readonly Field[];
+ readonly spanIds: readonly string[];
+ readonly traceId: string;
+} {
+ if (typeof value !== "object" || value === null || Array.isArray(value)) {
+ throw new Error("Trace span inspection input must be an object.");
+ }
+ const input = value as Record<string, unknown>;
+ if (typeof input.traceId !== "string" || !TRACE_ID.test(input.traceId)) {
+ throw new Error("traceId must be a lowercase 32-character hex id.");
+ }
+ if (
+ !Array.isArray(input.spanIds) ||
+ input.spanIds.length === 0 ||
+ input.spanIds.length > MAX_SPANS ||
+ input.spanIds.some((spanId) => typeof spanId !== "string" || !SPAN_ID.test(spanId))
+ ) {
+ throw new Error(`spanIds must contain between 1 and ${MAX_SPANS} lowercase span ids.`);
+ }
+ if (
+ !Array.isArray(input.include) ||
+ input.include.length === 0 ||
+ input.include.length > FIELDS.length ||
+ input.include.some((field) => !FIELDS.includes(field as Field))
+ ) {
+ throw new Error("include must contain arguments, result, or error.");
+ }
+ return {
+ include: input.include as Field[],
+ spanIds: [...new Set(input.spanIds as string[])],
+ traceId: input.traceId,
+ };
+}
diff --git a/packages/eve/src/self-modification/extension/subagents/agent/tools/search_traces.ts b/packages/eve/src/self-modification/extension/subagents/agent/tools/search_traces.ts
new file mode 100644
index 000000000..4b5e82246
--- /dev/null
+++ b/packages/eve/src/self-modification/extension/subagents/agent/tools/search_traces.ts
@@ -0,0 +1,208 @@
+import { defineTool } from "eve/tools";
+
+import { context, trace } from "#compiled/@opentelemetry/api/index.js";
+import { queryLocalTraceSummaries, type LocalTraceSortBy } from "#tracing/local-trace-query.js";
+import { summarizeLocalTrace, type LocalTraceSummary } from "#tracing/local-trace-summary.js";
+import { resolveConversationId } from "#tracing/conversation-context.js";
+import {
+ localTraceConversationMarker,
+ localTraceIndexedMarker,
+} from "#tracing/local-trace-discovery-index.js";
+
+import type { ResolvedSelfModificationConfig } from "../../../../config.js";
+import { defineLocalOnlyDynamic, resolveLocalOnly } from "../../../local-only.js";
+import {
+ readTraceSources,
+ TRACE_ID,
+ type LocalTraceSpanSource,
+} from "../../../trace-inspection.js";
+
+const MAX_ANALYZED_TRACES = 200;
+const MAX_UNINDEXED_TRACES = 100;
+const DEFAULT_SORT_BY = "latest";
+const MAX_RESULTS = 50;
+
+const inputSchema = {
+ type: "object",
+ additionalProperties: false,
+ properties: {
+ limit: {
+ type: "integer",
+ minimum: 1,
+ maximum: MAX_RESULTS,
+ default: MAX_RESULTS,
+ },
+ sortBy: {
+ enum: ["duration", "failures", "inputTokens", "latest"],
+ type: "string",
+ default: DEFAULT_SORT_BY,
+ description:
+ "Rank examined traces by elapsed time, error-span count, summed step input tokens, or start time. Defaults to latest.",
+ },
+ },
+} as const;
+
+interface SearchInput {
+ readonly limit: number;
+ readonly sortBy: LocalTraceSortBy;
+}
+
+const searchTracesTool = defineTool({
+ description: "Search summaries for traces in the invoking conversation.",
+ inputSchema,
+ outputSchema: { type: "object", additionalProperties: true },
+ async execute(input, ctx) {
+ const parsed = parseInput(input);
+ const currentTraceId = trace.getSpan(context.active())?.spanContext().traceId;
+ const conversationId = resolveConversationId(
+ ctx.session.parent?.rootSessionId ?? ctx.session.id,
+ );
+ const sandbox = await ctx.getSandbox();
+ const [available, indexed, conversation] = await Promise.all([
+ sandbox.run({ command: "ls -1dt /traces/*" }),
+ sandbox.run({ command: `ls -1 /traces/*/${localTraceIndexedMarker()}` }),
+ sandbox.run({ command: `ls -1 /traces/*/${localTraceConversationMarker(conversationId)}` }),
+ ]);
+ if (available.exitCode !== 0 && !isEmptyTraceListing(available)) {
+ throw new Error(`Could not list local traces: ${available.stderr}`);
+ }
+
+ const stored = [...traceIds(available.stdout)];
+ const eligible = stored.filter((traceId) => traceId !== currentTraceId);
+ const indexedIds = traceIds(indexed.stdout);
+ const matchingIds = new Set(traceIds(conversation.stdout));
+ const unindexed = eligible.filter((traceId) => !indexedIds.has(traceId));
+ const scannedUnindexed = unindexed.slice(0, MAX_UNINDEXED_TRACES);
+ const digests = new Map<string, LocalTraceSummary>();
+ let readFailures = 0;
+ const read = async (traceId: string): Promise<LocalTraceSpanSource[] | undefined> => {
+ try {
+ return await readTraceSources(traceId, ctx);
+ } catch (error) {
+ if (ctx.abortSignal.aborted) throw error;
+ readFailures += 1;
+ return undefined;
+ }
+ };
+
+ for (const traceId of scannedUnindexed) {
+ if (ctx.abortSignal.aborted) throw new Error("Trace search was cancelled.");
+ const sources = await read(traceId);
+ if (sources === undefined || !contains(sources, "gen_ai.conversation.id", conversationId))
+ continue;
+ matchingIds.add(traceId);
+ digests.set(
+ traceId,
+ summarizeLocalTrace(
+ traceId,
+ sources.map(({ span }) => span),
+ ),
+ );
+ }
+
+ const matched = eligible.filter((traceId) => matchingIds.has(traceId));
+ const considered = matched.slice(0, MAX_ANALYZED_TRACES);
+ const summaries: LocalTraceSummary[] = [];
+ for (const traceId of considered) {
+ if (ctx.abortSignal.aborted) throw new Error("Trace search was cancelled.");
+ const cached = digests.get(traceId);
+ if (cached !== undefined) {
+ summaries.push(cached);
+ continue;
+ }
+ const sources = await read(traceId);
+ if (sources !== undefined)
+ summaries.push(
+ summarizeLocalTrace(
+ traceId,
+ sources.map(({ span }) => span),
+ ),
+ );
+ }
+ const results = queryLocalTraceSummaries(summaries, parsed);
+ const omitted = unindexed.length - scannedUnindexed.length + matched.length - considered.length;
+ const coverage: {
+ complete: boolean;
+ considered: number;
+ matched: number;
+ stored: number;
+ warning?: string;
+ } = {
+ complete: omitted === 0 && readFailures === 0,
+ considered: considered.length,
+ matched: matched.length,
+ stored: stored.length,
+ };
+ const warnings = [];
+ if (omitted !== 0) warnings.push("Some older traces could not be included.");
+ if (readFailures !== 0) warnings.push("Some traces could not be read.");
+ if (warnings.length > 0) coverage.warning = warnings.join(" ");
+ return {
+ conversationId,
+ coverage,
+ matches: results.matches,
+ truncated: results.truncated,
+ };
+ },
+});
+
+function parseInput(value: unknown): SearchInput {
+ if (typeof value !== "object" || value === null || Array.isArray(value)) {
+ throw new Error("Trace search input must be an object.");
+ }
+ const input = value as Record<string, unknown>;
+ const limit = input.limit === undefined ? MAX_RESULTS : input.limit;
+ if (typeof limit !== "number" || !Number.isInteger(limit) || limit < 1 || limit > MAX_RESULTS) {
+ throw new Error(`limit must be an integer between 1 and ${MAX_RESULTS}.`);
+ }
+ if (
+ input.sortBy !== undefined &&
+ input.sortBy !== "latest" &&
+ input.sortBy !== "duration" &&
+ input.sortBy !== "failures" &&
+ input.sortBy !== "inputTokens"
+ ) {
+ throw new Error("sortBy must be latest, duration, failures, or inputTokens.");
+ }
+ return {
+ limit,
+ sortBy: (input.sortBy as LocalTraceSortBy | undefined) ?? DEFAULT_SORT_BY,
+ };
+}
+
+function traceIds(value: string): Set<string> {
+ const ids = new Set<string>();
+ for (const line of value.split(/\r?\n/u)) {
+ const id = line
+ .split("/")
+ .filter(Boolean)
+ .find((part) => TRACE_ID.test(part));
+ if (id !== undefined) ids.add(id);
+ }
+ return ids;
+}
+
+function isEmptyTraceListing(result: {
+ readonly stderr: string;
+ readonly stdout: string;
+}): boolean {
+ return (
+ result.stdout.trim() === "" &&
+ result.stderr.includes("/traces/*") &&
+ /no such file or directory/iu.test(result.stderr)
+ );
+}
+
+function contains(
+ sources: readonly LocalTraceSpanSource[],
+ attribute: string,
+ expected: string,
+): boolean {
+ return sources.some(({ span }) => span.attributes[attribute] === expected);
+}
+
+export function resolveSearchTracesTool(config: ResolvedSelfModificationConfig) {
+ return resolveLocalOnly(config, searchTracesTool);
+}
+
+export default defineLocalOnlyDynamic(searchTracesTool);
diff --git a/packages/eve/src/self-modification/extension/trace-inspection.ts b/packages/eve/src/self-modification/extension/trace-inspection.ts
new file mode 100644
index 000000000..06e9348fd
--- /dev/null
+++ b/packages/eve/src/self-modification/extension/trace-inspection.ts
@@ -0,0 +1,61 @@
+import type { ToolContext } from "eve/tools";
+
+import { parseLocalTraceSegment, type LocalTraceSpan } from "#tracing/local-trace-reader.js";
+
+export interface LocalTraceSpanSource {
+ readonly segmentFile: string;
+ readonly span: LocalTraceSpan;
+}
+
+export const TRACE_ID = /^[0-9a-f]{32}$/u;
+export const SPAN_ID = /^[0-9a-f]{16}$/u;
+
+const SEGMENT_FILE = /^[0-9a-f]{16}\.otlp\.json$/u;
+const MAX_SEGMENTS = 500;
+const MAX_SEGMENT_BYTES = 8 * 1024 * 1024;
+
+export async function readTraceSources(
+ traceId: string,
+ ctx: Pick<ToolContext, "abortSignal" | "getSandbox">,
+): Promise<LocalTraceSpanSource[]> {
+ const sandbox = await ctx.getSandbox();
+ const directory = `/traces/${traceId}/segments`;
+ const listed = await sandbox.run({ command: `ls -1 ${directory}` });
+ if (listed.exitCode !== 0) {
+ throw new Error(`Could not list local trace segments: ${listed.stderr}`);
+ }
+ const files = listed.stdout
+ .split(/\r?\n/u)
+ .filter((name) => SEGMENT_FILE.test(name))
+ .sort((left, right) => left.localeCompare(right));
+ if (files.length > MAX_SEGMENTS) {
+ throw new Error(`Trace has ${files.length} segments (maximum ${MAX_SEGMENTS}).`);
+ }
+
+ const sources: LocalTraceSpanSource[] = [];
+ for (const file of files) {
+ if (ctx.abortSignal.aborted) throw new Error("Trace inspection was cancelled.");
+ const content = await sandbox.readTextFile({ path: `${directory}/${file}` });
+ if (content === null || content.length > MAX_SEGMENT_BYTES) continue;
+ for (const span of parseLocalTraceSegment(content, traceId)) {
+ sources.push({ segmentFile: file, span });
+ }
+ }
+ return sources;
+}
+
+export function boundedText(value: string, maximum: number): string {
+ return value.length <= maximum ? value : `${value.slice(0, maximum - 1)}…`;
+}
+
+export function detailFields(source: LocalTraceSpanSource): {
+ readonly arguments: unknown;
+ readonly error: string | undefined;
+ readonly result: unknown;
+} {
+ return {
+ arguments: source.span.attributes["gen_ai.tool.call.arguments"],
+ error: source.span.statusMessage,
+ result: source.span.attributes["gen_ai.tool.call.result"],
+ };
+}
diff --git a/packages/eve/src/self-modification/inspect-trace.test.ts b/packages/eve/src/self-modification/inspect-trace.test.ts
new file mode 100644
index 000000000..1a3972ab5
--- /dev/null
+++ b/packages/eve/src/self-modification/inspect-trace.test.ts
@@ -0,0 +1,364 @@
+import { afterEach, beforeEach, describe, expect, it } from "vitest";
+
+import { resolveTraceAnalysisSkill } from "./extension/subagents/agent/skills/trace_analysis.js";
+import { resolveInspectTraceTool } from "./extension/subagents/agent/tools/inspect_trace.js";
+import { resolveInspectTraceSpansTool } from "./extension/subagents/agent/tools/inspect_trace_spans.js";
+import { resolveSearchTracesTool } from "./extension/subagents/agent/tools/search_traces.js";
+
+const traceId = "1".repeat(32);
+const otherTraceId = "2".repeat(32);
+const spanId = "a".repeat(16);
+const toolSpanId = "b".repeat(16);
+const originalEveDev = process.env.EVE_DEV;
+
+beforeEach(() => {
+ process.env.EVE_DEV = "1";
+});
+
+afterEach(() => {
+ if (originalEveDev === undefined) delete process.env.EVE_DEV;
+ else process.env.EVE_DEV = originalEveDev;
+});
+
+function segment(
+ input: {
+ conversationId?: string;
+ id?: string;
+ runId?: string;
+ sessionId?: string;
+ status?: { code: number; message?: string };
+ stepInputTokens?: readonly number[];
+ toolName?: string;
+ } = {},
+): string {
+ const id = input.id ?? traceId;
+ const attributes = [
+ input.runId === undefined
+ ? { key: "agent.session.id", value: { stringValue: input.sessionId ?? "session-a" } }
+ : { key: "agent.run.id", value: { stringValue: input.runId } },
+ { key: "gen_ai.conversation.id", value: { stringValue: input.conversationId ?? "root" } },
+ { key: "gen_ai.operation.name", value: { stringValue: "chat" } },
+ { key: "gen_ai.request.model", value: { stringValue: "test-model" } },
+ ];
+ const spans: Record<string, unknown>[] = [
+ {
+ attributes,
+ endTimeUnixNano: "20000000",
+ name: "chat",
+ spanId,
+ startTimeUnixNano: "10000000",
+ status: { code: 0 },
+ traceId: id,
+ },
+ ];
+ if (input.toolName !== undefined) {
+ spans.push({
+ attributes: [
+ { key: "gen_ai.operation.name", value: { stringValue: "execute_tool" } },
+ { key: "gen_ai.tool.name", value: { stringValue: input.toolName } },
+ { key: "gen_ai.tool.call.arguments", value: { stringValue: "x".repeat(250) } },
+ { key: "gen_ai.tool.call.result", value: { stringValue: "tool result" } },
+ ],
+ endTimeUnixNano: "30000000",
+ name: `execute_tool ${input.toolName}`,
+ spanId: toolSpanId,
+ startTimeUnixNano: "20000000",
+ status: input.status ?? { code: 0 },
+ traceId: id,
+ });
+ }
+ for (const [index, tokens] of (input.stepInputTokens ?? []).entries()) {
+ spans.push({
+ attributes: [{ key: "agent.usage.input_tokens", value: { intValue: tokens } }],
+ endTimeUnixNano: "30000000",
+ name: "agent.step",
+ spanId: (index + 1).toString(16).padStart(16, "0"),
+ startTimeUnixNano: "10000000",
+ status: { code: 0 },
+ traceId: id,
+ });
+ }
+ return JSON.stringify({ resourceSpans: [{ scopeSpans: [{ spans }] }] });
+}
+
+function traceContext(contents: Record<string, string>) {
+ return {
+ abortSignal: new AbortController().signal,
+ getSandbox: async () => ({
+ readTextFile: async ({ path }: { path: string }) =>
+ Object.entries(contents).find(([id]) => path.includes(id))?.[1] ?? null,
+ run: async ({ command }: { command: string }) => {
+ if (command === "ls -1dt /traces/*") {
+ return {
+ exitCode: 0,
+ stderr: "",
+ stdout: Object.keys(contents)
+ .map((id) => `/traces/${id}`)
+ .join("\n"),
+ };
+ }
+ if (command.includes("/segments")) {
+ return { exitCode: 0, stderr: "", stdout: `${spanId}.otlp.json\n` };
+ }
+ return { exitCode: 2, stderr: "", stdout: "" };
+ },
+ }),
+ session: { id: "root" },
+ } as never;
+}
+
+describe("self-modification trace inspection tools", () => {
+ it("defaults an omitted search limit", async () => {
+ const ctx = traceContext({
+ [traceId]: segment({ id: traceId, toolName: "read", status: { code: 2 } }),
+ [otherTraceId]: segment({ id: otherTraceId }),
+ });
+ const search = resolveSearchTracesTool({ localEnabled: true })!;
+
+ const defaultSearch = await search.execute({}, ctx);
+ const sortedSearch = await search.execute({ sortBy: "failures" }, ctx);
+
+ expect(defaultSearch).toMatchObject({ matches: [{ traceId }, { traceId: otherTraceId }] });
+ expect(sortedSearch).toMatchObject({
+ matches: [
+ { traceId, errorSpanCount: 1 },
+ { traceId: otherTraceId, errorSpanCount: 0 },
+ ],
+ });
+ });
+
+ it("ranks summed step usage and returns the same summary from search and paged inspection", async () => {
+ const ctx = traceContext({
+ [traceId]: segment({ stepInputTokens: [100, 100], toolName: "read", status: { code: 2 } }),
+ [otherTraceId]: segment({ id: otherTraceId, stepInputTokens: [150] }),
+ });
+ const search = await resolveSearchTracesTool({ localEnabled: true })!.execute(
+ { sortBy: "inputTokens", limit: 1 },
+ ctx,
+ );
+ const inspection = await resolveInspectTraceTool({ localEnabled: true })!.execute(
+ { traceId, limit: 1 },
+ ctx,
+ );
+ if (!("matches" in search) || !("summary" in inspection)) {
+ throw new Error("Expected non-streaming trace tools.");
+ }
+ expect(search).toMatchObject({
+ matches: [{ traceId, inputTokens: 200, errorSpanCount: 1, toolNames: ["read"] }],
+ truncated: true,
+ });
+ expect(inspection).toMatchObject({ summary: search.matches[0], hasMore: true });
+ expect(inspection.summary).toEqual(search.matches[0]);
+ expect(inspection).not.toHaveProperty("totals");
+ expect(inspection.summary).not.toHaveProperty("failedOperations");
+ });
+
+ it("searches structural trace metadata within the invoking conversation", async () => {
+ const ctx = {
+ abortSignal: new AbortController().signal,
+ getSandbox: async () => ({
+ readTextFile: async ({ path }: { path: string }) =>
+ path.includes(traceId)
+ ? segment({
+ id: traceId,
+ runId: "target",
+ toolName: "deploy",
+ status: { code: 2, message: "failed" },
+ })
+ : segment({ id: otherTraceId, runId: "other" }),
+ run: async ({ command }: { command: string }) => {
+ if (command === "ls -1dt /traces/*") {
+ return {
+ exitCode: 0,
+ stderr: "",
+ stdout: `/traces/${traceId}\n/traces/${otherTraceId}\n`,
+ };
+ }
+ if (command.includes("/segments"))
+ return { exitCode: 0, stderr: "", stdout: `${spanId}.otlp.json\n` };
+ return { exitCode: 2, stderr: "", stdout: "" };
+ },
+ }),
+ session: { id: "root" },
+ } as never;
+
+ const result = await resolveSearchTracesTool({ localEnabled: true })!.execute(
+ { limit: 10 },
+ ctx,
+ );
+
+ expect(result).toMatchObject({
+ coverage: { complete: true, considered: 2, matched: 2, stored: 2 },
+ matches: [
+ {
+ traceId,
+ errorSpanCount: 1,
+ sessionIds: ["target"],
+ toolNames: ["deploy"],
+ },
+ { traceId: otherTraceId, errorSpanCount: 0, sessionIds: ["other"], toolNames: [] },
+ ],
+ });
+ });
+
+ it("reports incomplete coverage without exposing unindexed trace internals", async () => {
+ const ids = Array.from({ length: 101 }, (_, index) => index.toString(16).padStart(32, "0"));
+ const result = await resolveSearchTracesTool({ localEnabled: true })!.execute({ limit: 10 }, {
+ abortSignal: new AbortController().signal,
+ getSandbox: async () => ({
+ readTextFile: async () => segment({ conversationId: "other" }),
+ run: async ({ command }: { command: string }) =>
+ command === "ls -1dt /traces/*"
+ ? { exitCode: 0, stderr: "", stdout: ids.map((id) => `/traces/${id}`).join("\n") }
+ : command.includes("/segments")
+ ? { exitCode: 0, stderr: "", stdout: `${spanId}.otlp.json\n` }
+ : { exitCode: 2, stderr: "", stdout: "" },
+ }),
+ session: { id: "root" },
+ } as never);
+
+ expect(result).toMatchObject({
+ coverage: {
+ complete: false,
+ considered: 0,
+ matched: 0,
+ stored: 101,
+ warning: expect.any(String),
+ },
+ });
+ expect(JSON.stringify(result)).not.toContain("legacy");
+ });
+
+ it("treats an empty trace mount as an empty result", async () => {
+ const result = await resolveSearchTracesTool({ localEnabled: true })!.execute({ limit: 10 }, {
+ abortSignal: new AbortController().signal,
+ getSandbox: async () => ({
+ run: async ({ command }: { command: string }) =>
+ command === "ls -1dt /traces/*"
+ ? {
+ exitCode: 2,
+ stderr: "ls: cannot access '/traces/*': No such file or directory",
+ stdout: "",
+ }
+ : { exitCode: 2, stderr: "", stdout: "" },
+ }),
+ session: { id: "root" },
+ } as never);
+
+ expect(result).toMatchObject({ coverage: { complete: true, stored: 0 }, matches: [] });
+ });
+
+ it("keeps readable matches when one trace disappears during the scan", async () => {
+ const missingTraceId = "3".repeat(32);
+ const result = await resolveSearchTracesTool({ localEnabled: true })!.execute({ limit: 10 }, {
+ abortSignal: new AbortController().signal,
+ getSandbox: async () => ({
+ readTextFile: async () => segment({ id: traceId }),
+ run: async ({ command }: { command: string }) => {
+ if (command === "ls -1dt /traces/*") {
+ return {
+ exitCode: 0,
+ stderr: "",
+ stdout: `/traces/${missingTraceId}\n/traces/${traceId}\n`,
+ };
+ }
+ if (command.includes(`/traces/${missingTraceId}/segments`)) {
+ return { exitCode: 2, stderr: "trace was pruned", stdout: "" };
+ }
+ if (command.includes("/segments")) {
+ return { exitCode: 0, stderr: "", stdout: `${spanId}.otlp.json\n` };
+ }
+ return { exitCode: 2, stderr: "", stdout: "" };
+ },
+ }),
+ session: { id: "root" },
+ } as never);
+
+ expect(result).toMatchObject({
+ coverage: {
+ complete: false,
+ matched: 1,
+ stored: 2,
+ warning: expect.stringContaining("read"),
+ },
+ matches: [{ traceId }],
+ });
+ });
+
+ it("resolves trace analysis only in local mode", () => {
+ delete process.env.EVE_DEV;
+ expect(resolveSearchTracesTool({ localEnabled: true })).toBeNull();
+ expect(resolveInspectTraceTool({ localEnabled: true })).toBeNull();
+ expect(resolveInspectTraceSpansTool({ localEnabled: true })).toBeNull();
+ expect(resolveTraceAnalysisSkill({ localEnabled: true })).toBeNull();
+ });
+
+ it("returns a structural timeline with argument previews and available fields", async () => {
+ const result = await resolveInspectTraceTool({ localEnabled: true })!.execute(
+ { traceId, limit: 2 },
+ traceContext({
+ [traceId]: segment({ id: traceId, toolName: "bash" }),
+ }),
+ );
+
+ expect(result).toMatchObject({
+ timeline: [
+ { category: "model", availableFields: [] },
+ {
+ argumentsPreview: "x".repeat(199) + "…",
+ availableFields: ["arguments", "result"],
+ category: "tool",
+ spanId: toolSpanId,
+ },
+ ],
+ summary: { modelCalls: 1, toolCalls: 1 },
+ });
+
+ const page = await resolveInspectTraceTool({ localEnabled: true })!.execute(
+ { limit: 1, offset: 1, traceId },
+ traceContext({ [traceId]: segment({ id: traceId, toolName: "bash" }) }),
+ );
+ expect(page).toMatchObject({
+ hasMore: false,
+ offset: 1,
+ timeline: [{ category: "tool", spanId: toolSpanId }],
+ total: 2,
+ });
+ });
+
+ it("returns selected payloads for several spans in one call", async () => {
+ const result = await resolveInspectTraceSpansTool({ localEnabled: true })!.execute(
+ { include: ["arguments", "result"], spanIds: [toolSpanId, "c".repeat(16)], traceId },
+ traceContext({ [traceId]: segment({ id: traceId, toolName: "bash" }) }),
+ );
+
+ expect(result).toEqual({
+ traceId,
+ truncated: false,
+ spans: [
+ expect.objectContaining({
+ availableFields: ["arguments", "result"],
+ fields: { arguments: "x".repeat(250), result: "tool result" },
+ found: true,
+ spanId: toolSpanId,
+ }),
+ { found: false, spanId: "c".repeat(16) },
+ ],
+ });
+ });
+
+ it("validates the focused tool inputs", async () => {
+ await expect(
+ resolveInspectTraceTool({ localEnabled: true })!.execute(
+ { operation: "summary", traceId },
+ {} as never,
+ ),
+ ).rejects.toThrow();
+ await expect(
+ resolveInspectTraceSpansTool({ localEnabled: true })!.execute(
+ { include: ["prompt"], spanIds: [spanId], traceId },
+ {} as never,
+ ),
+ ).rejects.toThrow("include must contain arguments, result, or error.");
+ });
+});
diff --git a/packages/eve/src/self-modification/package-consumption.scenario.test.ts b/packages/eve/src/self-modification/package-consumption.scenario.test.ts
index 4f9e451b0..f8abfd83e 100644
--- a/packages/eve/src/self-modification/package-consumption.scenario.test.ts
+++ b/packages/eve/src/self-modification/package-consumption.scenario.test.ts
@@ -5,7 +5,7 @@ import { dirname, join, resolve } from "node:path";
import { fileURLToPath } from "node:url";
import { promisify } from "node:util";
-import { afterEach, describe, it } from "vitest";
+import { afterEach, describe, expect, it } from "vitest";
import { renderSelfModificationConfig } from "./setup.js";
@@ -81,9 +81,46 @@ describe("packed package consumption", () => {
await access(join(packageRoot, "dist/src/self-modification/config.js"));
await access(join(packageRoot, "dist/src/self-modification/sandbox.js"));
await access(join(packageRoot, "dist/src/self-modification/setup.js"));
- await access(
- join(packageRoot, "dist/src/self-modification/extension/subagents/agent/tools/edit_file.js"),
- );
+ await Promise.all([
+ access(
+ join(
+ packageRoot,
+ "dist/src/self-modification/extension/subagents/agent/skills/trace_analysis.js",
+ ),
+ ),
+ access(
+ join(
+ packageRoot,
+ "dist/src/self-modification/extension/subagents/agent/tools/edit_file.js",
+ ),
+ ),
+ access(
+ join(
+ packageRoot,
+ "dist/src/self-modification/extension/subagents/agent/tools/inspect_trace.js",
+ ),
+ ),
+ access(
+ join(
+ packageRoot,
+ "dist/src/self-modification/extension/subagents/agent/tools/inspect_trace_spans.js",
+ ),
+ ),
+ access(
+ join(
+ packageRoot,
+ "dist/src/self-modification/extension/subagents/agent/tools/search_traces.js",
+ ),
+ ),
+ ]);
+ await Promise.all([
+ expect(
+ access(join(packageRoot, "dist/src/self-modification/extension/skills/trace_analysis.js")),
+ ).rejects.toMatchObject({ code: "ENOENT" }),
+ expect(
+ access(join(packageRoot, "dist/src/self-modification/extension/tools/inspect_trace.js")),
+ ).rejects.toMatchObject({ code: "ENOENT" }),
+ ]);
const root = await mkdtemp(join(tmpdir(), "eve-self-modification-package-"));
temporaryRoots.push(root);
diff --git a/packages/eve/src/tracing/agent-telemetry-contract.integration.test.ts b/packages/eve/src/tracing/agent-telemetry-contract.integration.test.ts
index eaf5b1e89..9431138f0 100644
--- a/packages/eve/src/tracing/agent-telemetry-contract.integration.test.ts
+++ b/packages/eve/src/tracing/agent-telemetry-contract.integration.test.ts
@@ -47,7 +47,7 @@ import {
parseLocalTraceSegment,
type LocalTraceSpan,
} from "#tracing/local-trace-reader.js";
-import { summarizeLocalTrace } from "#cli/commands/trace-detail.js";
+import { summarizeLocalTrace } from "#tracing/local-trace-summary.js";
import { buildConversationItems } from "#cli/dev/tui/traces/trace-conversation.js";
import { contentFilteringProcessor } from "#tracing/content-span-processor.js";
import { ConversationContextKey } from "#shared/conversation-context.js";
@@ -662,7 +662,7 @@ describe("exported agent telemetry contract", () => {
"trace parent:turn_2 outcome=cancelled channel=http:web delivery=delivery-cancelled",
" invoke_agent parent",
]);
- expect(summarizeLocalTrace(parsed)).toMatchObject({
+ expect(summarizeLocalTrace(parsed[0]!.traceId, parsed)).toMatchObject({
inputTokens: 10,
outputTokens: 5,
cacheReadTokens: 4,
diff --git a/packages/eve/src/tracing/local-instrumentation-runtime.scenario.test.ts b/packages/eve/src/tracing/local-instrumentation-runtime.scenario.test.ts
index 3843d9257..77f7f85ce 100644
--- a/packages/eve/src/tracing/local-instrumentation-runtime.scenario.test.ts
+++ b/packages/eve/src/tracing/local-instrumentation-runtime.scenario.test.ts
@@ -23,6 +23,10 @@ import {
turnIdempotencyKey,
} from "#instrumentation/lifecycle.js";
import { installLocalInstrumentationRuntime } from "#tracing/local-instrumentation-runtime.js";
+import {
+ localTraceConversationMarker,
+ localTraceIndexedMarker,
+} from "#tracing/local-trace-discovery-index.js";
import { LocalTraceSpanProcessor } from "#tracing/local-trace-span-processor.js";
const temporaryDirectories: string[] = [];
@@ -266,14 +270,20 @@ describe("local instrumentation runtime", () => {
}),
);
- firstProvider.getTracer("worker-1").startSpan("worker.one", {}, parent).end();
- secondProvider.getTracer("worker-2").startSpan("worker.two", {}, parent).end();
+ const attributes = { "gen_ai.conversation.id": "conversation-1" };
+ firstProvider.getTracer("worker-1").startSpan("worker.one", { attributes }, parent).end();
+ secondProvider.getTracer("worker-2").startSpan("worker.two", { attributes }, parent).end();
await Promise.all([firstProcessor.forceFlush(), secondProcessor.forceFlush()]);
- const segments = await readdir(
- join(appRoot, ".eve", "traces", "v1", "a".repeat(32), "segments"),
- );
+ const traceDirectory = join(appRoot, ".eve", "traces", "v1", "a".repeat(32));
+ const segments = await readdir(join(traceDirectory, "segments"));
expect(segments).toHaveLength(2);
+ await expect(
+ readFile(join(traceDirectory, localTraceConversationMarker("conversation-1")), "utf8"),
+ ).resolves.toBe("");
+ await expect(readFile(join(traceDirectory, localTraceIndexedMarker()), "utf8")).resolves.toBe(
+ "",
+ );
});
});
diff --git a/packages/eve/src/tracing/local-trace-analysis.test.ts b/packages/eve/src/tracing/local-trace-analysis.test.ts
new file mode 100644
index 000000000..050da718c
--- /dev/null
+++ b/packages/eve/src/tracing/local-trace-analysis.test.ts
@@ -0,0 +1,307 @@
+import { describe, expect, it } from "vitest";
+
+import type { LocalTraceSpan } from "./local-trace-reader.js";
+import { analyzeLocalTrace } from "./local-trace-analysis.js";
+import { summarizeLocalTrace } from "./local-trace-summary.js";
+
+const traceId = "1".repeat(32);
+
+function span(input: {
+ readonly attributes?: Record<string, unknown>;
+ readonly endMs: number;
+ readonly name: string;
+ readonly parentSpanId?: string;
+ readonly spanId: string;
+ readonly startMs: number;
+ readonly statusCode?: number;
+ readonly statusMessage?: string;
+}): LocalTraceSpan {
+ return {
+ attributes: input.attributes ?? {},
+ endTimeNs: BigInt(input.endMs) * 1_000_000n,
+ events: [],
+ name: input.name,
+ parentSpanId: input.parentSpanId,
+ spanId: input.spanId,
+ startTimeNs: BigInt(input.startMs) * 1_000_000n,
+ statusCode: input.statusCode ?? 0,
+ statusMessage: input.statusMessage,
+ traceId,
+ };
+}
+
+describe("analyzeLocalTrace", () => {
+ it("returns an ordered structural timeline and the same summary as search", () => {
+ const spans = [
+ span({
+ attributes: {
+ "agent.session.id": "child",
+ "agent.turn.id": "turn-1",
+ "gen_ai.operation.name": "execute_tool",
+ "gen_ai.tool.name": "read_file",
+ },
+ endMs: 25,
+ name: "execute_tool read_file",
+ spanId: "b".repeat(16),
+ startMs: 20,
+ }),
+ span({
+ attributes: {
+ "agent.model.id": "test-model",
+ "agent.session.id": "child",
+ "agent.turn.id": "turn-1",
+ "gen_ai.operation.name": "chat",
+ },
+ endMs: 15,
+ name: "ai.chat",
+ spanId: "a".repeat(16),
+ startMs: 0,
+ }),
+ ];
+ const analysis = analyzeLocalTrace(traceId, spans);
+
+ expect(analysis.summary).toEqual(summarizeLocalTrace(traceId, spans));
+ expect(analysis).toMatchObject({
+ summary: { durationMs: 25, modelCalls: 1, toolCalls: 1 },
+ modelWorkMs: 15,
+ toolWorkMs: 5,
+ });
+ expect(analysis.records).toEqual([
+ expect.objectContaining({
+ category: "model",
+ model: "test-model",
+ spanId: "a".repeat(16),
+ startOffsetMs: 0,
+ }),
+ expect.objectContaining({
+ category: "tool",
+ spanId: "b".repeat(16),
+ startOffsetMs: 20,
+ toolName: "read_file",
+ }),
+ ]);
+ });
+
+ it("collapses action wrappers, retaining execution timing and inherited selectors", () => {
+ const action = span({
+ attributes: {
+ "agent.action.kind": "tool-call",
+ "agent.action.name": "read_file",
+ "agent.action.call_id": "call-1",
+ "agent.session.id": "target",
+ "agent.turn.id": "turn-1",
+ },
+ endMs: 30,
+ name: "agent.action",
+ spanId: "a".repeat(16),
+ startMs: 0,
+ statusCode: 2,
+ statusMessage: "Read failed",
+ });
+ const execution = span({
+ attributes: { "gen_ai.operation.name": "execute_tool", "gen_ai.tool.name": "read_file" },
+ endMs: 25,
+ name: "execute_tool read_file",
+ parentSpanId: action.spanId,
+ spanId: "b".repeat(16),
+ startMs: 20,
+ statusCode: 2,
+ });
+ const analysis = analyzeLocalTrace(traceId, [execution, action], {
+ sessionId: "target",
+ turnId: "turn-1",
+ });
+
+ expect(analysis).toMatchObject({
+ summary: { durationMs: 30, toolCalls: 1, errorSpanCount: 2 },
+ toolWorkMs: 5,
+ });
+ expect(analysis.records).toEqual([
+ expect.objectContaining({
+ callId: "call-1",
+ category: "tool",
+ error: "Read failed",
+ outcome: "failed",
+ sessionId: "target",
+ spanId: execution.spanId,
+ startOffsetMs: 20,
+ turnId: "turn-1",
+ }),
+ ]);
+ });
+
+ it("keeps repeated calls distinct and counts tools without execution spans", () => {
+ const action = span({
+ attributes: { "agent.action.kind": "tool-call", "agent.action.name": "web_search" },
+ endMs: 10,
+ name: "agent.action",
+ spanId: "a".repeat(16),
+ startMs: 0,
+ });
+ const analysis = analyzeLocalTrace(traceId, [action, { ...action, spanId: "b".repeat(16) }]);
+ expect(analysis.summary.toolCalls).toBe(2);
+ expect(analysis.records).toHaveLength(2);
+ expect(analysis.records[0]).toMatchObject({ category: "tool", toolName: "web_search" });
+ });
+
+ it("preserves a background action lifetime without adding it to tool work", () => {
+ const analysis = analyzeLocalTrace(traceId, [
+ span({
+ attributes: { "agent.action.kind": "subagent-call", "agent.action.name": "worker" },
+ endMs: 100,
+ name: "agent.action",
+ spanId: "a".repeat(16),
+ startMs: 0,
+ }),
+ span({
+ attributes: { "gen_ai.operation.name": "execute_tool", "gen_ai.tool.name": "worker" },
+ endMs: 10,
+ name: "execute_tool worker",
+ parentSpanId: "a".repeat(16),
+ spanId: "b".repeat(16),
+ startMs: 5,
+ }),
+ ]);
+ expect(analysis).toMatchObject({ summary: { durationMs: 100, toolCalls: 1 }, toolWorkMs: 5 });
+ expect(analysis.records).toEqual([
+ expect.objectContaining({ actionDurationMs: 100, durationMs: 5 }),
+ ]);
+ });
+
+ it("reports per-model token usage without inventing missing metrics", () => {
+ const model = span({
+ attributes: {
+ "gen_ai.operation.name": "chat",
+ "agent.usage.input_tokens": 100,
+ "agent.usage.output_tokens": 10,
+ "agent.usage.cache_read_tokens": 80,
+ "agent.usage.cache_write_tokens": 0,
+ },
+ endMs: 10,
+ name: "chat test-model",
+ spanId: "a".repeat(16),
+ startMs: 0,
+ });
+ const analysis = analyzeLocalTrace(traceId, [
+ model,
+ {
+ ...model,
+ name: "invoke_agent",
+ attributes: { "agent.usage.input_tokens": 100 },
+ spanId: "b".repeat(16),
+ },
+ {
+ ...model,
+ attributes: {
+ "gen_ai.operation.name": "chat",
+ "agent.usage.input_tokens": -1,
+ "agent.usage.output_tokens": Infinity,
+ },
+ spanId: "c".repeat(16),
+ },
+ ]);
+ expect(analysis.records).toHaveLength(2);
+ expect(analysis.records[0]?.usage).toEqual({
+ inputTokens: 100,
+ outputTokens: 10,
+ cacheReadTokens: 80,
+ cacheWriteTokens: 0,
+ });
+ expect(analysis.records[1]?.usage).toBeUndefined();
+ });
+
+ it("counts errors outside the timeline and uses the current run identity", () => {
+ const analysis = analyzeLocalTrace(
+ traceId,
+ [
+ span({
+ attributes: { "agent.run.id": "run-1" },
+ endMs: 10,
+ name: "agent.step",
+ spanId: "a".repeat(16),
+ startMs: 0,
+ statusCode: 2,
+ }),
+ span({
+ attributes: { "agent.run.id": "run-1", "gen_ai.operation.name": "chat" },
+ endMs: 20,
+ name: "chat",
+ spanId: "b".repeat(16),
+ startMs: 10,
+ }),
+ ],
+ { sessionId: "run-1" },
+ );
+ expect(analysis.summary.errorSpanCount).toBe(1);
+ expect(analysis.records).toEqual([
+ expect.objectContaining({ category: "model", sessionId: "run-1" }),
+ ]);
+ });
+
+ it("filters structurally and ignores duplicate span ids", () => {
+ const shared = span({
+ attributes: {
+ "agent.session.id": "target",
+ "agent.turn.id": "turn-1",
+ "gen_ai.operation.name": "chat",
+ },
+ endMs: 10,
+ name: "ai.chat",
+ spanId: "a".repeat(16),
+ startMs: 0,
+ });
+ const analysis = analyzeLocalTrace(
+ traceId,
+ [
+ shared,
+ shared,
+ span({
+ attributes: { "agent.session.id": "analyzer", "gen_ai.operation.name": "chat" },
+ endMs: 30,
+ name: "ai.chat",
+ spanId: "b".repeat(16),
+ startMs: 20,
+ }),
+ ],
+ { excludeSessionId: "analyzer" },
+ );
+ expect(analysis.summary.modelCalls).toBe(1);
+ expect(analysis.records[0]?.sessionId).toBe("target");
+ });
+
+ it("distinguishes overlapping work from elapsed time and other actions from tools", () => {
+ const model = span({
+ attributes: { "agent.run.id": "run", "gen_ai.operation.name": "chat" },
+ endMs: 10,
+ name: "chat",
+ spanId: "a".repeat(16),
+ startMs: 0,
+ });
+ const analysis = analyzeLocalTrace(traceId, [
+ model,
+ { ...model, spanId: "b".repeat(16) },
+ span({
+ attributes: {
+ "agent.run.id": "run",
+ "agent.action.kind": "subagent-call",
+ "agent.action.name": "worker",
+ },
+ endMs: 10,
+ name: "agent.action",
+ spanId: "c".repeat(16),
+ startMs: 0,
+ }),
+ ]);
+ expect(analysis).toMatchObject({
+ summary: { durationMs: 10, modelCalls: 2, toolCalls: 0, toolNames: [] },
+ modelWorkMs: 20,
+ toolWorkMs: 0,
+ });
+ expect(analysis.groups).toEqual([
+ { sessionId: "run", modelCalls: 2, modelWorkMs: 20, toolCalls: 0, toolWorkMs: 0 },
+ ]);
+ expect(analysis.records.find((record) => record.category === "other")).toMatchObject({
+ actionName: "worker",
+ });
+ });
+});
diff --git a/packages/eve/src/tracing/local-trace-analysis.ts b/packages/eve/src/tracing/local-trace-analysis.ts
new file mode 100644
index 000000000..bea89d00d
--- /dev/null
+++ b/packages/eve/src/tracing/local-trace-analysis.ts
@@ -0,0 +1,183 @@
+import type { LocalTraceSpan } from "#tracing/local-trace-reader.js";
+import {
+ localTraceOperations,
+ traceModelName,
+ traceSessionId,
+ traceStringAttribute,
+ traceToolName,
+ uniqueLocalTraceSpans,
+ type LocalTraceOperation,
+} from "#tracing/local-trace-operations.js";
+import { summarizeLocalTrace, type LocalTraceSummary } from "#tracing/local-trace-summary.js";
+
+export interface TraceAnalysisSelector {
+ readonly excludeSessionId?: string;
+ readonly fromMs?: number;
+ readonly sessionId?: string;
+ readonly toMs?: number;
+ readonly turnId?: string;
+}
+
+export interface TraceTokenUsage {
+ readonly inputTokens?: number;
+ readonly outputTokens?: number;
+ readonly cacheReadTokens?: number;
+ readonly cacheWriteTokens?: number;
+}
+
+export interface TraceTimelineRecord {
+ readonly actionDurationMs?: number;
+ readonly actionName?: string;
+ readonly callId?: string;
+ readonly category: "model" | "other" | "tool";
+ readonly durationMs: number;
+ readonly endOffsetMs: number;
+ readonly error?: string;
+ readonly model?: string;
+ readonly outcome: "completed" | "failed";
+ readonly parentSpanId?: string;
+ readonly sessionId?: string;
+ readonly spanId: string;
+ readonly startOffsetMs: number;
+ readonly toolName?: string;
+ readonly turnId?: string;
+ readonly usage?: TraceTokenUsage;
+}
+
+export interface TraceAnalysis {
+ readonly summary: LocalTraceSummary;
+ readonly groups: readonly TraceAnalysisGroup[];
+ /** Accumulated operation time, not elapsed time; parallel calls can overlap. */
+ readonly modelWorkMs: number;
+ readonly toolWorkMs: number;
+ readonly records: readonly TraceTimelineRecord[];
+}
+
+type Mutable<T> = { -readonly [Key in keyof T]: T[Key] };
+
+export interface TraceAnalysisGroup {
+ readonly modelWorkMs: number;
+ readonly modelCalls: number;
+ readonly sessionId?: string;
+ readonly toolWorkMs: number;
+ readonly toolCalls: number;
+ readonly turnId?: string;
+}
+
+/** Builds the detailed timeline only when inspection, rather than search, needs it. */
+export function analyzeLocalTrace(
+ traceId: string,
+ spans: readonly LocalTraceSpan[],
+ selector: TraceAnalysisSelector = {},
+): TraceAnalysis {
+ const unique = uniqueLocalTraceSpans(spans);
+ const inherited = new Map(localTraceOperations(unique).map(({ span }) => [span.spanId, span]));
+ const selected = unique.filter((span) => matches(inherited.get(span.spanId) ?? span, selector));
+ const startTimeNs = selected.reduce(
+ (start, span) => (span.startTimeNs < start ? span.startTimeNs : start),
+ selected[0]?.startTimeNs ?? 0n,
+ );
+ const records = localTraceOperations(selected)
+ .map((operation) => recordFor(operation, startTimeNs))
+ .sort(
+ (left, right) =>
+ left.startOffsetMs - right.startOffsetMs ||
+ left.endOffsetMs - right.endOffsetMs ||
+ left.spanId.localeCompare(right.spanId),
+ );
+ return {
+ summary: summarizeLocalTrace(traceId, selected),
+ groups: groups(records),
+ modelWorkMs: sum(records, "model"),
+ toolWorkMs: sum(records, "tool"),
+ records,
+ };
+}
+
+function matches(span: LocalTraceSpan, selector: TraceAnalysisSelector): boolean {
+ const sessionId = traceSessionId(span);
+ const turnId = traceStringAttribute(span, "agent.turn.id");
+ if (selector.excludeSessionId !== undefined && sessionId === selector.excludeSessionId)
+ return false;
+ if (selector.sessionId !== undefined && sessionId !== selector.sessionId) return false;
+ if (selector.turnId !== undefined && turnId !== selector.turnId) return false;
+ const startMs = Number(span.startTimeNs / 1_000_000n);
+ return (
+ (selector.fromMs === undefined || startMs >= selector.fromMs) &&
+ (selector.toMs === undefined || startMs <= selector.toMs)
+ );
+}
+
+function recordFor(operation: LocalTraceOperation, startTimeNs: bigint): TraceTimelineRecord {
+ const { span, category } = operation;
+ const durationMs = Number(span.endTimeNs - span.startTimeNs) / 1_000_000;
+ const record: Mutable<TraceTimelineRecord> = {
+ category,
+ durationMs,
+ endOffsetMs: Number(span.endTimeNs - startTimeNs) / 1_000_000,
+ outcome: span.statusCode === 2 ? "failed" : "completed",
+ spanId: span.spanId,
+ startOffsetMs: Number(span.startTimeNs - startTimeNs) / 1_000_000,
+ };
+ const callId = traceStringAttribute(span, "agent.action.call_id");
+ const model = traceModelName(span);
+ const sessionId = traceSessionId(span);
+ const name = traceToolName(span);
+ const turnId = traceStringAttribute(span, "agent.turn.id");
+ if (callId !== undefined) record.callId = callId;
+ if (span.statusMessage !== undefined) record.error = span.statusMessage;
+ if (model !== undefined) record.model = model;
+ if (span.parentSpanId !== undefined) record.parentSpanId = span.parentSpanId;
+ if (sessionId !== undefined) record.sessionId = sessionId;
+ if (name !== undefined && category === "tool") record.toolName = name;
+ if (name !== undefined && category === "other") record.actionName = name;
+ if (turnId !== undefined) record.turnId = turnId;
+ if (operation.actionDurationMs !== undefined)
+ record.actionDurationMs = operation.actionDurationMs;
+ if (category === "model") {
+ const usage: Mutable<TraceTokenUsage> = {};
+ for (const [field, attribute] of Object.entries({
+ inputTokens: "agent.usage.input_tokens",
+ outputTokens: "agent.usage.output_tokens",
+ cacheReadTokens: "agent.usage.cache_read_tokens",
+ cacheWriteTokens: "agent.usage.cache_write_tokens",
+ })) {
+ const value = span.attributes[attribute];
+ if (typeof value === "number" && Number.isFinite(value) && value >= 0) {
+ usage[field as keyof TraceTokenUsage] = value;
+ }
+ }
+ if (Object.keys(usage).length > 0) record.usage = usage;
+ }
+ return record;
+}
+
+function groups(records: readonly TraceTimelineRecord[]): TraceAnalysisGroup[] {
+ const values = new Map<string, Mutable<TraceAnalysisGroup>>();
+ for (const record of records) {
+ const key = JSON.stringify([record.sessionId, record.turnId]);
+ let group = values.get(key);
+ if (group === undefined) {
+ group = { modelCalls: 0, modelWorkMs: 0, toolCalls: 0, toolWorkMs: 0 };
+ if (record.sessionId !== undefined) group.sessionId = record.sessionId;
+ if (record.turnId !== undefined) group.turnId = record.turnId;
+ values.set(key, group);
+ }
+ if (record.category === "model") {
+ group.modelCalls += 1;
+ group.modelWorkMs += record.durationMs;
+ }
+ if (record.category === "tool") {
+ group.toolCalls += 1;
+ group.toolWorkMs += record.durationMs;
+ }
+ }
+ return [...values.values()];
+}
+
+function sum(records: readonly TraceTimelineRecord[], category: "model" | "tool"): number {
+ return records.reduce(
+ (total, record) => total + (record.category === category ? record.durationMs : 0),
+ 0,
+ );
+}
diff --git a/packages/eve/src/tracing/local-trace-discovery-index.ts b/packages/eve/src/tracing/local-trace-discovery-index.ts
new file mode 100644
index 000000000..c7baad9fe
--- /dev/null
+++ b/packages/eve/src/tracing/local-trace-discovery-index.ts
@@ -0,0 +1,52 @@
+import { createHash } from "node:crypto";
+import { mkdir, writeFile } from "node:fs/promises";
+import { join } from "node:path";
+
+const INDEX_DIRECTORY = "index/v1";
+const CONVERSATIONS_DIRECTORY = `${INDEX_DIRECTORY}/conversations`;
+const INDEXED_MARKER = `${INDEX_DIRECTORY}/indexed`;
+
+/** A path-safe, non-reversible key for one recorded conversation identifier. */
+export function localTraceConversationIndexKey(conversationId: string): string {
+ return createHash("sha256").update(conversationId).digest("hex");
+}
+
+/** Marker path relative to one trace directory. */
+export function localTraceConversationMarker(conversationId: string): string {
+ return `${CONVERSATIONS_DIRECTORY}/${localTraceConversationIndexKey(conversationId)}`;
+}
+
+/** Marker proving that a trace was written by an index-aware writer. */
+export function localTraceIndexedMarker(): string {
+ return INDEXED_MARKER;
+}
+
+/**
+ * Records conversation membership without rewriting shared state. Empty marker
+ * creation is idempotent, so overlapping dev workers cannot lose each other's
+ * conversation entries.
+ */
+export async function indexLocalTraceConversation(input: {
+ readonly conversationId: string;
+ readonly traceDirectory: string;
+}): Promise<void> {
+ const conversationMarker = join(
+ input.traceDirectory,
+ localTraceConversationMarker(input.conversationId),
+ );
+ await mkdir(join(input.traceDirectory, CONVERSATIONS_DIRECTORY), { recursive: true });
+ await createMarker(conversationMarker);
+ await createMarker(join(input.traceDirectory, INDEXED_MARKER));
+}
+
+async function createMarker(path: string): Promise<void> {
+ try {
+ await writeFile(path, "", { flag: "wx" });
+ } catch (error) {
+ if (!isAlreadyPresent(error)) throw error;
+ }
+}
+
+function isAlreadyPresent(error: unknown): boolean {
+ return error instanceof Error && "code" in error && error.code === "EEXIST";
+}
diff --git a/packages/eve/src/tracing/local-trace-operations.ts b/packages/eve/src/tracing/local-trace-operations.ts
new file mode 100644
index 000000000..5bd7e9da3
--- /dev/null
+++ b/packages/eve/src/tracing/local-trace-operations.ts
@@ -0,0 +1,84 @@
+import type { LocalTraceSpan } from "#tracing/local-trace-reader.js";
+
+export interface LocalTraceOperation {
+ readonly span: LocalTraceSpan;
+ readonly category: "model" | "other" | "tool";
+ readonly actionDurationMs?: number;
+}
+
+export function uniqueLocalTraceSpans(spans: readonly LocalTraceSpan[]): LocalTraceSpan[] {
+ const byId = new Map<string, LocalTraceSpan>();
+ for (const span of spans) {
+ if (!byId.has(span.spanId)) byId.set(span.spanId, span);
+ }
+ return [...byId.values()];
+}
+
+/** Pairs action wrappers with execution spans without constructing a sorted timeline. */
+export function localTraceOperations(spans: readonly LocalTraceSpan[]): LocalTraceOperation[] {
+ const byId = new Map(spans.map((span) => [span.spanId, span]));
+ const pairedActions = new Set<string>();
+ const operations: LocalTraceOperation[] = [];
+ for (const span of spans) {
+ const category = categoryFor(span);
+ if (category === undefined) continue;
+ const action = span.parentSpanId === undefined ? undefined : byId.get(span.parentSpanId);
+ if (
+ span.attributes["gen_ai.operation.name"] === "execute_tool" &&
+ action?.name === "agent.action" &&
+ traceToolName(action) === traceToolName(span)
+ ) {
+ pairedActions.add(action.spanId);
+ operations.push({
+ category,
+ span: {
+ ...span,
+ attributes: { ...action.attributes, ...span.attributes },
+ parentSpanId: action.parentSpanId,
+ statusCode: action.statusCode === 2 ? action.statusCode : span.statusCode,
+ statusMessage: action.statusCode === 2 ? action.statusMessage : span.statusMessage,
+ },
+ ...(action.attributes["agent.action.kind"] === "subagent-call"
+ ? { actionDurationMs: Number(action.endTimeNs - action.startTimeNs) / 1_000_000 }
+ : {}),
+ });
+ } else {
+ operations.push({ category, span });
+ }
+ }
+ return operations.filter(({ span }) => !pairedActions.has(span.spanId));
+}
+
+function categoryFor(span: LocalTraceSpan): LocalTraceOperation["category"] | undefined {
+ if (span.attributes["gen_ai.operation.name"] === "chat") return "model";
+ if (span.attributes["gen_ai.operation.name"] === "execute_tool") return "tool";
+ if (span.name === "agent.action") {
+ return span.attributes["agent.action.kind"] === "tool-call" ? "tool" : "other";
+ }
+ return undefined;
+}
+
+export function traceModelName(span: LocalTraceSpan): string | undefined {
+ return (
+ traceStringAttribute(span, "agent.model.id") ??
+ traceStringAttribute(span, "gen_ai.request.model")
+ );
+}
+
+export function traceToolName(span: LocalTraceSpan): string | undefined {
+ return (
+ traceStringAttribute(span, "gen_ai.tool.name") ??
+ traceStringAttribute(span, "agent.action.name")
+ );
+}
+
+export function traceSessionId(span: LocalTraceSpan): string | undefined {
+ return (
+ traceStringAttribute(span, "agent.run.id") ?? traceStringAttribute(span, "agent.session.id")
+ );
+}
+
+export function traceStringAttribute(span: LocalTraceSpan, key: string): string | undefined {
+ const value = span.attributes[key];
+ return typeof value === "string" && value.length > 0 ? value : undefined;
+}
diff --git a/packages/eve/src/tracing/local-trace-query.test.ts b/packages/eve/src/tracing/local-trace-query.test.ts
new file mode 100644
index 000000000..066b40762
--- /dev/null
+++ b/packages/eve/src/tracing/local-trace-query.test.ts
@@ -0,0 +1,85 @@
+import { describe, expect, it } from "vitest";
+
+import { queryLocalTraceSummaries } from "./local-trace-query.js";
+import { summarizeLocalTrace, type LocalTraceSummary } from "./local-trace-summary.js";
+
+const trace = (input: Partial<LocalTraceSummary>): LocalTraceSummary => ({
+ ...summarizeLocalTrace("a".repeat(32), []),
+ ...input,
+});
+
+describe("queryLocalTraceSummaries", () => {
+ it("filters structural metadata before ranking and limiting results", () => {
+ const result = queryLocalTraceSummaries(
+ [
+ trace({
+ agentNames: ["worker"],
+ errorSpanCount: 1,
+ inputTokens: 50,
+ sessionIds: ["session"],
+ toolNames: ["read"],
+ traceId: "b".repeat(32),
+ }),
+ trace({
+ agentNames: ["worker"],
+ errorSpanCount: 2,
+ inputTokens: 100,
+ sessionIds: ["session"],
+ toolNames: ["read"],
+ traceId: "a".repeat(32),
+ }),
+ trace({ agentNames: ["other"], errorSpanCount: 3, traceId: "c".repeat(32) }),
+ ],
+ {
+ agentName: "worker",
+ failedOnly: true,
+ limit: 1,
+ sessionId: "session",
+ sortBy: "failures",
+ toolName: "read",
+ },
+ );
+
+ expect(result).toEqual({
+ matches: [expect.objectContaining({ traceId: "a".repeat(32) })],
+ truncated: true,
+ });
+ });
+
+ it.each([
+ ["duration", { durationMs: 10 }],
+ ["failures", { errorSpanCount: 2 }],
+ ["inputTokens", { inputTokens: 100 }],
+ ["latest", { startedAt: "2026-01-01T00:00:00.000Z" }],
+ ] as const)("ranks %s descending without mutating summaries", (sortBy, fields) => {
+ const older = trace({ traceId: "a".repeat(32) });
+ const higher = trace({ traceId: "b".repeat(32), ...fields });
+ const summaries = [older, higher];
+ expect(queryLocalTraceSummaries(summaries, { limit: 2, sortBy })).toEqual({
+ matches: [higher, older],
+ truncated: false,
+ });
+ expect(summaries).toEqual([older, higher]);
+ });
+
+ it("filters for error spans, including traces that recovered", () => {
+ const healthy = trace({ traceId: "a".repeat(32) });
+ const recovered = trace({ traceId: "b".repeat(32), errorSpanCount: 1 });
+ expect(
+ queryLocalTraceSummaries([healthy, recovered], {
+ failedOnly: true,
+ limit: 2,
+ sortBy: "latest",
+ }),
+ ).toEqual({ matches: [recovered], truncated: false });
+ });
+
+ it("uses trace id as a deterministic ranking tie-breaker", () => {
+ const result = queryLocalTraceSummaries(
+ [trace({ traceId: "b".repeat(32) }), trace({ traceId: "a".repeat(32) })],
+ { limit: 2, sortBy: "duration" },
+ );
+
+ expect(result.matches.map(({ traceId }) => traceId)).toEqual(["a".repeat(32), "b".repeat(32)]);
+ });
+});
diff --git a/packages/eve/src/tracing/local-trace-query.ts b/packages/eve/src/tracing/local-trace-query.ts
new file mode 100644
index 000000000..9224fbd6b
--- /dev/null
+++ b/packages/eve/src/tracing/local-trace-query.ts
@@ -0,0 +1,46 @@
+import type { LocalTraceSummary } from "#tracing/local-trace-summary.js";
+
+export type LocalTraceSortBy = "duration" | "failures" | "inputTokens" | "latest";
+
+export interface LocalTraceQueryFilter {
+ readonly agentName?: string;
+ /** Selects traces containing error spans, not necessarily failed activations. */
+ readonly failedOnly?: boolean;
+ readonly sessionId?: string;
+ readonly toolName?: string;
+}
+
+/** Filters whole traces without restricting their totals to matching spans. */
+export function queryLocalTraceSummaries(
+ summaries: readonly LocalTraceSummary[],
+ options: LocalTraceQueryFilter & { readonly limit: number; readonly sortBy: LocalTraceSortBy },
+): { readonly matches: readonly LocalTraceSummary[]; readonly truncated: boolean } {
+ const matches = summaries.filter((summary) => matchesFilter(summary, options));
+ matches.sort((left, right) => compareSummaries(left, right, options.sortBy));
+ return { matches: matches.slice(0, options.limit), truncated: matches.length > options.limit };
+}
+
+function matchesFilter(summary: LocalTraceSummary, filter: LocalTraceQueryFilter): boolean {
+ return (
+ (filter.agentName === undefined || summary.agentNames.includes(filter.agentName)) &&
+ (filter.sessionId === undefined || summary.sessionIds.includes(filter.sessionId)) &&
+ (filter.toolName === undefined || summary.toolNames.includes(filter.toolName)) &&
+ (filter.failedOnly !== true || summary.errorSpanCount > 0)
+ );
+}
+
+function compareSummaries(
+ left: LocalTraceSummary,
+ right: LocalTraceSummary,
+ sortBy: LocalTraceSortBy,
+): number {
+ const difference =
+ sortBy === "duration"
+ ? right.durationMs - left.durationMs
+ : sortBy === "failures"
+ ? right.errorSpanCount - left.errorSpanCount
+ : sortBy === "inputTokens"
+ ? right.inputTokens - left.inputTokens
+ : Date.parse(right.startedAt) - Date.parse(left.startedAt);
+ return difference || left.traceId.localeCompare(right.traceId);
+}
diff --git a/packages/eve/src/tracing/local-trace-span-processor.ts b/packages/eve/src/tracing/local-trace-span-processor.ts
index a992871b4..cb2f33216 100644
--- a/packages/eve/src/tracing/local-trace-span-processor.ts
+++ b/packages/eve/src/tracing/local-trace-span-processor.ts
@@ -6,8 +6,10 @@ import type { SpanProcessor } from "#compiled/@vercel/otel/index.js";
import { createLogger, formatError } from "#internal/logging.js";
import { atomicWriteFile } from "#shared/atomic-write-file.js";
+import { indexLocalTraceConversation } from "#tracing/local-trace-discovery-index.js";
interface ReadableSpanLike {
+ readonly attributes: Readonly<Record<string, unknown>>;
readonly spanContext: () => { readonly spanId: string; readonly traceId: string };
}
@@ -25,6 +27,10 @@ export function resolveLocalTraceSchemaDirectory(appRoot: string): string {
return join(resolveLocalTraceStoreDirectory(appRoot), LOCAL_TRACE_SCHEMA_DIRECTORY);
}
+function resolveLocalTraceDirectory(appRoot: string, traceId: string): string {
+ return join(resolveLocalTraceSchemaDirectory(appRoot), traceId);
+}
+
/**
* Resolves the directory holding one trace's span segments.
*
@@ -32,7 +38,7 @@ export function resolveLocalTraceSchemaDirectory(appRoot: string): string {
* the trace last received a span.
*/
export function resolveLocalTraceSegmentsDirectory(appRoot: string, traceId: string): string {
- return join(resolveLocalTraceSchemaDirectory(appRoot), traceId, LOCAL_TRACE_SEGMENTS_DIRECTORY);
+ return join(resolveLocalTraceDirectory(appRoot, traceId), LOCAL_TRACE_SEGMENTS_DIRECTORY);
}
/** Persists spans from agent-owned traces as immutable OTLP/JSON segments. */
@@ -57,12 +63,20 @@ export class LocalTraceSpanProcessor implements SpanProcessor {
if (!isHexId(traceId, 32) || !isHexId(spanId, 16)) return;
const payload = JsonTraceSerializer.serializeRequest([span]);
if (payload === undefined) return;
-
+ const recordedConversationId = span.attributes["gen_ai.conversation.id"];
+ const conversationId =
+ typeof recordedConversationId === "string" && recordedConversationId.length > 0
+ ? recordedConversationId
+ : undefined;
this.#queue = this.#queue
.then(async () => {
- const directory = resolveLocalTraceSegmentsDirectory(this.#appRoot, traceId);
- await mkdir(directory, { recursive: true });
- await atomicWriteFile(join(directory, `${spanId}.otlp.json`), payload);
+ const traceDirectory = resolveLocalTraceDirectory(this.#appRoot, traceId);
+ const segmentsDirectory = resolveLocalTraceSegmentsDirectory(this.#appRoot, traceId);
+ await mkdir(segmentsDirectory, { recursive: true });
+ await atomicWriteFile(join(segmentsDirectory, `${spanId}.otlp.json`), payload);
+ if (conversationId !== undefined) {
+ await indexLocalTraceConversation({ conversationId, traceDirectory });
+ }
})
.catch((error: unknown) => {
if (!this.#reportedFailure) {
diff --git a/packages/eve/src/tracing/local-trace-summary.test.ts b/packages/eve/src/tracing/local-trace-summary.test.ts
new file mode 100644
index 000000000..862720037
--- /dev/null
+++ b/packages/eve/src/tracing/local-trace-summary.test.ts
@@ -0,0 +1,174 @@
+import { describe, expect, it } from "vitest";
+
+import type { LocalTraceSpan } from "#tracing/local-trace-reader.js";
+
+import { localTraceSpanCostUsd, summarizeLocalTrace } from "./local-trace-summary.js";
+
+const traceId = "1".repeat(32);
+
+function span(overrides: Partial<LocalTraceSpan> = {}): LocalTraceSpan {
+ return {
+ attributes: {},
+ endTimeNs: 20_000_000n,
+ events: [],
+ name: "agent.step",
+ spanId: "a".repeat(16),
+ startTimeNs: 10_000_000n,
+ statusCode: 0,
+ traceId,
+ ...overrides,
+ };
+}
+
+describe("summarizeLocalTrace", () => {
+ it("sums step usage once, ignoring duplicate segments and model counters", () => {
+ const step = span({
+ attributes: {
+ "agent.model.id": "gpt-5",
+ "agent.usage.input_tokens": 1000,
+ "agent.usage.output_tokens": 100,
+ "agent.usage.cache_read_tokens": 800,
+ "gen_ai.usage.cost": 0.01,
+ },
+ });
+ const summary = summarizeLocalTrace(traceId, [
+ step,
+ step,
+ span({
+ spanId: "b".repeat(16),
+ name: "ai.streamText.doStream",
+ attributes: {
+ "gen_ai.operation.name": "chat",
+ "agent.usage.input_tokens": 1000,
+ "agent.usage.output_tokens": 100,
+ "gen_ai.request.model": "gpt-5",
+ },
+ }),
+ span({
+ spanId: "c".repeat(16),
+ attributes: {
+ "agent.model.id": "claude-sonnet-4",
+ "agent.usage.input_tokens": "500",
+ "agent.usage.output_tokens": 50,
+ "gen_ai.usage.cache_creation.input_tokens": 25,
+ },
+ }),
+ ]);
+ expect(summary).toMatchObject({
+ inputTokens: 1500,
+ outputTokens: 150,
+ cacheReadTokens: 800,
+ cacheWriteTokens: 25,
+ models: ["claude-sonnet-4", "gpt-5"],
+ errorSpanCount: 0,
+ spanCount: 3,
+ modelCalls: 1,
+ });
+ expect(summary.costUsd).toBeCloseTo(0.01);
+ });
+
+ it("collapses tool operations but counts every error span and excludes non-tool actions", () => {
+ const spans = [
+ span({ statusCode: 2 }),
+ span({
+ spanId: "b".repeat(16),
+ name: "agent.action",
+ statusCode: 2,
+ attributes: { "agent.action.kind": "tool-call", "agent.action.name": "read" },
+ }),
+ span({
+ spanId: "c".repeat(16),
+ parentSpanId: "b".repeat(16),
+ name: "execute_tool read",
+ statusCode: 2,
+ attributes: { "gen_ai.operation.name": "execute_tool", "gen_ai.tool.name": "read" },
+ }),
+ span({
+ spanId: "d".repeat(16),
+ name: "agent.action",
+ attributes: { "agent.action.kind": "subagent-call", "agent.action.name": "worker" },
+ }),
+ ];
+ const summary = summarizeLocalTrace(traceId, spans);
+ expect(summary).toMatchObject({
+ errorSpanCount: 3,
+ toolCalls: 1,
+ toolNames: ["read"],
+ spanCount: 4,
+ });
+ expect(summary.costUsd).toBeUndefined();
+ });
+
+ it("derives stable identifiers and elapsed time independently of read order", () => {
+ const spans = [
+ span({
+ attributes: {
+ "agent.name": "worker",
+ "agent.run.id": "child",
+ "gen_ai.conversation.id": "conversation",
+ },
+ endTimeNs: 50_000_000n,
+ }),
+ span({
+ spanId: "b".repeat(16),
+ startTimeNs: 0n,
+ attributes: {
+ "agent.name": "root",
+ "agent.session.id": "parent",
+ "gen_ai.conversation.id": "conversation",
+ },
+ }),
+ ];
+ const summary = summarizeLocalTrace(traceId, spans);
+ expect(summary).toMatchObject({
+ traceId,
+ durationMs: 50,
+ startedAt: "1970-01-01T00:00:00.000Z",
+ agentNames: ["root", "worker"],
+ sessionIds: ["child", "parent"],
+ conversationIds: ["conversation"],
+ });
+ expect(summarizeLocalTrace(traceId, [...spans].reverse())).toEqual(summary);
+ });
+
+ it("ignores invalid counters and preserves unavailable cost", () => {
+ const summary = summarizeLocalTrace(traceId, [
+ span({
+ attributes: {
+ "agent.usage.input_tokens": Infinity,
+ "agent.usage.output_tokens": -1,
+ "agent.usage.cache_read_tokens": "NaN",
+ "agent.usage.cache_write_tokens": " ",
+ },
+ }),
+ ]);
+ expect(summary).toMatchObject({
+ inputTokens: 0,
+ outputTokens: 0,
+ cacheReadTokens: 0,
+ cacheWriteTokens: 0,
+ });
+ expect(summary.costUsd).toBeUndefined();
+ });
+
+ it("returns an empty summary for a trace with no readable spans", () => {
+ expect(summarizeLocalTrace(traceId, [])).toMatchObject({
+ traceId,
+ spanCount: 0,
+ durationMs: 0,
+ errorSpanCount: 0,
+ modelCalls: 0,
+ toolCalls: 0,
+ });
+ });
+});
+
+describe("localTraceSpanCostUsd", () => {
+ it("prefers gateway cost and accepts recorded numeric strings", () => {
+ expect(
+ localTraceSpanCostUsd(
+ span({ attributes: { "gen_ai.usage.gateway_cost": "0.0123", "gen_ai.usage.cost": 0.5 } }),
+ ),
+ ).toBe(0.0123);
+ });
+});
diff --git a/packages/eve/src/tracing/local-trace-summary.ts b/packages/eve/src/tracing/local-trace-summary.ts
new file mode 100644
index 000000000..5ba219cf8
--- /dev/null
+++ b/packages/eve/src/tracing/local-trace-summary.ts
@@ -0,0 +1,129 @@
+import type { LocalTraceSpan } from "#tracing/local-trace-reader.js";
+import { AGENT_USAGE_ATTRIBUTES } from "#tracing/agent-span-contract.js";
+import {
+ localTraceOperations,
+ traceModelName,
+ traceSessionId,
+ traceStringAttribute,
+ traceToolName,
+ uniqueLocalTraceSpans,
+} from "#tracing/local-trace-operations.js";
+
+/** Structural metadata and totals for one retained trace, not its final outcome. */
+export interface LocalTraceSummary {
+ readonly traceId: string;
+ readonly startedAt: string;
+ /** Elapsed time from the earliest span start to the latest span end. */
+ readonly durationMs: number;
+ readonly spanCount: number;
+ readonly agentNames: readonly string[];
+ readonly conversationIds: readonly string[];
+ readonly sessionIds: readonly string[];
+ readonly models: readonly string[];
+ readonly toolNames: readonly string[];
+ readonly modelCalls: number;
+ readonly toolCalls: number;
+ /** Error-bearing spans; one failure can be recorded on multiple ancestors. */
+ readonly errorSpanCount: number;
+ readonly inputTokens: number;
+ readonly outputTokens: number;
+ readonly cacheReadTokens: number;
+ readonly cacheWriteTokens: number;
+ /** Total gateway cost in USD; undefined when no step reported cost. */
+ readonly costUsd?: number;
+}
+
+/**
+ * Only agent.step spans contribute usage: model spans repeat the same counters.
+ * Operation counts collapse action/execution pairs, but error counts retain all
+ * error-bearing spans so summaries agree with the raw trace tree.
+ */
+export function summarizeLocalTrace(
+ traceId: string,
+ spans: readonly LocalTraceSpan[],
+): LocalTraceSummary {
+ const unique = uniqueLocalTraceSpans(spans);
+ const models = new Set<string>();
+ const agentNames = new Set<string>();
+ const conversationIds = new Set<string>();
+ const sessionIds = new Set<string>();
+ let start = unique[0]?.startTimeNs ?? 0n;
+ let end = start;
+ let cacheReadTokens = 0;
+ let cacheWriteTokens = 0;
+ let costUsd: number | undefined;
+ let errorSpanCount = 0;
+ let inputTokens = 0;
+ let outputTokens = 0;
+ for (const span of unique) {
+ if (span.startTimeNs < start) start = span.startTimeNs;
+ if (span.endTimeNs > end) end = span.endTimeNs;
+ add(models, traceModelName(span));
+ add(agentNames, traceStringAttribute(span, "agent.name"));
+ add(conversationIds, traceStringAttribute(span, "gen_ai.conversation.id"));
+ add(sessionIds, traceSessionId(span));
+ if (span.statusCode === 2) errorSpanCount += 1;
+ if (span.name !== "agent.step") continue;
+ inputTokens += numberAttribute(span, "agent.usage.input_tokens") ?? 0;
+ outputTokens += numberAttribute(span, "agent.usage.output_tokens") ?? 0;
+ cacheReadTokens +=
+ numberAttribute(span, AGENT_USAGE_ATTRIBUTES.cacheReadTokens) ??
+ numberAttribute(span, "gen_ai.usage.cache_read.input_tokens") ??
+ 0;
+ cacheWriteTokens +=
+ numberAttribute(span, AGENT_USAGE_ATTRIBUTES.cacheWriteTokens) ??
+ numberAttribute(span, "gen_ai.usage.cache_creation.input_tokens") ??
+ 0;
+ const cost = localTraceSpanCostUsd(span);
+ if (cost !== undefined) costUsd = (costUsd ?? 0) + cost;
+ }
+ const toolNames = new Set<string>();
+ let modelCalls = 0;
+ let toolCalls = 0;
+ for (const operation of localTraceOperations(unique)) {
+ if (operation.category === "model") modelCalls += 1;
+ if (operation.category === "tool") {
+ toolCalls += 1;
+ add(toolNames, traceToolName(operation.span));
+ }
+ }
+ return {
+ traceId,
+ startedAt: new Date(Number(start / 1_000_000n)).toISOString(),
+ durationMs: Number(end - start) / 1_000_000,
+ spanCount: unique.length,
+ agentNames: [...agentNames].sort(),
+ conversationIds: [...conversationIds].sort(),
+ sessionIds: [...sessionIds].sort(),
+ models: [...models].sort(),
+ toolNames: [...toolNames].sort(),
+ modelCalls,
+ toolCalls,
+ errorSpanCount,
+ cacheReadTokens,
+ cacheWriteTokens,
+ costUsd,
+ inputTokens,
+ outputTokens,
+ };
+}
+
+export function localTraceSpanCostUsd(span: LocalTraceSpan): number | undefined {
+ return (
+ numberAttribute(span, "gen_ai.usage.gateway_cost") ?? numberAttribute(span, "gen_ai.usage.cost")
+ );
+}
+
+function numberAttribute(span: LocalTraceSpan, key: string): number | undefined {
+ const value = span.attributes[key];
+ if (typeof value === "number") return Number.isFinite(value) && value >= 0 ? value : undefined;
+ if (typeof value === "string" && value.trim() !== "") {
+ const parsed = Number(value);
+ return Number.isFinite(parsed) && parsed >= 0 ? parsed : undefined;
+ }
+ return undefined;
+}
+
+function add(values: Set<string>, value: string | undefined): void {
+ if (value !== undefined) values.add(value);
+}