The Open Framework for Building Agents
fix(eve): preserve nested connector trace lineage and action timing (#4369)
Chad Hietaladd36ab36c045
diff --git a/.changeset/tidy-connector-traces.md b/.changeset/tidy-connector-traces.mdnew file mode 100644index 000000000..316e2dd2c--- /dev/null+++ b/.changeset/tidy-connector-traces.md@@ -0,0 +1,5 @@+---+"eve": patch+---++Fix nested connector actions appearing as unrelated model tool calls in traces: they now carry their parent call ID and nest beneath the calling action. Action spans also retain their request start time and cover tool execution when SDK telemetry arrives first.diff --git a/apps/fixtures/agent-tui-client/agent/connections/stub-mcp-user.ts b/apps/fixtures/agent-tui-client/agent/connections/stub-mcp-user.tsindex 6ec3a1b0c..6a4735c6d 100644--- a/apps/fixtures/agent-tui-client/agent/connections/stub-mcp-user.ts+++ b/apps/fixtures/agent-tui-client/agent/connections/stub-mcp-user.ts@@ -3,6 +3,7 @@ import { createHash, randomBytes } from "node:crypto";import {ConnectionAuthorizationFailedError,ConnectionAuthorizationRequiredError,+ defineDynamic,defineInteractiveAuthorization,defineMcpClientConnection,type McpClientConnectionDefinition,@@ -24,9 +25,8 @@ import {* Three env vars gate participation:** - `EVE_TEST_MCP_STUB_USER_AUTH` must be `"1"`. Without it the- * connection is inert (`auth` is omitted), so other smokes- * booting agent-tui-client see no extra behavior. The stub MCP URL- * still falls back to a sentinel that fails fast on first use.+ * connection is absent, so other smokes booting agent-tui-client+ * cannot discover or call a stub server they did not start.* - `EVE_TEST_MCP_STUB_URL` points at the in-process stub MCP* server the smoke starts up.* - `EVE_TEST_OAUTH_EMULATOR_URL` points at the@@ -71,6 +71,7 @@ const pendingTokenExchanges = new Map<string, Promise<string>>();const principalTokens = new Map<string, string>();const definition: McpClientConnectionDefinition = {+ instanceKey: "oauth-smoke",url,description:"Smoke-test stub MCP behind a real OAuth 2.1 + PKCE flow against the @emulators/microsoft IdP. Exposes the same echo_marker tool as stub-mcp.",@@ -160,7 +161,11 @@ if (userAuthEnabled) {});}-export default defineMcpClientConnection(definition);+export default defineDynamic({+ events: {+ "session.started": () => (userAuthEnabled ? defineMcpClientConnection(definition) : null),+ },+});async function exchangeAuthorizationCode(input: {readonly callbackUrl: string;diff --git a/docs/reference/cli.md b/docs/reference/cli.mdindex 3e3e69dff..d68f24fef 100644--- a/docs/reference/cli.md+++ b/docs/reference/cli.md@@ -397,6 +397,8 @@ Each workflow tool call has an `agent.action` span and an `execute_tool <tool>`Outbound MCP `tools/call` requests add MCP semantic attributes to the matching `execute_tool` span. When eve has no tool span to enrich, it creates a `CLIENT` `tools/call <tool>` span. Each `tools/list` discovery also has a `CLIENT` span. Configured OpenTelemetry propagation fields are injected into MCP `params._meta` for JSON-RPC bodies up to 1 MiB and propagation metadata up to 8 KiB; larger requests are sent unchanged. eve removes its audience and session-lineage baggage before forwarding, and keeps the input/output content-capture policy local.+`connection_execute` records the connector call as a child of its outer `agent.action` span, with `agent.action.parent_call_id` identifying the outer call. Both spans retain `agent.action.kind=tool-call`; consumers can distinguish nested calls by the parent call ID. The nested span's `gen_ai.tool.call.arguments` holds the validated connector input, including schema defaults, when input capture is enabled. Its duration covers the reporting of the nested result, not the connector request; use the outer call's `execute_tool` span for execution timing.+Each root agent turn starts a new trace. With local caller trace context, that trace includes the first turn of each local subagent it starts. Remote child turns and later local child turns start separate traces. If a worker is replaced, the new worker uses the prepared trace context for the same turn. Supply the conversation ID to show all related traces, oldest first.Every span carries a real duration. A turn's root `invoke_agent` span is written when the turn settles, so a running turn shows only its steps.diff --git a/packages/eve/src/instrumentation/lifecycle.ts b/packages/eve/src/instrumentation/lifecycle.tsindex 0ae9d6ad4..085037e5a 100644--- a/packages/eve/src/instrumentation/lifecycle.ts+++ b/packages/eve/src/instrumentation/lifecycle.ts@@ -500,6 +500,10 @@ export type InstrumentationToolCallTerminalEvent =*/export interface InstrumentationActionStartedEvent {readonly type: "action.started";+ /** Time the request was observed, before durable event acceptance. */+ readonly startedAtMs?: number;+ /** The calling action, when this is a nested tool call. */+ readonly parentCallId?: string;readonly callId: string;readonly idempotencyKey: string;/** Content. Absent unless this provider's trace policy records this direction. */diff --git a/packages/eve/src/instrumentation/native-events.test.ts b/packages/eve/src/instrumentation/native-events.test.tsindex c72373cd4..6c5a067d1 100644--- a/packages/eve/src/instrumentation/native-events.test.ts+++ b/packages/eve/src/instrumentation/native-events.test.ts@@ -1,5 +1,5 @@import { TEST_USAGE } from "#internal/testing/events.js";-import { describe, expect, it } from "vitest";+import { describe, expect, it, vi } from "vitest";import { ContextContainer, contextStorage } from "#context/container.js";import { ActiveChannelDeliveriesKey } from "#context/keys.js";@@ -286,12 +286,19 @@ describe("createInstrumentationHandleEvent", () => {await contextStorage.run(context, async () => {const handleEvent = createInstrumentationHandleEvent({getAttemptScope: () => scope,- handleEvent: async () => {},+ handleEvent: async () => {+ clock.mockReturnValue(2_000);+ },hooks: { capturesContent: true, publish: async (event) => void events.push(event) },sessionId: "session-1",})!;- await handleEvent(requested);- await handleEvent(requested);+ const clock = vi.spyOn(Date, "now").mockReturnValue(1_000);+ try {+ await handleEvent(requested);+ await handleEvent(requested);+ } finally {+ clock.mockRestore();+ }});const restored = await deserializeContext(await serializeContext(context));@@ -384,6 +391,7 @@ describe("createInstrumentationHandleEvent", () => {expect(events.slice(0, 5)).toEqual([{callId: "delegate-1",+ startedAtMs: 1_000,idempotencyKey: actionIdempotencyKey("session-1", "turn-1", "delegate-1"),input: { task: "research" },kind: "subagent-call",@@ -393,6 +401,7 @@ describe("createInstrumentationHandleEvent", () => {},{callId: "skill-1",+ startedAtMs: 1_000,idempotencyKey: actionIdempotencyKey("session-1", "turn-1", "skill-1"),input: { name: "research" },kind: "load-skill",@@ -402,6 +411,7 @@ describe("createInstrumentationHandleEvent", () => {},{callId: "remote-1",+ startedAtMs: 1_000,idempotencyKey: actionIdempotencyKey("session-1", "turn-1", "remote-1"),input: { task: "analyze" },kind: "remote-agent-call",@@ -411,6 +421,7 @@ describe("createInstrumentationHandleEvent", () => {},{callId: "add-1",+ startedAtMs: 1_000,idempotencyKey: actionIdempotencyKey("session-1", "turn-1", "add-1"),input: { a: 1, b: 2 },kind: "tool-call",@@ -420,6 +431,7 @@ describe("createInstrumentationHandleEvent", () => {},{callId: "workflow-1",+ startedAtMs: 1_000,idempotencyKey: actionIdempotencyKey("session-1", "turn-1", "workflow-1"),input: { report: "weekly" },isWorkflowTool: true,diff --git a/packages/eve/src/instrumentation/native-events.ts b/packages/eve/src/instrumentation/native-events.tsindex e2be70e40..ccf64c53d 100644--- a/packages/eve/src/instrumentation/native-events.ts+++ b/packages/eve/src/instrumentation/native-events.ts@@ -65,6 +65,7 @@ export function createInstrumentationHandleEvent(const publishedInputs = new Set<string>();let activeTurnId = input.turnId;return async (event, messages) => {+ const startedAtMs = Date.now();await handleEvent(event, messages);const lifecycleEvent = toLifecycleEvent(event, input, activeTurnId);if (event.type === "turn.started") activeTurnId = event.data.turnId;@@ -104,7 +105,7 @@ export function createInstrumentationHandleEvent(}if (lifecycleEvent !== undefined) await hooks.publish(lifecycleEvent);if (event.type === "actions.requested") {- await publishActionStarts(event, input, hooks, publishedActions);+ await publishActionStarts(event, input, hooks, publishedActions, startedAtMs);} else if (event.type === "action.result") {await publishActionTerminal(event, input, hooks);} else if (event.type === "input.requested") {@@ -196,6 +197,7 @@ async function publishActionStarts(input: CreateInstrumentationHandleEventInput,hooks: InstrumentationHooks,published: Set<string>,+ startedAtMs: number,): Promise<void> {const scope = input.getAttemptScope?.();if (scope === undefined) return;@@ -209,6 +211,10 @@ async function publishActionStarts(await hooks.publish(Object.freeze({callId: action.callId,+ ...(action.kind === "tool-call" && action.parentCallId !== undefined+ ? { parentCallId: action.parentCallId }+ : undefined),+ startedAtMs,idempotencyKey,input: capturesInputs ? action.input : undefined,...(isRuntimeWorkflowToolAction(action) ? { isWorkflowTool: true } : undefined),diff --git a/packages/eve/src/tracing/agent-action-instrumentation.ts b/packages/eve/src/tracing/agent-action-instrumentation.tsindex 90dadc980..185bafc9d 100644--- a/packages/eve/src/tracing/agent-action-instrumentation.ts+++ b/packages/eve/src/tracing/agent-action-instrumentation.ts@@ -57,11 +57,17 @@ export function createAgentActionInstrumentation(input: {event: InstrumentationActionStartedEvent,) => SpanContext | undefined | PromiseLike<SpanContext | undefined>;readonly stateStore: AgentTraceStateStore;+ readonly toolStartTimeForAction: (idempotencyKey: string) => number | undefined;readonly tracer: Tracer;}): AgentActionInstrumentation {const byAttempt = new Map<string, Set<string>>();const onStarted = async (event: InstrumentationActionStartedEvent): Promise<void> => {+ // SDK execution can precede acceptance of the streamed action request.+ const startTimeMs = Math.min(+ event.startedAtMs ?? Date.now(),+ input.toolStartTimeForAction(event.idempotencyKey) ?? Infinity,+ );const traceContext = await input.resolveTraceContext(event);if (traceContext === undefined || !isSampledTrace(traceContext)) return;@@ -74,15 +80,20 @@ export function createAgentActionInstrumentation(input: {kind: event.kind,name: event.name,parent: {- spanId: input.idGenerator.deriveSpanId(attemptIdempotencyKey(event.scope)),+ spanId: input.idGenerator.deriveSpanId(+ event.parentCallId === undefined+ ? attemptIdempotencyKey(event.scope)+ : `action:${actionIdempotencyKey(event.scope.sessionId, event.scope.turnId, event.parentCallId)}`,+ ),traceFlags: traceContext.traceFlags,traceId: traceContext.traceId,},+ parentCallId: event.parentCallId,rootSessionId: event.scope.rootSessionId ?? event.scope.sessionId,traceSessionId: traceSessionIdOf(event.scope),sessionId: event.scope.sessionId,spanId: input.idGenerator.deriveSpanId(`action:${event.idempotencyKey}`),- startTimeMs: Date.now(),+ startTimeMs,stepIndex: event.scope.stepIndex,turnId: event.scope.turnId,};@@ -115,6 +126,9 @@ export function createAgentActionInstrumentation(input: {attributes: {"agent.action.call_id": state.callId,"agent.action.kind": state.kind,+ ...(state.parentCallId === undefined+ ? undefined+ : { "agent.action.parent_call_id": state.parentCallId }),"agent.action.name": state.name,"agent.framework.name": "eve","agent.framework.version": input.frameworkVersion,diff --git a/packages/eve/src/tracing/agent-otel-provider.test.ts b/packages/eve/src/tracing/agent-otel-provider.test.tsindex 57eaed98f..5ae07e6ef 100644--- a/packages/eve/src/tracing/agent-otel-provider.test.ts+++ b/packages/eve/src/tracing/agent-otel-provider.test.ts@@ -2029,6 +2029,8 @@ describe("createAgentOtelInstrumentation", () => {scope,type: "tool.call.completed",});+ const later = Date.now() + 100;+ const clock = vi.spyOn(Date, "now").mockReturnValue(later);await runtime.hooks.publish({callId: "tool-1",idempotencyKey: actionKey,@@ -2038,6 +2040,7 @@ describe("createAgentOtelInstrumentation", () => {scope,type: "action.started",});+ clock.mockRestore();await runtime.hooks.publish({idempotencyKey: actionKey,outcome: "completed",@@ -2072,6 +2075,7 @@ describe("createAgentOtelInstrumentation", () => {const tool = byName(spans, "execute_tool weather")[0]!;const uncorrelatedTool = byName(spans, "execute_tool final_output")[0]!;expect(tool.parentSpanContext?.spanId).toBe(action.spanContext().spanId);+ expect(nanos(action.startTime)).toBeLessThanOrEqual(nanos(tool.startTime));// Seconds, as `@ai-sdk/otel` records it; a publisher that measured nothing sets none.expect(tool.attributes["gen_ai.execute_tool.duration"]).toBe(1.5);expect(uncorrelatedTool.attributes["gen_ai.execute_tool.duration"]).toBeUndefined();diff --git a/packages/eve/src/tracing/agent-otel-provider.ts b/packages/eve/src/tracing/agent-otel-provider.tsindex 94aaf70cb..558a5d30d 100644--- a/packages/eve/src/tracing/agent-otel-provider.ts+++ b/packages/eve/src/tracing/agent-otel-provider.ts@@ -130,6 +130,7 @@ export function createAgentOtelInstrumentation(return turn?.context;},stateStore: input.stateStore,+ toolStartTimeForAction: (idempotencyKey) => tools.startTimeForAction(idempotencyKey),tracer: input.tracer,});const approvals = createAgentApprovalInstrumentation({diff --git a/packages/eve/src/tracing/agent-telemetry-contract.integration.test.ts b/packages/eve/src/tracing/agent-telemetry-contract.integration.test.tsindex 3c09215fd..c8f36431d 100644--- a/packages/eve/src/tracing/agent-telemetry-contract.integration.test.ts+++ b/packages/eve/src/tracing/agent-telemetry-contract.integration.test.ts@@ -48,6 +48,12 @@ import { buildConversationItems } from "#cli/dev/tui/traces/trace-conversation.jimport { contentFilteringProcessor } from "#tracing/content-span-processor.js";import { ConversationContextKey } from "#shared/conversation-context.js";import { captureLogRecords } from "#internal/testing/log-records.js";+import { ConnectionRegistryKey } from "#context/providers/connection-key.js";+import { resolveConnectionTools } from "#execution/tools/connection-tools.js";+import { emitNestedToolActions } from "#harness/nested-actions.js";+import { createInstrumentationHandleEvent } from "#instrumentation/native-events.js";+import { createActionResultEvent, createActionsRequestedEvent } from "#protocol/message.js";+import type { JsonValue } from "#shared/json.js";const traceContext = (agentName: string, audience: "public" | "private") => ({agentName,@@ -126,6 +132,169 @@ function scopeFor(sessionId: string, audience: "public" | "private"): Instrument}describe("exported agent telemetry contract", () => {+ it.each(["public", "private"] as const)(+ "parents nested connector actions and preserves resolved arguments under the %s content policy",+ async (audience) => {+ const runtime = createRuntime();+ const ctx = contextFor(audience);+ const scope = scopeFor("parent", audience);+ const hooks = runtime.hooks.forTrace!(traceContext("parent", audience));+ const executeTool = vi.fn(async () => ({ structuredContent: { title: "Issue" } }));+ ctx.set(ConnectionRegistryKey, {+ dispose: async () => {},+ getConnectionApproval: () => undefined,+ getConnectionNames: () => ["linear"],+ getConnections: () => [+ {+ connectionName: "linear",+ protocol: "mcp",+ url: "https://linear.example.com/mcp",+ description: "Issues",+ sourceId: "linear",+ sourceKind: "module",+ logicalPath: "connections/linear",+ },+ ],+ getClient: () => ({+ close: async () => {},+ connect: async () => {},+ executeTool,+ getToolMetadata: async () => [+ {+ name: "get_issue",+ description: "Read an issue",+ inputSchema: {+ type: "object",+ properties: {+ id: { type: "string" },+ includeRelations: { type: "boolean", default: false },+ },+ required: ["id"],+ },+ },+ ],+ }),+ });+ const accepted = vi.fn(async () => {});+ const handleEvent = createInstrumentationHandleEvent({+ getAttemptScope: () => scope,+ handleEvent: accepted,+ hooks,+ sessionId: "parent",+ })!;+ const input = { connection: "linear", tool: "get_issue", input: { id: "ISSUE-1" } };+ const toolKey = toolCallIdempotencyKey(scope, "call-1", 0);+ try {+ await contextStorage.run(ctx, async () => {+ await bindInstrumentationRuntime(runtime, ctx, {+ agentName: "parent",+ rootSessionId: "parent",+ sessionId: "parent",+ })!.preparePreamble({ sequence: 0, sessionStarted: false, turnId: "turn_0" });+ await hooks.publish({+ type: "step.attempt.started",+ idempotencyKey: attemptIdempotencyKey(scope),+ scope,+ operation: { modelId: "test", operationId: "ai.streamText", provider: "test" },+ });+ await handleEvent(+ createActionsRequestedEvent({+ actions: [+ { callId: "call-1", kind: "tool-call", toolName: "connection_execute", input },+ ],+ sequence: 0,+ stepIndex: 0,+ turnId: "turn_0",+ }),+ );+ await hooks.publish({+ type: "tool.call.started",+ idempotencyKey: toolKey,+ callId: "call-1",+ input,+ toolName: "connection_execute",+ scope,+ });+ const output = await runtime.runInContext(+ { idempotencyKey: toolKey, scope, type: "tool.call" },+ () =>+ resolveConnectionTools()!.connection_execute!.execute!(input, {+ callId: "call-1",+ messages: [],+ } as never),+ );+ await hooks.publish({+ type: "tool.call.completed",+ idempotencyKey: toolKey,+ output: { type: "result", output },+ scope,+ });+ await emitNestedToolActions(+ handleEvent,+ { sequence: 0, stepIndex: 0, turnId: "turn_0" },+ "call-1",+ );+ await handleEvent(+ createActionResultEvent({+ result: {+ callId: "call-1",+ kind: "tool-result",+ toolName: "connection_execute",+ output: output as JsonValue,+ },+ sequence: 0,+ stepIndex: 0,+ turnId: "turn_0",+ }),+ );+ await hooks.publish({+ type: "step.attempt.completed",+ idempotencyKey: attemptIdempotencyKey(scope),+ scope,+ });+ });+ await runtime.forceFlush();+ expect(executeTool.mock.calls[0]).toEqual([+ "get_issue",+ { id: "ISSUE-1", includeRelations: false },+ { abortSignal: undefined, callId: "call-1" },+ ]);+ expect(accepted.mock.calls).toHaveLength(4);+ const actions = runtime.exporter+ .getFinishedSpans()+ .filter((span) => span.name === "agent.action");+ expect(actions).toHaveLength(2);+ const outer = actions.find((span) => span.attributes["agent.action.call_id"] === "call-1")!;+ const nested = actions.find(+ (span) => span.attributes["agent.action.call_id"] === "call-1:1",+ )!;+ expect(outer.attributes["agent.action.kind"]).toBe("tool-call");+ expect(nested.parentSpanContext?.spanId).toBe(outer.spanContext().spanId);+ expect(nested.attributes).toMatchObject({+ "agent.action.kind": "tool-call",+ "agent.action.parent_call_id": "call-1",+ "agent.action.name": "linear__get_issue",+ });+ const tool = runtime.exporter+ .getFinishedSpans()+ .find((span) => span.name === "execute_tool connection_execute")!;+ expect(tool.parentSpanContext?.spanId).toBe(outer.spanContext().spanId);+ expect(nested.attributes["gen_ai.tool.call.arguments"]).toBe(+ audience === "public" ? '{"id":"ISSUE-1","includeRelations":false}' : undefined,+ );+ expect(tool.attributes["gen_ai.tool.call.arguments"]).toBe(+ audience === "public" ? JSON.stringify(input) : undefined,+ );+ const metadataNested = runtime.metadata+ .getFinishedSpans()+ .find((span) => span.attributes["agent.action.call_id"] === "call-1:1")!;+ expect(metadataNested.attributes).not.toHaveProperty("gen_ai.tool.call.arguments");+ } finally {+ await runtime.shutdown();+ }+ },+ );+it("preserves explicit trace-session identity on every remote and local-child span", async () => {vi.stubEnv("VERCEL_ENV", "preview");const runtime = { ...createRuntime(), memoryOperations: true };diff --git a/packages/eve/src/tracing/agent-tool-instrumentation.ts b/packages/eve/src/tracing/agent-tool-instrumentation.tsindex ce81914be..db3b9ea10 100644--- a/packages/eve/src/tracing/agent-tool-instrumentation.ts+++ b/packages/eve/src/tracing/agent-tool-instrumentation.ts@@ -42,6 +42,7 @@ interface ToolSpanState {interface AgentToolInstrumentation {actionStarted(event: InstrumentationActionStartedEvent): Promise<void>;+ startTimeForAction(idempotencyKey: string): number | undefined;contextFor(attemptId: string, idempotencyKey: string): Context | undefined;drain(attemptId: string, failure?: { readonly error: unknown }): void;readonly events: {@@ -102,6 +103,7 @@ export function createAgentToolInstrumentation(input: {};return {+ startTimeForAction: (idempotencyKey) => byAction.get(idempotencyKey)?.startTimeMs,async actionStarted(event) {const state = byAction.get(event.idempotencyKey);if (state === undefined || state.span !== undefined || state.finished === true) return;diff --git a/packages/eve/src/tracing/agent-trace-context-codec.ts b/packages/eve/src/tracing/agent-trace-context-codec.tsindex decc521d7..a14af65d4 100644--- a/packages/eve/src/tracing/agent-trace-context-codec.ts+++ b/packages/eve/src/tracing/agent-trace-context-codec.ts@@ -177,6 +177,7 @@ function deserializeAction(value: unknown): AgentActionTraceState | undefined {kind: value.kind,name: value.name,parent: value.parent,+ parentCallId: typeof value.parentCallId === "string" ? value.parentCallId : undefined,rootSessionId: value.rootSessionId,traceSessionId: decodeTraceSessionId(value),sessionId: value.sessionId,diff --git a/packages/eve/src/tracing/agent-trace-state.ts b/packages/eve/src/tracing/agent-trace-state.tsindex caafd0bee..c694bb117 100644--- a/packages/eve/src/tracing/agent-trace-state.ts+++ b/packages/eve/src/tracing/agent-trace-state.ts@@ -61,6 +61,7 @@ export interface AgentActionTraceState {readonly kind: InstrumentationActionKind;readonly name: string;readonly parent: InstrumentationTraceContext;+ readonly parentCallId?: string;readonly rootSessionId: string;readonly sessionId: string;readonly spanId: string;