The Open Framework for Building Agents
feat(eve): add OpenTelemetry tracing for MCP calls (#3777)
Chad Hietala5b5f746cb3bc
diff --git a/.changeset/mcp-trace-propagation.md b/.changeset/mcp-trace-propagation.mdnew file mode 100644index 000000000..e36e4b6c4--- /dev/null+++ b/.changeset/mcp-trace-propagation.md@@ -0,0 +1,5 @@+---+"eve": patch+---++MCP connection calls now carry OpenTelemetry context in MCP metadata and add MCP semantic attributes to their tool traces. Tool payload capture continues to follow the configured trace content policy.diff --git a/docs/reference/cli.md b/docs/reference/cli.mdindex 0aaf62f9e..f92659d1d 100644--- a/docs/reference/cli.md+++ b/docs/reference/cli.md@@ -317,6 +317,8 @@ Every subagent activation starts its own trace. The first child's `invoke_agent`Each `agent()` call inside an authored workflow has its own `agent.action` caller span, including sequential, parallel, and background calls. A workflow tool that coordinates one of those calls has an enclosing `invoke_workflow <tool>` span; a workflow tool without agent calls remains `agent.action`. The tool also keeps its `execute_tool <tool>` span. Only agent execution uses `invoke_agent`.+Outbound MCP `tools/call` requests add MCP semantic attributes to the matching `execute_tool` span. When eve has no tool span to enrich, it creates a `CLIENT` `tools/call <tool>` span. Each `tools/list` discovery also has a `CLIENT` span. Configured OpenTelemetry propagation fields are injected into MCP `params._meta` for JSON-RPC bodies up to 1 MiB and propagation metadata up to 8 KiB; larger requests are sent unchanged. eve removes its audience and session-lineage baggage before forwarding, and keeps the input/output content-capture policy local.+A durable conversation produces one bounded trace per turn. Worker replacements reuse the prepared context for the same turn, while a later turn or an independently replayed attempt starts a fresh trace. Passing the conversation ID shows every trace it produced, oldest first.Every span carries a real duration. A turn's root `invoke_agent` span is written when the turn settles, so a running turn shows only its steps.diff --git a/packages/eve/src/runtime/connections/mcp-client.test.ts b/packages/eve/src/runtime/connections/mcp-client.test.tsindex a865e3e2c..2a597329d 100644--- a/packages/eve/src/runtime/connections/mcp-client.test.ts+++ b/packages/eve/src/runtime/connections/mcp-client.test.ts@@ -198,6 +198,7 @@ describe("McpConnectionClient", () => {expect(createMCPClient).toHaveBeenCalledWith({protocolVersionDiscovery: undefined,transport: {+ fetch: expect.any(Function),headers: {Authorization: "Bearer test-token","X-Api-Key": "key123",@@ -225,6 +226,7 @@ describe("McpConnectionClient", () => {expect(createMCPClient).toHaveBeenNthCalledWith(1, {protocolVersionDiscovery: undefined,transport: {+ fetch: expect.any(Function),headers: { Authorization: "Bearer test-token" },type: "http",url: "https://mcp.example.com",@@ -233,6 +235,7 @@ describe("McpConnectionClient", () => {expect(createMCPClient).toHaveBeenNthCalledWith(2, {protocolVersionDiscovery: undefined,transport: {+ fetch: expect.any(Function),headers: { Authorization: "Bearer test-token" },type: "sse",url: "https://mcp.example.com",@@ -258,6 +261,7 @@ describe("McpConnectionClient", () => {expect(createMCPClient).toHaveBeenNthCalledWith(2, {protocolVersionDiscovery: undefined,transport: {+ fetch: expect.any(Function),headers: { Authorization: "Bearer test-token" },type: "sse",url: "https://mcp.example.com",@@ -281,6 +285,7 @@ describe("McpConnectionClient", () => {expect(createMCPClient).toHaveBeenNthCalledWith(2, {protocolVersionDiscovery: undefined,transport: {+ fetch: expect.any(Function),headers: { Authorization: "Bearer test-token" },type: "sse",url: "https://mcp.example.com",@@ -306,6 +311,7 @@ describe("McpConnectionClient", () => {expect(createMCPClient).toHaveBeenNthCalledWith(2, {protocolVersionDiscovery: undefined,transport: {+ fetch: expect.any(Function),headers: { Authorization: "Bearer test-token" },type: "sse",url: "https://mcp.example.com",@@ -326,6 +332,7 @@ describe("McpConnectionClient", () => {expect(createMCPClient).toHaveBeenCalledWith({protocolVersionDiscovery: undefined,transport: {+ fetch: expect.any(Function),headers: { Authorization: "Bearer test-token" },type: "http",url: "https://mcp.example.com",diff --git a/packages/eve/src/runtime/connections/mcp-client.ts b/packages/eve/src/runtime/connections/mcp-client.tsindex 48fadbb7f..0cd887421 100644--- a/packages/eve/src/runtime/connections/mcp-client.ts+++ b/packages/eve/src/runtime/connections/mcp-client.ts@@ -12,6 +12,11 @@ import {omitProvidedArgumentsFromSchema,resolveProvidedArguments,} from "#runtime/connections/provided-arguments.js";+import {+ createMcpTraceFetch,+ withMcpToolCallSpan,+ withMcpToolsListSpan,+} from "#runtime/connections/mcp-tracing.js";import type {AuthorizationDefinition,ConnectionClient,@@ -73,11 +78,15 @@ export class McpConnectionClient implements ConnectionClient {async #createClient(): Promise<MCPClient> {const headers = await resolveHeaders(this.#connection);const url = this.#connection.url;+ const fetch = createMcpTraceFetch({+ connectionName: this.#connection.connectionName,+ getProtocolVersion: () => this.#client?.initializeResult?.protocolVersion,+ });try {return await createMCPClient({protocolVersionDiscovery: this.#connection.protocolVersionDiscovery,- transport: { type: "http", url, headers },+ transport: { fetch, headers, type: "http", url },});} catch (error) {if (!isMcpHttpFallbackRetryableError(error)) {@@ -85,7 +94,7 @@ export class McpConnectionClient implements ConnectionClient {}return await createMCPClient({protocolVersionDiscovery: this.#connection.protocolVersionDiscovery,- transport: { type: "sse", url, headers },+ transport: { fetch, headers, type: "sse", url },});}}@@ -127,6 +136,7 @@ export class McpConnectionClient implements ConnectionClient {`Tool "${toolName}" not found in connection "${this.#connection.connectionName}".`,);}+ const execute = sdkTool.execute;const resolvedArgs = await resolveProvidedArguments({args,@@ -135,7 +145,14 @@ export class McpConnectionClient implements ConnectionClient {toolName,});- return await sdkTool.execute(resolvedArgs, { abortSignal: options.abortSignal } as never);+ return await withMcpToolCallSpan({+ arguments: args,+ connectionName: this.#connection.connectionName,+ execute: async () =>+ await execute(resolvedArgs, { abortSignal: options.abortSignal } as never),+ protocolVersion: this.#client?.initializeResult?.protocolVersion,+ toolName,+ });} catch (error) {return await this.#rethrowClassified(error);}@@ -170,7 +187,11 @@ export class McpConnectionClient implements ConnectionClient {async #fetchToolsInner(): Promise<McpToolCache> {const client = await this.connect();- const listResult = await client.listTools();+ const listResult = await withMcpToolsListSpan({+ connectionName: this.#connection.connectionName,+ execute: () => client.listTools(),+ protocolVersion: client.initializeResult?.protocolVersion,+ });const filter = this.#connection.tools;const filteredTools =diff --git a/packages/eve/src/runtime/connections/mcp-tracing.test.ts b/packages/eve/src/runtime/connections/mcp-tracing.test.tsnew file mode 100644index 000000000..11d5185dc--- /dev/null+++ b/packages/eve/src/runtime/connections/mcp-tracing.test.ts@@ -0,0 +1,299 @@+import {+ context as apiContext,+ propagation as apiPropagation,+ trace as apiTrace,+} from "@opentelemetry/api";+import { AsyncLocalStorageContextManager } from "@opentelemetry/context-async-hooks";+import { W3CTraceContextPropagator } from "@opentelemetry/core";+import {+ BasicTracerProvider,+ InMemorySpanExporter,+ SimpleSpanProcessor,+} from "@opentelemetry/sdk-trace-base";+import {+ context as runtimeContext,+ ROOT_CONTEXT,+ SpanKind,+ trace as runtimeTrace,+} from "#compiled/@opentelemetry/api/index.js";+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";++import {+ createMcpTraceFetch,+ withMcpToolCallSpan,+ withMcpToolsListSpan,+} from "#runtime/connections/mcp-tracing.js";+import {+ withAgentToolContentPolicy,+ withAgentToolSpanContext,+} from "#tracing/agent-tool-span-context.js";++describe("MCP trace propagation", () => {+ let exporter: InMemorySpanExporter;+ let provider: BasicTracerProvider;++ beforeEach(() => {+ exporter = new InMemorySpanExporter();+ provider = new BasicTracerProvider({+ spanProcessors: [new SimpleSpanProcessor(exporter)],+ });+ apiContext.setGlobalContextManager(new AsyncLocalStorageContextManager().enable());+ apiPropagation.setGlobalPropagator(new W3CTraceContextPropagator());+ apiTrace.setGlobalTracerProvider(provider);+ });++ afterEach(async () => {+ await provider.shutdown();+ apiTrace.disable();+ apiPropagation.disable();+ apiContext.disable();+ });++ it("injects configured propagation into params._meta and annotates the tool span", async () => {+ const setAttributes = vi.fn();+ const toolContext = withAgentToolSpanContext(ROOT_CONTEXT, {+ recordInputs: false,+ recordOutputs: false,+ setAttributes,+ });+ const fetcher = vi.fn(+ async (_request: Parameters<typeof fetch>[0], _init?: Parameters<typeof fetch>[1]) =>+ new Response(null, { headers: { "mcp-session-id": "session-1" } }),+ );+ const fetch = createMcpTraceFetch({+ connectionName: "linear",+ fetcher,+ getActiveContext: () => toolContext,+ getProtocolVersion: () => "2025-11-25",+ injectContext(_context, carrier) {+ carrier.traceparent = `00-${"1".repeat(32)}-${"2".repeat(16)}-01`;+ carrier.tracestate = "vendor=value";+ carrier.baggage =+ "request=one,eve.audience=private;ceiling=i0o0,eve.conversation.id=hidden,eve.parent_session=%7B%7D";+ },+ });++ await fetch("https://mcp.example.com", {+ body: JSON.stringify({+ id: 42,+ jsonrpc: "2.0",+ method: "tools/call",+ params: {+ _meta: { caller: "eve" },+ arguments: { issue: "ISSUE-1" },+ name: "get_issue",+ },+ }),+ headers: { "mcp-protocol-version": "2025-11-25" },+ method: "POST",+ });++ const requestInit = fetcher.mock.calls[0]?.[1] as RequestInit | undefined;+ const requestBody = JSON.parse(String(requestInit?.body)) as {+ readonly params: { readonly _meta: Record<string, unknown> };+ };+ expect(requestBody.params._meta).toEqual({+ baggage: "request=one",+ caller: "eve",+ traceparent: `00-${"1".repeat(32)}-${"2".repeat(16)}-01`,+ tracestate: "vendor=value",+ });+ expect(requestBody.params._meta).not.toHaveProperty("recordInputs");+ expect(requestBody.params._meta).not.toHaveProperty("recordOutputs");+ expect(setAttributes).toHaveBeenCalledWith(+ expect.objectContaining({+ "eve.connection.name": "linear",+ "gen_ai.operation.name": "execute_tool",+ "gen_ai.tool.name": "get_issue",+ "jsonrpc.request.id": "42",+ "mcp.method.name": "tools/call",+ "mcp.protocol.version": "2025-11-25",+ "network.protocol.name": "http",+ "network.transport": "tcp",+ }),+ );+ expect(setAttributes).toHaveBeenCalledWith({ "mcp.session.id": "session-1" });+ });++ it("leaves oversized JSON-RPC requests untouched", async () => {+ const fetcher = vi.fn(+ async (_request: Parameters<typeof fetch>[0], _init?: Parameters<typeof fetch>[1]) =>+ new Response(null),+ );+ const fetch = createMcpTraceFetch({+ connectionName: "linear",+ fetcher,+ getActiveContext: () => ROOT_CONTEXT,+ getProtocolVersion: () => undefined,+ injectContext(_context, carrier) {+ carrier.traceparent = `00-${"1".repeat(32)}-${"2".repeat(16)}-01`;+ },+ });+ const body = JSON.stringify({+ id: 1,+ jsonrpc: "2.0",+ method: "tools/call",+ params: { arguments: { value: "x".repeat(1024 * 1024) }, name: "large" },+ });++ await fetch("https://mcp.example.com", { body, method: "POST" });++ const requestInit = fetcher.mock.calls[0]?.[1] as RequestInit | undefined;+ expect(requestInit?.body).toBe(body);+ });++ it("leaves requests untouched when custom propagation baggage is oversized", async () => {+ const fetcher = vi.fn(+ async (_request: Parameters<typeof fetch>[0], _init?: Parameters<typeof fetch>[1]) =>+ new Response(null),+ );+ const fetch = createMcpTraceFetch({+ connectionName: "linear",+ fetcher,+ getActiveContext: () => ROOT_CONTEXT,+ getProtocolVersion: () => undefined,+ injectContext(_context, carrier) {+ carrier.baggage = `vendor=${"x".repeat(8192)}`;+ },+ });+ const requestInit = {+ body: JSON.stringify({+ id: 1,+ jsonrpc: "2.0",+ method: "tools/call",+ params: { arguments: {}, name: "get_issue" },+ }),+ method: "POST",+ } satisfies RequestInit;++ await fetch("https://mcp.example.com", requestInit);++ expect(fetcher).toHaveBeenCalledOnce();+ expect(fetcher.mock.calls[0]?.[1]).toBe(requestInit);+ });++ it("propagates the fallback CLIENT span context for tools/call", async () => {+ const parent = runtimeTrace.getTracer("test.mcp").startSpan("parent");+ const parentContext = withAgentToolContentPolicy(runtimeTrace.setSpan(ROOT_CONTEXT, parent), {+ recordInputs: true,+ recordOutputs: true,+ });+ const fetcher = vi.fn(+ async (_request: Parameters<typeof fetch>[0], _init?: Parameters<typeof fetch>[1]) =>+ new Response(null, { headers: { "mcp-session-id": "session-1" } }),+ );+ const fetch = createMcpTraceFetch({+ connectionName: "linear",+ fetcher,+ getProtocolVersion: () => "2025-11-25",+ });+ const result = {+ content: [{ text: "Issue title", type: "text" }],+ isError: false,+ };++ await runtimeContext.with(parentContext, async () => {+ await withMcpToolCallSpan({+ arguments: { issue: "ISSUE-1" },+ connectionName: "linear",+ execute: async () => {+ await fetch("https://mcp.example.com", {+ body: JSON.stringify({+ id: 42,+ jsonrpc: "2.0",+ method: "tools/call",+ params: { arguments: { issue: "ISSUE-1" }, name: "get_issue" },+ }),+ headers: { "mcp-protocol-version": "2025-11-25" },+ method: "POST",+ });+ return result;+ },+ protocolVersion: "2025-11-25",+ toolName: "get_issue",+ });+ });+ parent.end();+ await provider.forceFlush();++ const call = exporter.getFinishedSpans().find((span) => span.name === "tools/call get_issue");+ expect(call).toBeDefined();+ expect(call!.kind).toBe(SpanKind.CLIENT);+ expect(call!.ended).toBe(true);+ expect(call!.parentSpanContext?.spanId).toBe(parent.spanContext().spanId);+ expect(call!.attributes).toMatchObject({+ "eve.connection.name": "linear",+ "gen_ai.operation.name": "execute_tool",+ "gen_ai.tool.call.arguments": '{"issue":"ISSUE-1"}',+ "gen_ai.tool.call.result":+ '{"content":[{"text":"Issue title","type":"text"}],"isError":false}',+ "gen_ai.tool.name": "get_issue",+ "mcp.method.name": "tools/call",+ "mcp.protocol.version": "2025-11-25",+ "mcp.session.id": "session-1",+ });+ expectInjectedSpanContext(fetcher, call!.spanContext());+ });++ it("propagates and records the tools/list CLIENT span", async () => {+ const parent = runtimeTrace.getTracer("test.mcp").startSpan("parent");+ const fetcher = vi.fn(+ async (_request: Parameters<typeof fetch>[0], _init?: Parameters<typeof fetch>[1]) =>+ new Response(null, { headers: { "mcp-session-id": "session-2" } }),+ );+ const fetch = createMcpTraceFetch({+ connectionName: "linear",+ fetcher,+ getProtocolVersion: () => "2025-11-25",+ });++ await runtimeContext.with(runtimeTrace.setSpan(ROOT_CONTEXT, parent), async () => {+ await withMcpToolsListSpan({+ connectionName: "linear",+ execute: async () =>+ await fetch("https://mcp.example.com", {+ body: JSON.stringify({+ id: "list-1",+ jsonrpc: "2.0",+ method: "tools/list",+ params: {},+ }),+ headers: { "mcp-protocol-version": "2025-11-25" },+ method: "POST",+ }),+ protocolVersion: "2025-11-25",+ });+ });+ parent.end();+ await provider.forceFlush();++ const list = exporter.getFinishedSpans().find((span) => span.name === "tools/list");+ expect(list).toBeDefined();+ expect(list!.kind).toBe(SpanKind.CLIENT);+ expect(list!.ended).toBe(true);+ expect(list!.parentSpanContext?.spanId).toBe(parent.spanContext().spanId);+ expect(list!.attributes).toMatchObject({+ "eve.connection.name": "linear",+ "jsonrpc.request.id": "list-1",+ "mcp.method.name": "tools/list",+ "mcp.protocol.version": "2025-11-25",+ "mcp.session.id": "session-2",+ });+ expectInjectedSpanContext(fetcher, list!.spanContext());+ });+});++function expectInjectedSpanContext(+ fetcher: ReturnType<typeof vi.fn>,+ spanContext: { readonly spanId: string; readonly traceId: string },+): void {+ const requestInit = fetcher.mock.calls[0]?.[1] as RequestInit | undefined;+ const requestBody = JSON.parse(String(requestInit?.body)) as {+ readonly params: { readonly _meta: Record<string, unknown> };+ };+ const traceparent = requestBody.params._meta["traceparent"];+ expect(typeof traceparent).toBe("string");+ const [, traceId, spanId] = String(traceparent).split("-");+ expect(traceId).toBe(spanContext.traceId);+ expect(spanId).toBe(spanContext.spanId);+}diff --git a/packages/eve/src/runtime/connections/mcp-tracing.ts b/packages/eve/src/runtime/connections/mcp-tracing.tsnew file mode 100644index 000000000..c5d74b74b--- /dev/null+++ b/packages/eve/src/runtime/connections/mcp-tracing.ts@@ -0,0 +1,380 @@+import {+ context as otelContext,+ createContextKey,+ propagation,+ SpanKind,+ trace,+ type Attributes,+ type Context,+ type Span,+ type TextMapSetter,+} from "#compiled/@opentelemetry/api/index.js";++import { contentAttribute } from "#tracing/agent-otel-content.js";+import { recordAgentSpanError } from "#tracing/agent-span-error.js";+import {+ agentToolContentPolicy,+ agentToolSpanContext,+ annotateAgentToolSpan,+ recordAgentToolSpanError,+ withAgentToolSpanContext,+} from "#tracing/agent-tool-span-context.js";+import { truncateTelemetryText } from "#tracing/telemetry-budget.js";+import { replaceBaggageMember } from "#protocol/baggage.js";+import { isObject } from "#shared/guards.js";++const MAX_MCP_TRACE_REQUEST_BYTES = 1024 * 1024;+const MAX_MCP_TRACE_CONTEXT_BYTES = 8192;+const utf8Encoder = new TextEncoder();+const EVE_TRACE_BAGGAGE_KEYS = [+ "eve.audience",+ "eve.conversation.id",+ "eve.parent_session",+] as const;+const traceContextSetter: TextMapSetter<Record<string, string>> = {+ set(carrier, key, value) {+ carrier[key] = value;+ },+};+const MCP_METHOD_NAME_CONTEXT_KEY = createContextKey("eve.mcp.method-name");++export function createMcpTraceFetch(input: {+ readonly connectionName: string;+ readonly fetcher?: typeof fetch;+ readonly getActiveContext?: () => Context;+ readonly getProtocolVersion: () => string | undefined;+ readonly injectContext?: McpTraceContextInjector;+}): typeof fetch {+ const fetcher = input.fetcher ?? ((request, init) => globalThis.fetch(request, init));+ const getActiveContext = input.getActiveContext ?? (() => otelContext.active());+ const injectContext = input.injectContext ?? injectOpenTelemetryTraceContext;++ return async (request, init) => {+ const message = readJsonRpcRequest(init?.body);+ if (message === undefined) return await fetcher(request, init);++ const activeContext = getActiveContext();+ const propagated = injectMcpTraceContext(message, activeContext, injectContext);+ const annotateRequest =+ message.method === "tools/call" || mcpMethodName(activeContext) === message.method;+ if (annotateRequest) {+ annotateAgentToolSpan(+ mcpRequestAttributes({+ connectionName: input.connectionName,+ message,+ params: isObject(message.params) ? message.params : {},+ protocolVersion:+ headerValue(init?.headers, "mcp-protocol-version") ?? input.getProtocolVersion(),+ }),+ activeContext,+ );+ }++ const response = await fetcher(+ request,+ propagated === undefined ? init : { ...init, body: JSON.stringify(propagated) },+ );+ if (annotateRequest) {+ const sessionId = response.headers.get("mcp-session-id");+ if (sessionId !== null) {+ annotateAgentToolSpan(+ { "mcp.session.id": truncateTelemetryText(sessionId, 512) },+ activeContext,+ );+ }+ }+ return response;+ };+}++export function injectMcpTraceContext(+ message: JsonRpcRequest,+ activeContext: Context,+ injectContext: McpTraceContextInjector = injectOpenTelemetryTraceContext,+): JsonRpcRequest | undefined {+ if (!withinTraceRequestLimit(JSON.stringify(message))) return undefined;++ if (message.params !== undefined && !isObject(message.params)) return undefined;+ const propagated: Record<string, string> = {};+ injectContext(activeContext, propagated);+ if (!withinTraceContextLimit(propagated)) return undefined;+ stripEveTraceBaggage(propagated);++ const params = isObject(message.params) ? message.params : {};+ const existingMeta = isObject(params["_meta"]) ? params["_meta"] : {};+ return {+ ...message,+ params: {+ ...params,+ _meta: { ...existingMeta, ...propagated },+ },+ };+}++export async function withMcpToolsListSpan<T>(input: {+ readonly connectionName: string;+ readonly execute: () => Promise<T>;+ readonly protocolVersion?: string;+}): Promise<T> {+ const parent = otelContext.active();+ const attributes = mcpRequestAttributes({+ connectionName: input.connectionName,+ method: "tools/list",+ protocolVersion: input.protocolVersion,+ });+ const span = trace.getTracer("eve.mcp").startSpan(+ "tools/list",+ {+ attributes,+ kind: SpanKind.CLIENT,+ },+ parent,+ );+ const spanContext = withMcpMethodName(+ withAgentToolSpanContext(trace.setSpan(parent, span), {+ ...agentToolContentPolicy(parent),+ recordError: (error, errorType) => recordAgentSpanError(span, error, errorType),+ setAttributes: (spanAttributes) => setSpanAttributes(span, spanAttributes),+ }),+ "tools/list",+ );+ try {+ return await otelContext.with(spanContext, async () => {+ try {+ return await input.execute();+ } catch (error) {+ const code = jsonRpcErrorCode(error);+ if (code !== undefined) {+ annotateAgentToolSpan({ "rpc.response.status_code": code }, spanContext);+ }+ recordAgentToolSpanError(error, errorType(error, code), spanContext);+ throw error;+ }+ });+ } finally {+ span.end();+ }+}++export async function withMcpToolCallSpan<T>(input: {+ readonly arguments: unknown;+ readonly connectionName: string;+ readonly execute: () => Promise<T>;+ readonly protocolVersion?: string;+ readonly toolName: string;+}): Promise<T> {+ const parent = otelContext.active();+ const existing = agentToolSpanContext(parent);+ const policy = agentToolContentPolicy(parent);+ if (existing?.setAttributes !== undefined) {+ annotateAgentToolSpan(mcpToolCallAttributes(input), parent);+ return await runMcpToolCall(input, parent, undefined, policy);+ }++ const span = trace.getTracer("eve.mcp").startSpan(+ `tools/call ${truncateTelemetryText(input.toolName, 128)}`,+ {+ attributes: mcpToolCallAttributes(input),+ kind: SpanKind.CLIENT,+ },+ parent,+ );+ const spanContext = withMcpMethodName(+ withAgentToolSpanContext(trace.setSpan(parent, span), {+ ...policy,+ recordError: (error, errorType) => recordAgentSpanError(span, error, errorType),+ setAttributes: (attributes) => setSpanAttributes(span, attributes),+ }),+ "tools/call",+ );++ try {+ return await otelContext.with(spanContext, () =>+ runMcpToolCall(input, spanContext, span, policy),+ );+ } finally {+ span.end();+ }+}++function runMcpToolCall<T>(+ input: {+ readonly arguments: unknown;+ readonly execute: () => Promise<T>;+ },+ context: Context,+ fallbackSpan: Span | undefined,+ policy: ReturnType<typeof agentToolContentPolicy>,+): Promise<T> {+ if (fallbackSpan !== undefined && policy.recordInputs) {+ const argumentsAttribute = contentAttribute(input.arguments);+ if (argumentsAttribute !== undefined) {+ fallbackSpan.setAttribute("gen_ai.tool.call.arguments", argumentsAttribute);+ }+ }++ return input+ .execute()+ .then((result) => {+ if (isToolErrorResult(result)) {+ recordAgentToolSpanError(undefined, "tool_error", context);+ }+ if (fallbackSpan !== undefined && policy.recordOutputs) {+ const resultAttribute = contentAttribute(result);+ if (resultAttribute !== undefined) {+ fallbackSpan.setAttribute("gen_ai.tool.call.result", resultAttribute);+ }+ }+ return result;+ })+ .catch((error: unknown) => {+ const code = jsonRpcErrorCode(error);+ if (code !== undefined) {+ annotateAgentToolSpan({ "rpc.response.status_code": code }, context);+ }+ if (fallbackSpan !== undefined) {+ recordAgentToolSpanError(error, errorType(error, code), context);+ }+ throw error;+ });+}++function mcpToolCallAttributes(input: {+ readonly connectionName: string;+ readonly protocolVersion?: string;+ readonly toolName: string;+}): Attributes {+ return mcpRequestAttributes({+ connectionName: input.connectionName,+ method: "tools/call",+ protocolVersion: input.protocolVersion,+ toolName: input.toolName,+ });+}++function mcpRequestAttributes(input: {+ readonly connectionName: string;+ readonly message?: JsonRpcRequest;+ readonly method?: string;+ readonly params?: Record<string, unknown>;+ readonly protocolVersion?: string;+ readonly toolName?: string;+}): Attributes {+ const method = input.method ?? input.message?.method;+ const params = input.params ?? (isObject(input.message?.params) ? input.message.params : {});+ const attributes: Attributes = {+ "eve.connection.name": input.connectionName,+ "mcp.method.name": method ?? "unknown",+ "network.protocol.name": "http",+ "network.transport": "tcp",+ };+ const toolName =+ input.toolName ?? (typeof params["name"] === "string" ? params["name"] : undefined);+ if (method === "tools/call") {+ attributes["gen_ai.operation.name"] = "execute_tool";+ if (toolName !== undefined) attributes["gen_ai.tool.name"] = toolName;+ }+ const requestId = input.message?.id;+ if (typeof requestId === "number" || typeof requestId === "string") {+ attributes["jsonrpc.request.id"] = String(requestId);+ }+ const meta = params["_meta"];+ const protocolVersion =+ input.protocolVersion ??+ (isObject(meta) && typeof meta["io.modelcontextprotocol/protocolVersion"] === "string"+ ? meta["io.modelcontextprotocol/protocolVersion"]+ : undefined);+ if (protocolVersion !== undefined) {+ attributes["mcp.protocol.version"] = protocolVersion;+ }+ return attributes;+}++interface JsonRpcRequest extends Record<string, unknown> {+ readonly method: string;+ readonly params?: unknown;+}++export type McpTraceContextInjector = (context: Context, carrier: Record<string, string>) => void;++function injectOpenTelemetryTraceContext(context: Context, carrier: Record<string, string>): void {+ propagation.inject(context, carrier, traceContextSetter);+}++function stripEveTraceBaggage(carrier: Record<string, string>): void {+ let baggage = carrier["baggage"];+ if (baggage === undefined) return;+ for (const key of EVE_TRACE_BAGGAGE_KEYS) {+ baggage = replaceBaggageMember(baggage, key, undefined);+ }+ if (baggage === undefined) delete carrier["baggage"];+ else carrier["baggage"] = baggage;+}++function readJsonRpcRequest(body: RequestInit["body"]): JsonRpcRequest | undefined {+ if (typeof body !== "string" || !withinTraceRequestLimit(body)) return undefined;+ try {+ const value: unknown = JSON.parse(body);+ return isObject(value) && typeof value["method"] === "string"+ ? (value as JsonRpcRequest)+ : undefined;+ } catch {+ return undefined;+ }+}++function withinTraceRequestLimit(value: string): boolean {+ return (+ value.length <= MAX_MCP_TRACE_REQUEST_BYTES &&+ utf8Encoder.encode(value).byteLength <= MAX_MCP_TRACE_REQUEST_BYTES+ );+}++function withinTraceContextLimit(carrier: Readonly<Record<string, string>>): boolean {+ let bytes = 0;+ for (const [key, value] of Object.entries(carrier)) {+ if (key.length + value.length > MAX_MCP_TRACE_CONTEXT_BYTES) return false;+ bytes += utf8Encoder.encode(key).byteLength + utf8Encoder.encode(value).byteLength + 2;+ if (bytes > MAX_MCP_TRACE_CONTEXT_BYTES) return false;+ }+ return bytes > 0;+}++function headerValue(headers: RequestInit["headers"], name: string): string | undefined {+ if (headers === undefined) return undefined;+ try {+ return new Headers(headers).get(name) ?? undefined;+ } catch {+ return undefined;+ }+}++function isToolErrorResult(value: unknown): boolean {+ return isObject(value) && value["isError"] === true;+}++function jsonRpcErrorCode(error: unknown): string | undefined {+ if (!isObject(error)) return undefined;+ const code = error["code"];+ return typeof code === "number" || typeof code === "string" ? String(code) : undefined;+}++function errorType(error: unknown, code: string | undefined): string | undefined {+ if (code !== undefined) return code;+ return error instanceof Error ? error.name || "Error" : undefined;+}++function setSpanAttributes(span: Span, attributes: Attributes): void {+ for (const [name, value] of Object.entries(attributes)) {+ if (value !== undefined) span.setAttribute(name, value);+ }+}++function withMcpMethodName(context: Context, methodName: string): Context {+ return context.setValue(MCP_METHOD_NAME_CONTEXT_KEY, methodName);+}++function mcpMethodName(context: Context): string | undefined {+ const methodName = context.getValue(MCP_METHOD_NAME_CONTEXT_KEY);+ return typeof methodName === "string" ? methodName : undefined;+}diff --git a/packages/eve/src/tracing/agent-otel-provider.test.ts b/packages/eve/src/tracing/agent-otel-provider.test.tsindex 63d8c26de..62b7adf9b 100644--- a/packages/eve/src/tracing/agent-otel-provider.test.ts+++ b/packages/eve/src/tracing/agent-otel-provider.test.ts@@ -49,6 +49,11 @@ import {} from "#instrumentation/lifecycle.js";import type { ChannelAudience } from "#shared/channel-audience.js";import { channelAudienceFromContext } from "#tracing/channel-audience-context.js";+import {+ agentToolContentPolicy,+ agentToolSpanContext,+ annotateAgentToolSpan,+} from "#tracing/agent-tool-span-context.js";import { contentFilteringProcessor } from "#tracing/content-span-processor.js";import { parseLocalTraceSegment } from "#tracing/local-trace-reader.js";import { CONTENT_ATTRIBUTE_LIMIT } from "#tracing/agent-otel-content.js";@@ -2532,6 +2537,111 @@ describe("createAgentOtelInstrumentation", () => {withSpy.mockRestore();});+ it("enriches the existing tool span with MCP attributes under its content policy", async () => {+ const runtime = createRuntime(new InMemoryAgentTraceStateStore(), () => ({+ emit: true,+ recordInputs: false,+ recordOutputs: false,+ }));+ const scope: InstrumentationAttemptScope = {+ attemptId: "session-1:turn-1:0:0",+ attemptIndex: 0,+ channelAudience: "private",+ functionId: "weather",+ sessionId: "session-1",+ stepIndex: 0,+ turnId: "turn-1",+ };+ const actionKey = actionIdempotencyKey(scope.sessionId, scope.turnId, "tool-1");+ const toolKey = "tool:session-1:turn-1:tool-1:0";++ await publishTurnStarted({+ channelAudience: "private",+ hooks: runtime.hooks,+ sessionId: scope.sessionId,+ turnId: scope.turnId,+ turnSequence: 0,+ });+ await runtime.hooks.publish({+ idempotencyKey: attemptIdempotencyKey(scope),+ operation: { modelId: "model", operationId: "ai.streamText", provider: "test" },+ scope,+ type: "step.attempt.started",+ });+ await runtime.hooks.publish({+ callId: "tool-1",+ idempotencyKey: actionKey,+ input: { issue: "ISSUE-1" },+ kind: "tool-call",+ name: "linear__get_issue",+ scope,+ type: "action.started",+ });+ await runtime.hooks.publish({+ callId: "tool-1",+ idempotencyKey: toolKey,+ input: { issue: "ISSUE-1" },+ scope,+ toolName: "linear__get_issue",+ type: "tool.call.started",+ });++ const withSpy = vi.spyOn(context, "with");+ await runtime.runInContext({ idempotencyKey: toolKey, scope, type: "tool.call" }, async () => {+ const active = withSpy.mock.calls[0]?.[0];+ expect(active).toBeDefined();+ expect(agentToolContentPolicy(active!)).toEqual({+ recordInputs: false,+ recordOutputs: false,+ });+ annotateAgentToolSpan(+ {+ "eve.connection.name": "linear",+ "gen_ai.operation.name": "execute_tool",+ "gen_ai.tool.name": "get_issue",+ "jsonrpc.request.id": "7",+ "mcp.method.name": "tools/call",+ },+ active!,+ );+ });+ withSpy.mockRestore();++ await runtime.hooks.publish({+ idempotencyKey: toolKey,+ output: { output: { title: "private issue" }, type: "result" },+ scope,+ type: "tool.call.completed",+ });+ await runtime.hooks.publish({+ idempotencyKey: actionKey,+ outcome: "completed",+ output: { output: { title: "private issue" }, type: "result" },+ scope,+ type: "action.completed",+ });+ await runtime.hooks.publish({+ idempotencyKey: attemptIdempotencyKey(scope),+ scope,+ type: "step.attempt.completed",+ });+ await completeTurn(runtime.hooks, scope.sessionId, scope.turnId);+ await runtime.provider.forceFlush();++ const spans = runtime.exporter.getFinishedSpans();+ const tool = byName(spans, "execute_tool linear__get_issue")[0]!;+ expect(tool.attributes).toMatchObject({+ "eve.connection.name": "linear",+ "gen_ai.operation.name": "execute_tool",+ "gen_ai.tool.name": "get_issue",+ "jsonrpc.request.id": "7",+ "mcp.method.name": "tools/call",+ });+ expect(tool.attributes).not.toHaveProperty("gen_ai.tool.call.arguments");+ expect(tool.attributes).not.toHaveProperty("gen_ai.tool.call.result");+ expect(spans.filter((span) => span.name.startsWith("tools/call "))).toHaveLength(0);+ });+it("does not create approval spans for other input requests", async () => {const runtime = createRuntime();const scope: InstrumentationAttemptScope = {@@ -3040,6 +3150,9 @@ describe("createAgentOtelInstrumentation", () => {);expect(withSpy).toHaveBeenCalledOnce();+ const active = withSpy.mock.calls[0]?.[0];+ expect(active).toBeDefined();+ expect(agentToolSpanContext(active!)).toBeUndefined();withSpy.mockRestore();});diff --git a/packages/eve/src/tracing/agent-otel-provider.ts b/packages/eve/src/tracing/agent-otel-provider.tsindex 8ef8abfcd..652515a21 100644--- a/packages/eve/src/tracing/agent-otel-provider.ts+++ b/packages/eve/src/tracing/agent-otel-provider.ts@@ -75,6 +75,7 @@ import {type AgentSamplingOperation,} from "#tracing/agent-span-contract.js";import { withErrorContent } from "#tracing/error-content-context.js";+import { withAgentToolContentPolicy } from "#tracing/agent-tool-span-context.js";import { recordAgentSpanError as recordError } from "#tracing/agent-span-error.js";import { resolveInstrumentationEnvironment } from "#internal/application/dev-environment.js";import type { ConversationEnvironment } from "#shared/conversation-context.js";@@ -600,17 +601,17 @@ export function createAgentOtelInstrumentation(seed?.forwardedTracePolicy,environment,);- return parent === undefined- ? execute()- : context.with(- markAgentTraceContext(- withErrorContent(- parent,- recordOutputs && effective?.action === "record" && effective.recordOutputs,- ),- ),- execute,- );+ const toolContentPolicy = {+ recordInputs: recordInputs && effective?.action === "record" && effective.recordInputs,+ recordOutputs: recordOutputs && effective?.action === "record" && effective.recordOutputs,+ };+ if (parent === undefined) return execute();+ const withErrorPolicy = withErrorContent(parent, toolContentPolicy.recordOutputs);+ const operationContext =+ operation.type === "tool.call"+ ? withAgentToolContentPolicy(withErrorPolicy, toolContentPolicy)+ : withErrorPolicy;+ return context.with(markAgentTraceContext(operationContext), execute);},};diff --git a/packages/eve/src/tracing/agent-tool-instrumentation.ts b/packages/eve/src/tracing/agent-tool-instrumentation.tsindex f9c17edbb..49fa621e5 100644--- a/packages/eve/src/tracing/agent-tool-instrumentation.ts+++ b/packages/eve/src/tracing/agent-tool-instrumentation.ts@@ -2,6 +2,7 @@ import {ROOT_CONTEXT,SpanKind,type Context,+ type Attributes,type Span,type SpanContext,type Tracer,@@ -21,11 +22,13 @@ import { withChannelAudience } from "#tracing/channel-audience-context.js";import type { AgentSpanIdGenerator } from "#tracing/agent-span-id-generator.js";import type { AgentActionContext } from "#tracing/agent-action-instrumentation.js";import { recordAgentSpanError as recordError } from "#tracing/agent-span-error.js";+import { withAgentToolSpanContext } from "#tracing/agent-tool-span-context.js";interface ToolSpanState {readonly actionKey: string;readonly attemptId: string;- readonly context: Context;+ context: Context;+ readonly additionalAttributes: Attributes;readonly event: InstrumentationToolCallStartedEvent;readonly fallbackParent: Context;readonly idempotencyKey: string;@@ -34,6 +37,7 @@ interface ToolSpanState {finished?: true;span?: Span;terminal?: InstrumentationToolCallTerminalEvent;+ pendingError?: { readonly error: unknown; readonly errorType?: string };}export interface AgentToolInstrumentation {@@ -147,6 +151,7 @@ export function createAgentToolInstrumentation(input: {const state: ToolSpanState = {actionKey,attemptId: event.scope.attemptId,+ additionalAttributes: {},context: withChannelAudience(contextFromSpanContext({isRemote: false,@@ -162,6 +167,21 @@ export function createAgentToolInstrumentation(input: {spanId,startTimeMs: Date.now(),};+ state.context = withAgentToolSpanContext(state.context, {+ recordInputs: input.recordInputs,+ recordOutputs: input.recordOutputs,+ setAttributes(attributes) {+ Object.assign(state.additionalAttributes, attributes);+ if (state.span === undefined) return;+ for (const [name, value] of Object.entries(attributes)) {+ if (value !== undefined) state.span.setAttribute(name, value);+ }+ },+ recordError(error, errorType) {+ state.pendingError = { error, errorType };+ if (state.span !== undefined) recordError(state.span, error, errorType);+ },+ });getAttemptStates(event.scope.attemptId).set(event.idempotencyKey, state);byAction.set(actionKey, state);return state;@@ -173,13 +193,16 @@ export function createAgentToolInstrumentation(input: {input.tracer.startSpan(`execute_tool ${state.event.toolName}`,{- attributes: toolAttributes(state.event),+ attributes: { ...toolAttributes(state.event), ...state.additionalAttributes },kind: SpanKind.INTERNAL,startTime: state.startTimeMs,},parent,),);+ if (state.pendingError !== undefined) {+ recordError(state.span, state.pendingError.error, state.pendingError.errorType);+ }if (input.recordInputs) {const args = contentAttribute(state.event.input);if (args !== undefined) state.span.setAttribute("gen_ai.tool.call.arguments", args);diff --git a/packages/eve/src/tracing/agent-tool-span-context.test.ts b/packages/eve/src/tracing/agent-tool-span-context.test.tsnew file mode 100644index 000000000..b9227c111--- /dev/null+++ b/packages/eve/src/tracing/agent-tool-span-context.test.ts@@ -0,0 +1,37 @@+import { ROOT_CONTEXT } from "#compiled/@opentelemetry/api/index.js";+import { describe, expect, it, vi } from "vitest";++import {+ agentToolContentPolicy,+ annotateAgentToolSpan,+ recordAgentToolSpanError,+ withAgentToolContentPolicy,+ withAgentToolSpanContext,+} from "#tracing/agent-tool-span-context.js";++describe("agent tool span context", () => {+ it("narrows the capture policy without dropping the span writer", () => {+ const setAttributes = vi.fn();+ const recordError = vi.fn();+ const context = withAgentToolSpanContext(ROOT_CONTEXT, {+ recordError,+ recordInputs: true,+ recordOutputs: true,+ setAttributes,+ });+ const narrowed = withAgentToolContentPolicy(context, {+ recordInputs: false,+ recordOutputs: false,+ });++ expect(agentToolContentPolicy(narrowed)).toEqual({+ recordInputs: false,+ recordOutputs: false,+ });+ expect(annotateAgentToolSpan({ "mcp.method.name": "tools/call" }, narrowed)).toBe(true);+ recordAgentToolSpanError(new Error("private result"), "tool_error", narrowed);++ expect(setAttributes).toHaveBeenCalledWith({ "mcp.method.name": "tools/call" });+ expect(recordError).toHaveBeenCalledWith(undefined, "tool_error");+ });+});diff --git a/packages/eve/src/tracing/agent-tool-span-context.ts b/packages/eve/src/tracing/agent-tool-span-context.tsnew file mode 100644index 000000000..1203f20a5--- /dev/null+++ b/packages/eve/src/tracing/agent-tool-span-context.ts@@ -0,0 +1,73 @@+import {+ context as otelContext,+ createContextKey,+ type Attributes,+ type Context,+} from "#compiled/@opentelemetry/api/index.js";++export interface AgentToolContentPolicy {+ readonly recordInputs: boolean;+ readonly recordOutputs: boolean;+}++export interface AgentToolSpanContext extends AgentToolContentPolicy {+ readonly recordError?: (error: unknown, errorType?: string) => void;+ readonly setAttributes?: (attributes: Attributes) => void;+}++const AGENT_TOOL_SPAN_CONTEXT_KEY = createContextKey("eve.agent.tool-span-context");+const NO_CONTENT_POLICY: AgentToolContentPolicy = { recordInputs: false, recordOutputs: false };++export function withAgentToolSpanContext(context: Context, value: AgentToolSpanContext): Context {+ return context.setValue(AGENT_TOOL_SPAN_CONTEXT_KEY, value);+}++export function withAgentToolContentPolicy(+ context: Context,+ policy: AgentToolContentPolicy,+): Context {+ const current = agentToolSpanContext(context);+ return context.setValue(AGENT_TOOL_SPAN_CONTEXT_KEY, {+ ...current,+ ...policy,+ } satisfies AgentToolSpanContext);+}++export function agentToolSpanContext(+ context: Context = otelContext.active(),+): AgentToolSpanContext | undefined {+ return context.getValue(AGENT_TOOL_SPAN_CONTEXT_KEY) as AgentToolSpanContext | undefined;+}++export function agentToolContentPolicy(+ context: Context = otelContext.active(),+): AgentToolContentPolicy {+ const spanContext = agentToolSpanContext(context);+ return spanContext === undefined+ ? NO_CONTENT_POLICY+ : {+ recordInputs: spanContext.recordInputs,+ recordOutputs: spanContext.recordOutputs,+ };+}++export function annotateAgentToolSpan(+ attributes: Attributes,+ context: Context = otelContext.active(),+): boolean {+ const setAttributes = agentToolSpanContext(context)?.setAttributes;+ if (setAttributes === undefined) return false;+ setAttributes(attributes);+ return true;+}++export function recordAgentToolSpanError(+ error: unknown,+ errorType?: string,+ context: Context = otelContext.active(),+): boolean {+ const spanContext = agentToolSpanContext(context);+ if (spanContext?.recordError === undefined) return false;+ spanContext.recordError(spanContext.recordOutputs ? error : undefined, errorType);+ return true;+}