The Open Framework for Building Agents
fix(eve): stamp trace session IDs on child spans (#5173)
Chad Hietalabaf3030c4173
diff --git a/.changeset/bright-spans-connect.md b/.changeset/bright-spans-connect.mdnew file mode 100644index 000000000..0348d8fb6--- /dev/null+++ b/.changeset/bright-spans-connect.md@@ -0,0 +1,5 @@+---+"eve": patch+---++Attach the agent trace session ID to auto-instrumented child spans on Vercel so Agent Runs can include network and other third-party spans in a session trace.diff --git a/packages/eve/src/execution/invoke-tool-trace-ownership.integration.test.ts b/packages/eve/src/execution/invoke-tool-trace-ownership.integration.test.tsindex e2e589024..f93e86083 100644--- a/packages/eve/src/execution/invoke-tool-trace-ownership.integration.test.ts+++ b/packages/eve/src/execution/invoke-tool-trace-ownership.integration.test.ts@@ -17,6 +17,7 @@ import { registerInstrumentationRuntime } from "#instrumentation/runtime.js";import { createBundledRuntimeCompiledArtifactsSource } from "#runtime/compiled-artifacts-source.js";import { defineJsonSchema } from "#tools/schema.js";import { AgentTraceSpanProcessor } from "#tracing/local/agent-trace-span-processor.js";+import { traceSessionIdFromContext } from "#tracing/eve/conversation-context.js";const CLIENT_TRACE_ID = "4bf92f3577b34da6a3ce929d0e0e4736";const alice: SessionAuthContext = {@@ -35,6 +36,7 @@ const inputSchema = defineJsonSchema({let openGate: () => void = () => undefined;let gate: Promise<void> = Promise.resolve();const seenActive: boolean[] = [];+const seenTraceSessionIds: Array<string | undefined> = [];function tool(name: string, execute: (input: { text?: string }) => unknown): HarnessToolDefinition {return {@@ -49,6 +51,7 @@ const tools = [tool("lookup", async () => {const traceId = apiTrace.getActiveSpan()?.spanContext().traceId;seenActive.push(traceId !== undefined && processor.activeTraceIds().has(traceId));+ seenTraceSessionIds.push(traceSessionIdFromContext(apiContext.active()));await apiTrace.getTracer("test.tool").startActiveSpan("nested lookup", async (span) => {span.end();});@@ -82,6 +85,7 @@ let provider: BasicTracerProvider;beforeEach(() => {seenActive.length = 0;+ seenTraceSessionIds.length = 0;exporter = new InMemorySpanExporter();// The production local-trace processor: it claims traces and releases completed ones on flush.processor = new AgentTraceSpanProcessor([new SimpleSpanProcessor(exporter)]);@@ -133,6 +137,7 @@ describe("invokeTool trace ownership", () => {it("holds the trace while the call runs and releases it when the call ends", async () => {await invokeTool(runtime, "lookup", {}, { auth: alice });expect(seenActive).toEqual([true]);+ expect(seenTraceSessionIds[0]).toMatch(/^call_session_/u);expect(names()).toEqual(["execute_tool lookup", "nested lookup"]);expect(processor.activeTraceIds().size).toBe(0);});diff --git a/packages/eve/src/tracing/eve/agent-otel-provider.test.ts b/packages/eve/src/tracing/eve/agent-otel-provider.test.tsindex b27d4f29c..1f4dd63b5 100644--- a/packages/eve/src/tracing/eve/agent-otel-provider.test.ts+++ b/packages/eve/src/tracing/eve/agent-otel-provider.test.ts@@ -35,6 +35,7 @@ import {import { AgentSpanIdGenerator } from "#compiled/@vercel/agent-runs/index.js";import { otelTelemetry } from "#compiled/@vercel/agent-runs/index.js";import { eveOutputMapping } from "#tracing/eve/profile.js";+import { traceSessionIdFromContext } from "#tracing/eve/conversation-context.js";import { ContextAgentTraceStateStore } from "#tracing/eve/agent-trace-context-store.js";import type { AgentTraceStateStore } from "#tracing/eve/agent-trace-state.js";import { InMemoryAgentTraceStateStore } from "#internal/testing/in-memory-agent-trace-state-store.js";@@ -3007,6 +3008,7 @@ describe("createAgentOtelInstrumentation", () => {const active = withSpy.mock.calls[0]?.[0];expect(active).toBeDefined();expect(activeTraceOperation(active!)?.type).toBe("model");+ expect(traceSessionIdFromContext(active)).toBe("session-1");withSpy.mockRestore();});diff --git a/packages/eve/src/tracing/eve/conversation-context.test.ts b/packages/eve/src/tracing/eve/conversation-context.test.tsindex a6eb9404b..efd76a685 100644--- a/packages/eve/src/tracing/eve/conversation-context.test.ts+++ b/packages/eve/src/tracing/eve/conversation-context.test.ts@@ -1,9 +1,12 @@import { describe, expect, it } from "vitest";+import { ROOT_CONTEXT } from "#compiled/@opentelemetry/api/index.js";import { ContextContainer, contextStorage } from "#context/container.js";import { ConversationIdKey } from "#context/keys.js";import { readConversationId, resolveConversationId } from "#shared/conversation-identity.js";import {readConversationBaggage,+ traceSessionIdFromContext,+ withOperationTraceContext,writeConversationBaggage,} from "#tracing/eve/conversation-context.js";@@ -14,6 +17,16 @@ describe("conversation correlation", () => {expect(baggage).toContain("eve.audience=private;ceiling=i0o0");});+ it("carries the trace session independently of the current run", () => {+ const context = withOperationTraceContext(ROOT_CONTEXT, {+ rootSessionId: "root-session",+ sessionId: "child-run",+ traceSessionId: "trace-session",+ });++ expect(traceSessionIdFromContext(context)).toBe("trace-session");+ });+it("rejects malformed, duplicate, oversized, and unsafe identifiers", () => {expect(readConversationBaggage("eve.conversation.id")).toBeUndefined();expect(readConversationBaggage("eve.conversation.id=%ZZ")).toBeUndefined();diff --git a/packages/eve/src/tracing/eve/conversation-context.ts b/packages/eve/src/tracing/eve/conversation-context.tsindex 4f0be5061..385024b39 100644--- a/packages/eve/src/tracing/eve/conversation-context.ts+++ b/packages/eve/src/tracing/eve/conversation-context.ts@@ -4,6 +4,7 @@ import { readConversationId, resolveConversationId } from "#shared/conversation-const BAGGAGE_KEY = "eve.conversation.id";const CONVERSATION_KEY = createContextKey("eve.trace.conversation.id");+const TRACE_SESSION_KEY = createContextKey("eve.trace.session.id");export function withConversationId(context: Context, conversationId: string): Context {return context.setValue(CONVERSATION_KEY, conversationId);@@ -12,22 +13,38 @@ export function withConversationId(context: Context, conversationId: string): Cointerface OperationConversation {readonly rootSessionId?: string;readonly sessionId: string;+ readonly traceSessionId?: string;+}++export function traceSessionIdOf(scope: OperationConversation): string {+ return scope.traceSessionId ?? scope.rootSessionId ?? scope.sessionId;}export function operationConversationId(input: OperationConversation): string {return resolveConversationId(input.rootSessionId ?? input.sessionId);}-export function withOperationConversation(context: Context, input: OperationConversation): Context {- return withConversationId(context, operationConversationId(input));+export function withOperationTraceContext(context: Context, input: OperationConversation): Context {+ return withConversationId(context, operationConversationId(input)).setValue(+ TRACE_SESSION_KEY,+ traceSessionIdOf(input),+ );}export function conversationIdFromContext(context: unknown): string | undefined {+ return readConversationId(readContextValue(context, CONVERSATION_KEY));+}++export function traceSessionIdFromContext(context: unknown): string | undefined {+ const value = readContextValue(context, TRACE_SESSION_KEY);+ return typeof value === "string" && value.length > 0 ? value : undefined;+}++function readContextValue(context: unknown, key: symbol): unknown {if (typeof context !== "object" || context === null) return undefined;const getValue = Reflect.get(context, "getValue");- return typeof getValue === "function"- ? readConversationId(Reflect.apply(getValue, context, [CONVERSATION_KEY]))- : undefined;+ if (typeof getValue !== "function") return undefined;+ return Reflect.apply(getValue, context, [key]);}export function readConversationBaggage(value: string | null): string | undefined {diff --git a/packages/eve/src/tracing/eve/invoke-tool-span.ts b/packages/eve/src/tracing/eve/invoke-tool-span.tsindex 6f7b8b834..ccb28e5a5 100644--- a/packages/eve/src/tracing/eve/invoke-tool-span.ts+++ b/packages/eve/src/tracing/eve/invoke-tool-span.ts@@ -27,7 +27,10 @@ import {import { DIRECT_TOOL_CALL_ATTRIBUTE, DIRECT_TOOL_CALL_VALUE } from "#tracing/local/inspection.js";import { markAgentTraceContext } from "#tracing/eve/agent-trace-context.js";import { withChannelAudience } from "#tracing/eve/channel-audience-context.js";-import { operationConversationId } from "#tracing/eve/conversation-context.js";+import {+ operationConversationId,+ withOperationTraceContext,+} from "#tracing/eve/conversation-context.js";import { MCP_SERIALIZER } from "#tracing/eve/mcp.js";import { eveOutputMapping } from "#tracing/eve/profile.js";@@ -163,7 +166,7 @@ async function traced(const recordOutputs = content.action === "record" && content.recordOutputs;const spanName = `execute_tool ${input.toolName}`;- const parent = withChannelAudience(base, conversation.audience);+ const parent = withOperationTraceContext(withChannelAudience(base, conversation.audience), scope);const attributes: Record<string, Attributes[string]> = {"agent.tool.is_framework": false,"gen_ai.agent.name": input.agentName,diff --git a/packages/eve/src/tracing/eve/operation-input.ts b/packages/eve/src/tracing/eve/operation-input.tsindex b7ad25acc..58c3740d8 100644--- a/packages/eve/src/tracing/eve/operation-input.ts+++ b/packages/eve/src/tracing/eve/operation-input.ts@@ -7,14 +7,6 @@ import {type TraceCheckpointer,} from "#compiled/@vercel/agent-runs/index.js";-export function traceSessionIdOf(scope: {- readonly traceSessionId?: string;- readonly rootSessionId?: string;- readonly sessionId: string;-}): string {- return scope.traceSessionId ?? scope.rootSessionId ?? scope.sessionId;-}-export function checkpointContent(value: string | undefined): unknown {if (value === undefined) return undefined;// Historical checkpoints contain truncated JSON; omit payloads, not the operation.diff --git a/packages/eve/src/tracing/eve/otel-registration.scenario.test.ts b/packages/eve/src/tracing/eve/otel-registration.scenario.test.tsindex 47f4d925c..ce2a84af0 100644--- a/packages/eve/src/tracing/eve/otel-registration.scenario.test.ts+++ b/packages/eve/src/tracing/eve/otel-registration.scenario.test.ts@@ -15,7 +15,7 @@ import {trace as runtimeTrace,} from "#compiled/@opentelemetry/api/index.js";import { registerOtelPipeline } from "./otel-registration.js";-import { withConversationId } from "./conversation-context.js";+import { withConversationId, withOperationTraceContext } from "./conversation-context.js";const require = createRequire(import.meta.url);const authoredApi = require("@opentelemetry/api") as typeof import("@opentelemetry/api");@@ -91,6 +91,7 @@ describe("OTel registration", () => {);it("delegates an authored tracer cached before registration", async () => {+ vi.stubEnv("VERCEL_ENV", "preview");const authoredTracer = authoredApi.trace.getTracer("authored");expect(authoredTracer.startSpan("before-registration").isRecording()).toBe(false);@@ -103,7 +104,11 @@ describe("OTel registration", () => {const parent = runtimeTrace.getTracer("eve").startSpan("eve.parent");const parentContext = parent.spanContext();const activeContext = withConversationId(- runtimeTrace.setSpan(COMPILED_ROOT_CONTEXT, parent),+ withOperationTraceContext(runtimeTrace.setSpan(COMPILED_ROOT_CONTEXT, parent), {+ rootSessionId: "root-session",+ sessionId: "child-run",+ traceSessionId: "trace-session",+ }),"caller-conversation",);const child = await runtimeContext.with(activeContext, async () => {@@ -128,6 +133,7 @@ describe("OTel registration", () => {.find((span) => span.name === "authored.child");expect(exportedChild?.parentSpanContext?.spanId).toBe(parentContext.spanId);expect(exportedChild?.attributes["gen_ai.conversation.id"]).toBe("caller-conversation");+ expect(exportedChild?.attributes["vercel.session_id"]).toBe("trace-session");expect(exporter.getFinishedSpans().find((span) => span.name === "authored.conversation")?.attributes["gen_ai.conversation.id"diff --git a/packages/eve/src/tracing/eve/otel-registration.ts b/packages/eve/src/tracing/eve/otel-registration.tsindex 2f983e0f8..fb9c7afc5 100644--- a/packages/eve/src/tracing/eve/otel-registration.ts+++ b/packages/eve/src/tracing/eve/otel-registration.ts@@ -18,7 +18,10 @@ import { type Attributes } from "#compiled/@vercel/agent-runs/index.js";import { invocationName } from "#compiled/@vercel/agent-runs/otel.js";export type SamplingOperation = { readonly name: string; readonly attributes?: Attributes };-import { conversationIdFromContext } from "#tracing/eve/conversation-context.js";+import {+ conversationIdFromContext,+ traceSessionIdFromContext,+} from "#tracing/eve/conversation-context.js";const REGISTRATION_SPAN_NAME = "eve.otel.registration";const REPLAY_DEDUPLICATION_LIMIT = 100_000;@@ -98,17 +101,23 @@ class PrivateSpanFilteringProcessor implements SpanProcessor {onStart(span: unknown, parentContext: unknown): void {if (isRegistrationSpan(span)) return;+ const mutableSpan = span as {+ readonly attributes?: Record<string, unknown>;+ setAttribute(key: string, value: string): void;+ };const conversationId = conversationIdFromContext(parentContext);if (conversationId !== undefined &&- (span as { attributes?: Record<string, unknown> }).attributes?.["gen_ai.conversation.id"] ===- undefined- ) {- (span as { setAttribute(key: string, value: string): void }).setAttribute(- "gen_ai.conversation.id",- conversationId,- );- }+ mutableSpan.attributes?.["gen_ai.conversation.id"] === undefined+ )+ mutableSpan.setAttribute("gen_ai.conversation.id", conversationId);+ const traceSessionId = traceSessionIdFromContext(parentContext);+ if (+ process.env.VERCEL_ENV !== undefined &&+ traceSessionId !== undefined &&+ mutableSpan.attributes?.["vercel.session_id"] === undefined+ )+ mutableSpan.setAttribute("vercel.session_id", traceSessionId);const identity = spanIdentity(span);if (identity !== undefined) addBounded(this.startedSpans, identity);for (const processor of this.processors) processor.onStart(span, parentContext);diff --git a/packages/eve/src/tracing/eve/session.ts b/packages/eve/src/tracing/eve/session.tsindex 31da18d53..fa6e5a941 100644--- a/packages/eve/src/tracing/eve/session.ts+++ b/packages/eve/src/tracing/eve/session.ts@@ -1,4 +1,4 @@-import type { SpanContext } from "#compiled/@opentelemetry/api/index.js";+import type { Context, SpanContext } from "#compiled/@opentelemetry/api/index.js";import {type InstrumentationSessionStartedEvent,@@ -28,14 +28,12 @@ import type { AgentTurnTraceState } from "#tracing/eve/agent-trace-state.js";import { applyPrincipalTraceDecision } from "#instrumentation/principal-summary.js";import { normalizeInstrumentationChannelKind } from "#internal/instrumentation.js";import type { ConversationEnvironment } from "#shared/conversation-context.js";-import { traceSessionIdOf } from "#tracing/eve/operation-input.js";import { ROOT_CONTEXT } from "#compiled/@opentelemetry/api/index.js";import { withChannelAudience } from "#tracing/eve/channel-audience-context.js";-import { withOperationConversation } from "#tracing/eve/conversation-context.js";+import { traceSessionIdOf, withOperationTraceContext } from "#tracing/eve/conversation-context.js";import type {AgentSpanIdGenerator,AgentRun,- ExecutionContext,TraceLink,TurnOperation,} from "#compiled/@vercel/agent-runs/index.js";@@ -61,11 +59,7 @@ interface EveSessionTracing {event: InstrumentationTurnStartedEvent,) => Promise<InstrumentationTraceSeed>;/** Hydrates a turn this process or an earlier one started. */- turnFor(- sessionId: string,- turnId: string,- context?: ExecutionContext,- ): Promise<TurnOperation | undefined>;+ turnFor(sessionId: string, turnId: string, context?: Context): Promise<TurnOperation | undefined>;/** Whether the trace policy dropped this turn, so its work must not record spans. */dropped(sessionId: string, turnId: string): Promise<boolean>;turnTerminal(event: InstrumentationTurnTerminalEvent): Promise<void>;@@ -110,13 +104,16 @@ export function createEveSessionTracing(input: EveSessionTracingInput): EveSessiconst turnKey = (sessionId: string, turnId: string) => JSON.stringify([sessionId, turnId]);- async function turnFor(sessionId: string, turnId: string, context?: ExecutionContext) {+ async function turnFor(sessionId: string, turnId: string, context?: Context) {const turn = await input.stateStore.get("turn", turnKey(sessionId, turnId));if (turn === undefined || !isSampledTrace(turn.context)) return undefined;const session = await input.stateStore.get("session", sessionId);return input.tracing.resume({identity: eveTurnIdentity({ sessionId, rootSessionId: turn.rootSessionId, turnId }),- context: context ?? hostContext(session?.channelAudience, { sessionId, ...turn }),+ context: withOperationTraceContext(+ context ?? withChannelAudience(ROOT_CONTEXT, session?.channelAudience),+ { sessionId, ...turn },+ ),});}@@ -257,12 +254,16 @@ export function createEveSessionTracing(input: EveSessionTracingInput): EveSessi};}-/** Spans this process starts inherit the channel audience and the operation's conversation. */+/** Spans this process starts inherit the operation's audience and trace correlation. */function hostContext(audience: ChannelAudience | undefined,- operation: { readonly sessionId: string; readonly rootSessionId?: string },+ operation: {+ readonly sessionId: string;+ readonly rootSessionId?: string;+ readonly traceSessionId?: string;+ },) {- return withOperationConversation(withChannelAudience(ROOT_CONTEXT, audience), operation);+ return withOperationTraceContext(withChannelAudience(ROOT_CONTEXT, audience), operation);}function activationLinks(turn: AgentTurnTraceState): TraceLink[] | undefined {