The Open Framework for Building Agents
feat(eve): record agent.turn spans with real durations in local traces (#1781)
Chad Hietala63a76f00d9b8
diff --git a/.changeset/real-duration-agent-turn-spans.md b/.changeset/real-duration-agent-turn-spans.mdnew file mode 100644index 000000000..23cd4c408--- /dev/null+++ b/.changeset/real-duration-agent-turn-spans.md@@ -0,0 +1,5 @@+---+"eve": patch+---++Local traces now record `agent.turn` with the turn's real duration instead of a zero-duration marker, and the separate `agent.turn.terminal` marker span is gone — terminal and transition events land on the turn span itself. `agent.session` window roots remain zero-duration markers because an idle session never closes.diff --git a/docs/reference/cli.md b/docs/reference/cli.mdindex 02382a662..675b54daa 100644--- a/docs/reference/cli.md+++ b/docs/reference/cli.md@@ -271,7 +271,7 @@ A session long enough to outgrow one trace — far longer than anything you willOne parent turn can dispatch several subagents into the same window, so a child's turn spans name the dispatch: `agent.parent.session.id`, `agent.parent.turn.id`, and `agent.parent.call_id` identify the tool call that created the child, and `agent.subagent.name` the subagent it invoked. Top-level sessions carry none of these.-`agent.session` and `agent.turn` outlive the worker that opened them, so they are recorded as zero-duration markers. The span tree shows their descendant extent instead; `agent.step` and the model and tool spans beneath it carry real durations. A third-party OTel backend reports zero for the markers.+Every span carries a real duration except `agent.session`: an idle session never closes, so it is recorded as a zero-duration marker and the span tree shows its descendant extent instead. A turn's span is written when the turn settles, so a running turn shows only its steps.Model and tool-call spans carry their inputs and outputs — system prompt, prompt messages, and response text for models; call arguments and results for tools — each capped at 32 KB. Set `EVE_TRACES_CONTENT=off` to keep payloads out of the spool.diff --git a/packages/eve/scripts/vendor-compiled/declarations/@opentelemetry/api.d.ts b/packages/eve/scripts/vendor-compiled/declarations/@opentelemetry/api.d.tsindex 4e1ddf70b..7954271d7 100644--- a/packages/eve/scripts/vendor-compiled/declarations/@opentelemetry/api.d.ts+++ b/packages/eve/scripts/vendor-compiled/declarations/@opentelemetry/api.d.ts@@ -7,7 +7,7 @@ export interface SpanContext {}export interface Span {- addEvent(name: string, attributes?: Attributes): this;+ addEvent(name: string, attributes?: Attributes, timestamp?: number): this;end(): void;recordException(exception: Error | string | { message?: string; name?: string; stack?: string },@@ -24,6 +24,7 @@ export interface Tracer {attributes?: Attributes | undefined;kind?: SpanKind | undefined;root?: boolean | undefined;+ startTime?: number | undefined;},context?: Context,): Span;diff --git a/packages/eve/scripts/vendor-compiled/declarations/@vercel/otel.d.ts b/packages/eve/scripts/vendor-compiled/declarations/@vercel/otel.d.tsindex b1b1e3145..e6cfcfc8e 100644--- a/packages/eve/scripts/vendor-compiled/declarations/@vercel/otel.d.ts+++ b/packages/eve/scripts/vendor-compiled/declarations/@vercel/otel.d.ts@@ -5,8 +5,14 @@ export interface SpanProcessor {shutdown(): Promise<void>;}+export interface IdGenerator {+ generateSpanId(): string;+ generateTraceId(): string;+}+export interface Configuration {readonly autoDetectResources?: boolean;+ readonly idGenerator?: IdGenerator;readonly instrumentations?: readonly unknown[];readonly propagators?: readonly ["none"];readonly serviceName?: string;diff --git a/packages/eve/src/cli/commands/trace.integration.test.ts b/packages/eve/src/cli/commands/trace.integration.test.tsindex 8c16fd4b4..5b60f0feb 100644--- a/packages/eve/src/cli/commands/trace.integration.test.ts+++ b/packages/eve/src/cli/commands/trace.integration.test.ts@@ -189,7 +189,7 @@ describe("eve traces", () => {const session = "a".repeat(16);const turn = "b".repeat(16);const step = "c".repeat(16);- const terminal = "d".repeat(16);+ const orphanMarker = "d".repeat(16);await writeSegment(root,TRACE_ONE,@@ -200,7 +200,7 @@ describe("eve traces", () => {await writeSegment(root,TRACE_ONE,- span(turn, "agent.turn", 10, 10, session, { "agent.session.id": "session-one" }),+ span(turn, "agent.turn", 10, 95, session, { "agent.session.id": "session-one" }),);await writeSegment(root,@@ -210,7 +210,7 @@ describe("eve traces", () => {await writeSegment(root,TRACE_ONE,- span(terminal, "agent.turn.terminal", 95, 95, turn, {+ span(orphanMarker, "user.marker", 95, 95, turn, {"agent.session.id": "session-one",}),);@@ -221,8 +221,8 @@ describe("eve traces", () => {expect(output.out[0]).toContain("agent.session 85ms");expect(output.out[0]).toContain("agent.turn 85ms");expect(output.out[0]).toContain("agent.step 70ms");- // A marker with no descendants has no extent to borrow.- expect(output.out[0]).toContain("agent.turn.terminal 0ms");+ // A zero-duration span with no descendants has no extent to borrow.+ expect(output.out[0]).toContain("user.marker 0ms");});it("shows usage and cost chips on span rows and totals in the header", async () => {diff --git a/packages/eve/src/cli/commands/trace.ts b/packages/eve/src/cli/commands/trace.tsindex b5d761be7..d7cae8401 100644--- a/packages/eve/src/cli/commands/trace.ts+++ b/packages/eve/src/cli/commands/trace.ts@@ -285,10 +285,11 @@ function renderSpanTree(/*** Extent of each span's own range unioned with its descendants', keyed by span id.*- * eve records a span whose lifetime crosses a durable worker boundary — a- * session window or a turn — as a zero-duration marker, because a span object- * cannot cross that boundary to be ended later. The tree falls back to this- * extent for those rows so it shows where the time went.+ * eve records a span with no guaranteed close — an `agent.session` window+ * root, whose session may idle forever — as a zero-duration marker, because+ * a span object cannot cross a durable worker boundary to be ended later.+ * The tree falls back to this extent for those rows so it shows where the+ * time went.*/function subtreeExtents(spans: readonly LocalTraceSpan[],diff --git a/packages/eve/src/cli/dev/tui/traces/trace-conversation.ts b/packages/eve/src/cli/dev/tui/traces/trace-conversation.tsindex cff1003da..fbd94f8b3 100644--- a/packages/eve/src/cli/dev/tui/traces/trace-conversation.ts+++ b/packages/eve/src/cli/dev/tui/traces/trace-conversation.ts@@ -20,7 +20,7 @@ import { SURFACE_CLOSE } from "./trace-surfaces.js";export interface ConversationItem {readonly kind: "system" | "user" | "assistant" | "tool";- /** Detail-drawer source for this item (the turn marker span for users). */+ /** Detail-drawer source for this item (the turn span for users). */readonly span: LocalTraceSpan;/** Tool name for tool items. */readonly name?: string;diff --git a/packages/eve/src/tracing/agent-otel-provider.test.ts b/packages/eve/src/tracing/agent-otel-provider.test.tsindex 3019fe046..d8380faf1 100644--- a/packages/eve/src/tracing/agent-otel-provider.test.ts+++ b/packages/eve/src/tracing/agent-otel-provider.test.ts@@ -10,9 +10,10 @@ import { describe, expect, it } from "vitest";import { createAiSdkHookBridge } from "#harness/ai-sdk-hook-bridge.js";import {createAgentOtelInstrumentation,- InMemoryAgentTraceStateStore,SESSION_WINDOW_TURN_LIMIT,} from "#tracing/agent-otel-provider.js";+import { AgentSpanIdGenerator } from "#tracing/agent-span-id-generator.js";+import { InMemoryAgentTraceStateStore } from "#tracing/agent-trace-state.js";import {createInstrumentationHooks,type InstrumentationAttemptScope,@@ -31,11 +32,14 @@ interface TestRuntime {function createRuntime(stateStore = new InMemoryAgentTraceStateStore()): TestRuntime {const exporter = new InMemorySpanExporter();+ const idGenerator = new AgentSpanIdGenerator();const provider = new BasicTracerProvider({+ idGenerator,spanProcessors: [new SimpleSpanProcessor(exporter)],});const agentOtel = createAgentOtelInstrumentation({frameworkVersion: "test",+ idGenerator,stateStore,tracer: provider.getTracer("eve.agent"),});@@ -200,10 +204,24 @@ async function publishTurnStarted(input: {});}+/** The turn span itself is emitted at the turn's session transition. */+async function completeTurn(+ hooks: InstrumentationHooks,+ sessionId: string,+ turnId: string,+): Promise<void> {+ await hooks.publish({ sessionId, turnId, type: "turn.completed" });+ await hooks.publish({ sessionId, turnId, type: "session.waiting" });+}+function byName(spans: readonly ReadableSpan[], name: string): ReadableSpan[] {return spans.filter((span) => span.name === name);}+function nanos(hrTime: readonly [number, number]): bigint {+ return BigInt(hrTime[0]) * 1_000_000_000n + BigInt(hrTime[1]);+}+describe("createAgentOtelInstrumentation", () => {it("emits the agent hierarchy in one session trace", async () => {const runtime = createRuntime();@@ -218,7 +236,6 @@ describe("createAgentOtelInstrumentation", () => {const spans = runtime.exporter.getFinishedSpans();const turn = byName(spans, "agent.turn")[0]!;- const turnTerminal = byName(spans, "agent.turn.terminal")[0]!;const step = byName(spans, "agent.step")[0]!;const operation = byName(spans, "ai.streamText")[0]!;const model = byName(spans, "ai.streamText.doStream")[0]!;@@ -233,18 +250,22 @@ describe("createAgentOtelInstrumentation", () => {"agent.session.window": 0,});expect(turn.parentSpanContext?.spanId).toBe(session.spanContext().spanId);- expect(turnTerminal.parentSpanContext?.spanId).toBe(turn.spanContext().spanId);expect(step.parentSpanContext?.spanId).toBe(turn.spanContext().spanId);expect(operation.parentSpanContext?.spanId).toBe(step.spanContext().spanId);expect(model.parentSpanContext?.spanId).toBe(operation.spanContext().spanId);expect(action.parentSpanContext?.spanId).toBe(step.spanContext().spanId);expect(tool.parentSpanContext?.spanId).toBe(action.spanContext().spanId);expect(new Set(spans.map((span) => span.spanContext().traceId))).toHaveLength(1);- expect(turn.events.map((event) => event.name)).toEqual(["turn.started"]);- expect(turnTerminal.events.map((event) => event.name)).toEqual([+ expect(turn.events.map((event) => event.name)).toEqual([+ "turn.started","turn.completed","session.waiting",]);+ // Turn timestamps are millisecond-quantized (`Date.now`), so the end+ // comparison against the step's sub-millisecond clock gets 1ms of slack.+ expect(turn.attributes).toMatchObject({ "agent.name": "weather", "agent.session.window": 0 });+ expect(nanos(turn.startTime)).toBeLessThanOrEqual(nanos(step.startTime));+ expect(nanos(turn.endTime)).toBeGreaterThanOrEqual(nanos(step.endTime) - 1_000_000n);expect(step.attributes).toMatchObject({"agent.framework.name": "eve","agent.model.id": "claude-test",@@ -372,12 +393,15 @@ describe("createAgentOtelInstrumentation", () => {it("captures nothing when content capture is off", async () => {const exporter = new InMemorySpanExporter();+ const idGenerator = new AgentSpanIdGenerator();const provider = new BasicTracerProvider({+ idGenerator,spanProcessors: [new SimpleSpanProcessor(exporter)],});const agentOtel = createAgentOtelInstrumentation({captureContent: false,frameworkVersion: "test",+ idGenerator,stateStore: new InMemoryAgentTraceStateStore(),tracer: provider.getTracer("eve.agent"),});@@ -506,8 +530,11 @@ describe("createAgentOtelInstrumentation", () => {});await replacementRuntime.provider.forceFlush();- const turn = byName(firstRuntime.exporter.getFinishedSpans(), "agent.turn")[0]!;+ // The replacement's session transition emits the turn span with the span+ // id the first worker allocated, so descendants from both workers attach.+ expect(byName(firstRuntime.exporter.getFinishedSpans(), "agent.turn")).toHaveLength(0);const replacementSpans = replacementRuntime.exporter.getFinishedSpans();+ const turn = byName(replacementSpans, "agent.turn")[0]!;const step = byName(replacementSpans, "agent.step")[0]!;const action = byName(replacementSpans, "agent.action")[0]!;@@ -554,6 +581,7 @@ describe("createAgentOtelInstrumentation", () => {turnId: `turn-${sequence}`,turnSequence: sequence,});+ await completeTurn(runtime.hooks, "session-1", `turn-${sequence}`);}await runtime.provider.forceFlush();@@ -596,6 +624,7 @@ describe("createAgentOtelInstrumentation", () => {turnId: "child-turn-1",turnSequence: 0,});+ await completeTurn(runtime.hooks, "child-1", "child-turn-1");await runtime.provider.forceFlush();const spans = runtime.exporter.getFinishedSpans();@@ -626,6 +655,7 @@ describe("createAgentOtelInstrumentation", () => {turnId: "child-turn-1",turnSequence: 0,});+ await completeTurn(runtime.hooks, "child-1", "child-turn-1");await runtime.provider.forceFlush();const spans = runtime.exporter.getFinishedSpans();@@ -648,6 +678,7 @@ describe("createAgentOtelInstrumentation", () => {turnId: "child-turn-1",turnSequence: 0,});+ await completeTurn(runtime.hooks, "child-1", "child-turn-1");await runtime.provider.forceFlush();const childTurn = byName(runtime.exporter.getFinishedSpans(), "agent.turn").find(@@ -665,6 +696,7 @@ describe("createAgentOtelInstrumentation", () => {turnId: "turn-1",turnSequence: 0,});+ await completeTurn(runtime.hooks, "session-1", "turn-1");await runtime.provider.forceFlush();const turn = byName(runtime.exporter.getFinishedSpans(), "agent.turn")[0]!;diff --git a/packages/eve/src/tracing/agent-otel-provider.ts b/packages/eve/src/tracing/agent-otel-provider.tsindex 73ac55810..548bc237c 100644--- a/packages/eve/src/tracing/agent-otel-provider.ts+++ b/packages/eve/src/tracing/agent-otel-provider.ts@@ -16,6 +16,8 @@ import {textContentAttribute,toolResultsContentAttribute,} from "#tracing/agent-otel-content.js";+import type { AgentSpanIdGenerator } from "#tracing/agent-span-id-generator.js";+import type { AgentSessionTraceState, AgentTraceStateStore } from "#tracing/agent-trace-state.js";import type {InstrumentationAttemptMetadataEvent,InstrumentationAttemptScope,@@ -52,70 +54,6 @@ interface ToolSpanState extends SpanState {/** Sized so an ordinary session stays one trace and only an outsized one rolls. */export const SESSION_WINDOW_TURN_LIMIT = 200;-export interface AgentSessionTraceState {- readonly agentName?: string;- readonly channelKind?: string;- readonly context: SpanContext;- readonly rootSessionId: string;- readonly turnsInWindow: number;- readonly window: number;-}--export interface AgentTurnTraceState {- readonly context: SpanContext;- readonly rootSessionId: string;- readonly sequence: number;- readonly terminal?: {- readonly error?: unknown;- readonly type: InstrumentationTurnTerminalEvent["type"];- };-}--/** Provider-owned serializable storage for durable agent trace state. */-export interface AgentTraceStateStore {- deleteSession(sessionId: string): void | PromiseLike<void>;- deleteTurn(sessionId: string, turnId: string): void | PromiseLike<void>;- getSession(- sessionId: string,- ): AgentSessionTraceState | undefined | PromiseLike<AgentSessionTraceState | undefined>;- getTurn(- sessionId: string,- turnId: string,- ): AgentTurnTraceState | undefined | PromiseLike<AgentTurnTraceState | undefined>;- setSession(sessionId: string, state: AgentSessionTraceState): void | PromiseLike<void>;- setTurn(sessionId: string, turnId: string, state: AgentTurnTraceState): void | PromiseLike<void>;-}--/** In-memory trace state used by tests and non-durable runtimes. */-export class InMemoryAgentTraceStateStore implements AgentTraceStateStore {- readonly #sessions = new Map<string, AgentSessionTraceState>();- readonly #turns = new Map<string, AgentTurnTraceState>();-- deleteSession(sessionId: string): void {- this.#sessions.delete(sessionId);- }-- deleteTurn(sessionId: string, turnId: string): void {- this.#turns.delete(turnKey(sessionId, turnId));- }-- getSession(sessionId: string): AgentSessionTraceState | undefined {- return this.#sessions.get(sessionId);- }-- getTurn(sessionId: string, turnId: string): AgentTurnTraceState | undefined {- return this.#turns.get(turnKey(sessionId, turnId));- }-- setSession(sessionId: string, state: AgentSessionTraceState): void {- this.#sessions.set(sessionId, state);- }-- setTurn(sessionId: string, turnId: string, state: AgentTurnTraceState): void {- this.#turns.set(turnKey(sessionId, turnId), state);- }-}-export interface AgentOtelInstrumentationInput {/*** Capture model prompts/responses and tool call inputs/outputs as span@@ -124,6 +62,7 @@ export interface AgentOtelInstrumentationInput {*/readonly captureContent?: boolean;readonly frameworkVersion: string;+ readonly idGenerator: AgentSpanIdGenerator;readonly stateStore: AgentTraceStateStore;readonly tracer: Tracer;}@@ -164,34 +103,27 @@ export function createAgentOtelInstrumentation(type: "session.started",}),);- const span = input.tracer.startSpan(- "agent.turn",- {- attributes: {- "agent.framework.name": "eve",- "agent.framework.version": input.frameworkVersion,- "agent.name": session.agentName,- ...parentLineageAttributes(event.parentLineage),- "agent.root.session.id": event.rootSessionId,- "agent.session.id": event.sessionId,- "agent.session.window": session.window,- "agent.turn.id": event.turnId,- "agent.turn.sequence": event.sequence,- },- },- contextFromSpanContext(session.context),- );- span.addEvent("turn.started");+ // The turn outlives this worker, so no live span can cover it: the span+ // id is allocated now for descendants to parent through, and the span+ // itself is emitted at the turn's session transition.+ const turnContext: SpanContext = {+ isRemote: false,+ spanId: input.idGenerator.allocateSpanId(),+ traceFlags: session.context.traceFlags,+ traceId: session.context.traceId,+ };await input.stateStore.setSession(event.sessionId, {...session,turnsInWindow: session.turnsInWindow + 1,});await input.stateStore.setTurn(event.sessionId, event.turnId, {- context: span.spanContext(),+ context: turnContext,+ lineage: event.parentLineage,+ parentSpanId: session.context.spanId,rootSessionId: event.rootSessionId,sequence: event.sequence,+ startTimeMs: Date.now(),});- span.end();};const onAttemptStarted = async (event: InstrumentationAttemptStartedEvent): Promise<void> => {@@ -269,18 +201,33 @@ export function createAgentOtelInstrumentation(if (event.turnId !== undefined) {const turn = await input.stateStore.getTurn(event.sessionId, event.turnId);if (turn !== undefined) {- const span = input.tracer.startSpan(- "agent.turn.terminal",- {- attributes: {- "agent.root.session.id": turn.rootSessionId,- "agent.session.id": event.sessionId,- "agent.turn.id": event.turnId,- "agent.turn.sequence": turn.sequence,+ const session = await input.stateStore.getSession(event.sessionId);+ const span = input.idGenerator.withSpanId(turn.context.spanId, () =>+ input.tracer.startSpan(+ "agent.turn",+ {+ attributes: {+ "agent.framework.name": "eve",+ "agent.framework.version": input.frameworkVersion,+ "agent.name": session?.agentName,+ ...parentLineageAttributes(turn.lineage),+ "agent.root.session.id": turn.rootSessionId,+ "agent.session.id": event.sessionId,+ "agent.session.window": session?.window,+ "agent.turn.id": event.turnId,+ "agent.turn.sequence": turn.sequence,+ },+ startTime: turn.startTimeMs,},- },- contextFromSpanContext(turn.context),+ contextFromSpanContext({+ isRemote: false,+ spanId: turn.parentSpanId,+ traceFlags: turn.context.traceFlags,+ traceId: turn.context.traceId,+ }),+ ),);+ span.addEvent("turn.started", undefined, turn.startTimeMs);if (turn.terminal !== undefined) {span.addEvent(turn.terminal.type);if (turn.terminal.type === "turn.failed") {@@ -466,8 +413,9 @@ export function createAgentOtelInstrumentation(root: true,});span.addEvent(window.index === 0 ? "session.started" : "session.window.opened");- // The window outlives this worker, so the root is recorded as a marker and- // later spans parent through its persisted context, as `agent.turn` does.+ // The window outlives this worker and has no guaranteed close — an idle+ // session never ends — so the root is recorded as a zero-duration marker+ // and later spans parent through its persisted context.span.end();return span.spanContext();};@@ -573,10 +521,6 @@ export function createAgentOtelInstrumentation(}}-function turnKey(sessionId: string, turnId: string): string {- return `${sessionId}:${turnId}`;-}-function parentLineageAttributes(lineage: InstrumentationParentLineage | undefined,): Record<string, string> {diff --git a/packages/eve/src/tracing/agent-span-id-generator.ts b/packages/eve/src/tracing/agent-span-id-generator.tsnew file mode 100644index 000000000..740ea8477--- /dev/null+++ b/packages/eve/src/tracing/agent-span-id-generator.ts@@ -0,0 +1,50 @@+/**+ * Id generator shared by `registerOTel` and the agent OTel provider. A span+ * whose lifetime crosses durable worker boundaries (`agent.turn`) is emitted+ * at its terminal; priming the next span id lets that span carry the+ * pre-allocated id its descendants already parented to.+ */+export class AgentSpanIdGenerator {+ #primedSpanId: string | undefined;++ /** Reserves a span id for a span emitted later via {@link withSpanId}. */+ allocateSpanId(): string {+ return randomHexId(16);+ }++ generateSpanId(): string {+ const primed = this.#primedSpanId;+ if (primed !== undefined) {+ this.#primedSpanId = undefined;+ return primed;+ }+ return randomHexId(16);+ }++ generateTraceId(): string {+ return randomHexId(32);+ }++ /** Runs `startSpan` so the next span started carries `spanId`. */+ withSpanId<T>(spanId: string, startSpan: () => T): T {+ this.#primedSpanId = spanId;+ try {+ return startSpan();+ } finally {+ this.#primedSpanId = undefined;+ }+ }+}++const HEX_DIGITS = "0123456789abcdef";++function randomHexId(length: number): string {+ let id: string;+ do {+ id = "";+ for (let index = 0; index < length; index += 1) {+ id += HEX_DIGITS[Math.trunc(Math.random() * 16)];+ }+ } while (!/[1-9a-f]/u.test(id));+ return id;+}diff --git a/packages/eve/src/tracing/agent-trace-context-store.test.ts b/packages/eve/src/tracing/agent-trace-context-store.test.tsindex 66e2d5d2d..9357f1446 100644--- a/packages/eve/src/tracing/agent-trace-context-store.test.ts+++ b/packages/eve/src/tracing/agent-trace-context-store.test.ts@@ -22,8 +22,16 @@ describe("ContextAgentTraceStateStore", () => {});store.setTurn("session-1", "turn-1", {context: spanContext("1", "3"),+ lineage: {+ callId: "call-1",+ sessionId: "parent-session",+ subagentName: "researcher",+ turnId: "parent-turn",+ },+ parentSpanId: "2".repeat(16),rootSessionId: "session-1",sequence: 0,+ startTimeMs: 1_700_000_000_000,terminal: { error: new Error("failed"), type: "turn.failed" },});});@@ -35,6 +43,16 @@ describe("ContextAgentTraceStateStore", () => {expect(store.getSession("session-1")?.context).toEqual(spanContext("1", "2"));expect(store.getSession("session-1")).toMatchObject({ turnsInWindow: 3, window: 1 });expect(store.getTurn("session-1", "turn-1")?.context).toEqual(spanContext("1", "3"));+ expect(store.getTurn("session-1", "turn-1")).toMatchObject({+ lineage: {+ callId: "call-1",+ sessionId: "parent-session",+ subagentName: "researcher",+ turnId: "parent-turn",+ },+ parentSpanId: "2".repeat(16),+ startTimeMs: 1_700_000_000_000,+ });expect(store.getTurn("session-1", "turn-1")?.terminal?.error).toMatchObject({message: "failed",});@@ -52,8 +70,10 @@ describe("ContextAgentTraceStateStore", () => {});store.setTurn("session-1", "turn-1", {context: spanContext("1", "3"),+ parentSpanId: "2".repeat(16),rootSessionId: "session-1",sequence: 0,+ startTimeMs: 1_700_000_000_000,});store.deleteTurn("session-1", "turn-1");diff --git a/packages/eve/src/tracing/agent-trace-context-store.ts b/packages/eve/src/tracing/agent-trace-context-store.tsindex 1efee94d8..16fb85431 100644--- a/packages/eve/src/tracing/agent-trace-context-store.ts+++ b/packages/eve/src/tracing/agent-trace-context-store.ts@@ -2,11 +2,12 @@ import type { SpanContext } from "#compiled/@opentelemetry/api/index.js";import { contextStorage, loadContext } from "#context/container.js";import { ContextKey } from "#context/key.js";+import type { InstrumentationParentLineage } from "#harness/instrumentation-lifecycle.js";import type {AgentSessionTraceState,AgentTraceStateStore,AgentTurnTraceState,-} from "#tracing/agent-otel-provider.js";+} from "#tracing/agent-trace-state.js";interface AgentTraceContextState {readonly sessions: Readonly<Record<string, AgentSessionTraceState>>;@@ -136,10 +137,16 @@ function deserializeState(data: unknown): AgentTraceContextState {});const turns = deserializeRecord(data.turns, (value) => {if (!isRecord(value) || !isSpanContext(value.context)) return undefined;+ if (typeof value.parentSpanId !== "string" || typeof value.startTimeMs !== "number") {+ return undefined;+ }return {context: value.context,+ lineage: deserializeLineage(value.lineage),+ parentSpanId: value.parentSpanId,rootSessionId: typeof value.rootSessionId === "string" ? value.rootSessionId : "",sequence: typeof value.sequence === "number" ? value.sequence : 0,+ startTimeMs: value.startTimeMs,terminal: deserializeTerminal(value.terminal),} satisfies AgentTurnTraceState;});@@ -159,6 +166,23 @@ function deserializeRecord<T>(return result;}+function deserializeLineage(value: unknown): InstrumentationParentLineage | undefined {+ if (+ !isRecord(value) ||+ typeof value.callId !== "string" ||+ typeof value.sessionId !== "string" ||+ typeof value.turnId !== "string"+ ) {+ return undefined;+ }+ return {+ callId: value.callId,+ sessionId: value.sessionId,+ subagentName: typeof value.subagentName === "string" ? value.subagentName : undefined,+ turnId: value.turnId,+ };+}+function deserializeTerminal(value: unknown): AgentTurnTraceState["terminal"] {if (!isRecord(value) || typeof value.type !== "string") return undefined;const type = value.type;diff --git a/packages/eve/src/tracing/agent-trace-state.ts b/packages/eve/src/tracing/agent-trace-state.tsnew file mode 100644index 000000000..6564b9685--- /dev/null+++ b/packages/eve/src/tracing/agent-trace-state.ts@@ -0,0 +1,77 @@+import type { SpanContext } from "#compiled/@opentelemetry/api/index.js";++import type {+ InstrumentationParentLineage,+ InstrumentationTurnTerminalEvent,+} from "#harness/instrumentation-lifecycle.js";++export interface AgentSessionTraceState {+ readonly agentName?: string;+ readonly channelKind?: string;+ readonly context: SpanContext;+ readonly rootSessionId: string;+ readonly turnsInWindow: number;+ readonly window: number;+}++export interface AgentTurnTraceState {+ readonly context: SpanContext;+ readonly lineage?: InstrumentationParentLineage;+ readonly parentSpanId: string;+ readonly rootSessionId: string;+ readonly sequence: number;+ readonly startTimeMs: number;+ readonly terminal?: {+ readonly error?: unknown;+ readonly type: InstrumentationTurnTerminalEvent["type"];+ };+}++/** Provider-owned serializable storage for durable agent trace state. */+export interface AgentTraceStateStore {+ deleteSession(sessionId: string): void | PromiseLike<void>;+ deleteTurn(sessionId: string, turnId: string): void | PromiseLike<void>;+ getSession(+ sessionId: string,+ ): AgentSessionTraceState | undefined | PromiseLike<AgentSessionTraceState | undefined>;+ getTurn(+ sessionId: string,+ turnId: string,+ ): AgentTurnTraceState | undefined | PromiseLike<AgentTurnTraceState | undefined>;+ setSession(sessionId: string, state: AgentSessionTraceState): void | PromiseLike<void>;+ setTurn(sessionId: string, turnId: string, state: AgentTurnTraceState): void | PromiseLike<void>;+}++/** In-memory trace state used by tests and non-durable runtimes. */+export class InMemoryAgentTraceStateStore implements AgentTraceStateStore {+ readonly #sessions = new Map<string, AgentSessionTraceState>();+ readonly #turns = new Map<string, AgentTurnTraceState>();++ deleteSession(sessionId: string): void {+ this.#sessions.delete(sessionId);+ }++ deleteTurn(sessionId: string, turnId: string): void {+ this.#turns.delete(turnKey(sessionId, turnId));+ }++ getSession(sessionId: string): AgentSessionTraceState | undefined {+ return this.#sessions.get(sessionId);+ }++ getTurn(sessionId: string, turnId: string): AgentTurnTraceState | undefined {+ return this.#turns.get(turnKey(sessionId, turnId));+ }++ setSession(sessionId: string, state: AgentSessionTraceState): void {+ this.#sessions.set(sessionId, state);+ }++ setTurn(sessionId: string, turnId: string, state: AgentTurnTraceState): void {+ this.#turns.set(turnKey(sessionId, turnId), state);+ }+}++function turnKey(sessionId: string, turnId: string): string {+ return `${sessionId}:${turnId}`;+}diff --git a/packages/eve/src/tracing/local-instrumentation-runtime.scenario.test.ts b/packages/eve/src/tracing/local-instrumentation-runtime.scenario.test.tsindex 2e655a620..f90ab4fa0 100644--- a/packages/eve/src/tracing/local-instrumentation-runtime.scenario.test.ts+++ b/packages/eve/src/tracing/local-instrumentation-runtime.scenario.test.ts@@ -110,6 +110,17 @@ describe("local instrumentation runtime", () => {},]);await runtime.hooks.publish({ scope, type: "attempt.completed" });+ await runtime.hooks.publish({+ sessionId: "session-1",+ turnId: "turn-1",+ type: "turn.completed",+ });+ // Settling the turn emits the turn span with the pre-allocated id.+ await runtime.hooks.publish({+ sessionId: "session-1",+ turnId: "turn-1",+ type: "session.waiting",+ });});await runtime.forceFlush();@@ -139,6 +150,7 @@ describe("local instrumentation runtime", () => {"user.tool-work",]),);+ expect(span(spans, "agent.step").parentSpanId).toBe(span(spans, "agent.turn").spanId);expect(span(spans, "ai.streamText").parentSpanId).toBe(span(spans, "agent.step").spanId);expect(span(spans, "ai.streamText.doStream").parentSpanId).toBe(span(spans, "ai.streamText").spanId,diff --git a/packages/eve/src/tracing/local-instrumentation-runtime.ts b/packages/eve/src/tracing/local-instrumentation-runtime.tsindex e0fb3a2ff..47201de28 100644--- a/packages/eve/src/tracing/local-instrumentation-runtime.ts+++ b/packages/eve/src/tracing/local-instrumentation-runtime.ts@@ -3,6 +3,7 @@ import { registerOTel } from "#compiled/@vercel/otel/index.js";import { ContextAgentTraceStateStore } from "#tracing/agent-trace-context-store.js";import { createAgentOtelInstrumentation } from "#tracing/agent-otel-provider.js";+import { AgentSpanIdGenerator } from "#tracing/agent-span-id-generator.js";import { AgentTraceSpanProcessor } from "#tracing/agent-trace-span-processor.js";import {createInstrumentationHooks,@@ -34,8 +35,10 @@ export function installLocalInstrumentationRuntime(input: {const processor = new AgentTraceSpanProcessor(retention.enabled ? [new LocalTraceSpanProcessor(input.appRoot)] : [],);+ const idGenerator = new AgentSpanIdGenerator();registerOTel({autoDetectResources: false,+ idGenerator,instrumentations: [],propagators: ["none"],serviceName: input.serviceName,@@ -51,6 +54,7 @@ export function installLocalInstrumentationRuntime(input: {const agentOtel = createAgentOtelInstrumentation({captureContent: process.env.EVE_TRACES_CONTENT !== "off",frameworkVersion: input.frameworkVersion,+ idGenerator,stateStore: new ContextAgentTraceStateStore(),tracer: trace.getTracer("eve.agent", input.frameworkVersion),});