Skip to content

armgit / eve

Public Imported from GitHub
0

The Open Framework for Building Agents

feat(eve): record agent.turn spans with real durations in local traces (#1781)

Chad Hietala63a76f00d9b8
diff --git a/.changeset/real-duration-agent-turn-spans.md b/.changeset/real-duration-agent-turn-spans.md
new file mode 100644
index 000000000..23cd4c408
--- /dev/null
+++ b/.changeset/real-duration-agent-turn-spans.md
@@ -0,0 +1,5 @@
+---
+"eve": patch
+---
+
+Local traces now record `agent.turn` with the turn's real duration instead of a zero-duration marker, and the separate `agent.turn.terminal` marker span is gone — terminal and transition events land on the turn span itself. `agent.session` window roots remain zero-duration markers because an idle session never closes.
diff --git a/docs/reference/cli.md b/docs/reference/cli.md
index 02382a662..675b54daa 100644
--- a/docs/reference/cli.md
+++ b/docs/reference/cli.md
@@ -271,7 +271,7 @@ A session long enough to outgrow one trace — far longer than anything you will
One parent turn can dispatch several subagents into the same window, so a child's turn spans name the dispatch: `agent.parent.session.id`, `agent.parent.turn.id`, and `agent.parent.call_id` identify the tool call that created the child, and `agent.subagent.name` the subagent it invoked. Top-level sessions carry none of these.
-`agent.session` and `agent.turn` outlive the worker that opened them, so they are recorded as zero-duration markers. The span tree shows their descendant extent instead; `agent.step` and the model and tool spans beneath it carry real durations. A third-party OTel backend reports zero for the markers.
+Every span carries a real duration except `agent.session`: an idle session never closes, so it is recorded as a zero-duration marker and the span tree shows its descendant extent instead. A turn's span is written when the turn settles, so a running turn shows only its steps.
Model and tool-call spans carry their inputs and outputs — system prompt, prompt messages, and response text for models; call arguments and results for tools — each capped at 32 KB. Set `EVE_TRACES_CONTENT=off` to keep payloads out of the spool.
diff --git a/packages/eve/scripts/vendor-compiled/declarations/@opentelemetry/api.d.ts b/packages/eve/scripts/vendor-compiled/declarations/@opentelemetry/api.d.ts
index 4e1ddf70b..7954271d7 100644
--- a/packages/eve/scripts/vendor-compiled/declarations/@opentelemetry/api.d.ts
+++ b/packages/eve/scripts/vendor-compiled/declarations/@opentelemetry/api.d.ts
@@ -7,7 +7,7 @@ export interface SpanContext {
}
export interface Span {
- addEvent(name: string, attributes?: Attributes): this;
+ addEvent(name: string, attributes?: Attributes, timestamp?: number): this;
end(): void;
recordException(
exception: Error | string | { message?: string; name?: string; stack?: string },
@@ -24,6 +24,7 @@ export interface Tracer {
attributes?: Attributes | undefined;
kind?: SpanKind | undefined;
root?: boolean | undefined;
+ startTime?: number | undefined;
},
context?: Context,
): Span;
diff --git a/packages/eve/scripts/vendor-compiled/declarations/@vercel/otel.d.ts b/packages/eve/scripts/vendor-compiled/declarations/@vercel/otel.d.ts
index b1b1e3145..e6cfcfc8e 100644
--- a/packages/eve/scripts/vendor-compiled/declarations/@vercel/otel.d.ts
+++ b/packages/eve/scripts/vendor-compiled/declarations/@vercel/otel.d.ts
@@ -5,8 +5,14 @@ export interface SpanProcessor {
shutdown(): Promise<void>;
}
+export interface IdGenerator {
+ generateSpanId(): string;
+ generateTraceId(): string;
+}
+
export interface Configuration {
readonly autoDetectResources?: boolean;
+ readonly idGenerator?: IdGenerator;
readonly instrumentations?: readonly unknown[];
readonly propagators?: readonly ["none"];
readonly serviceName?: string;
diff --git a/packages/eve/src/cli/commands/trace.integration.test.ts b/packages/eve/src/cli/commands/trace.integration.test.ts
index 8c16fd4b4..5b60f0feb 100644
--- a/packages/eve/src/cli/commands/trace.integration.test.ts
+++ b/packages/eve/src/cli/commands/trace.integration.test.ts
@@ -189,7 +189,7 @@ describe("eve traces", () => {
const session = "a".repeat(16);
const turn = "b".repeat(16);
const step = "c".repeat(16);
- const terminal = "d".repeat(16);
+ const orphanMarker = "d".repeat(16);
await writeSegment(
root,
TRACE_ONE,
@@ -200,7 +200,7 @@ describe("eve traces", () => {
await writeSegment(
root,
TRACE_ONE,
- span(turn, "agent.turn", 10, 10, session, { "agent.session.id": "session-one" }),
+ span(turn, "agent.turn", 10, 95, session, { "agent.session.id": "session-one" }),
);
await writeSegment(
root,
@@ -210,7 +210,7 @@ describe("eve traces", () => {
await writeSegment(
root,
TRACE_ONE,
- span(terminal, "agent.turn.terminal", 95, 95, turn, {
+ span(orphanMarker, "user.marker", 95, 95, turn, {
"agent.session.id": "session-one",
}),
);
@@ -221,8 +221,8 @@ describe("eve traces", () => {
expect(output.out[0]).toContain("agent.session 85ms");
expect(output.out[0]).toContain("agent.turn 85ms");
expect(output.out[0]).toContain("agent.step 70ms");
- // A marker with no descendants has no extent to borrow.
- expect(output.out[0]).toContain("agent.turn.terminal 0ms");
+ // A zero-duration span with no descendants has no extent to borrow.
+ expect(output.out[0]).toContain("user.marker 0ms");
});
it("shows usage and cost chips on span rows and totals in the header", async () => {
diff --git a/packages/eve/src/cli/commands/trace.ts b/packages/eve/src/cli/commands/trace.ts
index b5d761be7..d7cae8401 100644
--- a/packages/eve/src/cli/commands/trace.ts
+++ b/packages/eve/src/cli/commands/trace.ts
@@ -285,10 +285,11 @@ function renderSpanTree(
/**
* Extent of each span's own range unioned with its descendants', keyed by span id.
*
- * eve records a span whose lifetime crosses a durable worker boundary — a
- * session window or a turn — as a zero-duration marker, because a span object
- * cannot cross that boundary to be ended later. The tree falls back to this
- * extent for those rows so it shows where the time went.
+ * eve records a span with no guaranteed close — an `agent.session` window
+ * root, whose session may idle forever — as a zero-duration marker, because
+ * a span object cannot cross a durable worker boundary to be ended later.
+ * The tree falls back to this extent for those rows so it shows where the
+ * time went.
*/
function subtreeExtents(
spans: readonly LocalTraceSpan[],
diff --git a/packages/eve/src/cli/dev/tui/traces/trace-conversation.ts b/packages/eve/src/cli/dev/tui/traces/trace-conversation.ts
index cff1003da..fbd94f8b3 100644
--- a/packages/eve/src/cli/dev/tui/traces/trace-conversation.ts
+++ b/packages/eve/src/cli/dev/tui/traces/trace-conversation.ts
@@ -20,7 +20,7 @@ import { SURFACE_CLOSE } from "./trace-surfaces.js";
export interface ConversationItem {
readonly kind: "system" | "user" | "assistant" | "tool";
- /** Detail-drawer source for this item (the turn marker span for users). */
+ /** Detail-drawer source for this item (the turn span for users). */
readonly span: LocalTraceSpan;
/** Tool name for tool items. */
readonly name?: string;
diff --git a/packages/eve/src/tracing/agent-otel-provider.test.ts b/packages/eve/src/tracing/agent-otel-provider.test.ts
index 3019fe046..d8380faf1 100644
--- a/packages/eve/src/tracing/agent-otel-provider.test.ts
+++ b/packages/eve/src/tracing/agent-otel-provider.test.ts
@@ -10,9 +10,10 @@ import { describe, expect, it } from "vitest";
import { createAiSdkHookBridge } from "#harness/ai-sdk-hook-bridge.js";
import {
createAgentOtelInstrumentation,
- InMemoryAgentTraceStateStore,
SESSION_WINDOW_TURN_LIMIT,
} from "#tracing/agent-otel-provider.js";
+import { AgentSpanIdGenerator } from "#tracing/agent-span-id-generator.js";
+import { InMemoryAgentTraceStateStore } from "#tracing/agent-trace-state.js";
import {
createInstrumentationHooks,
type InstrumentationAttemptScope,
@@ -31,11 +32,14 @@ interface TestRuntime {
function createRuntime(stateStore = new InMemoryAgentTraceStateStore()): TestRuntime {
const exporter = new InMemorySpanExporter();
+ const idGenerator = new AgentSpanIdGenerator();
const provider = new BasicTracerProvider({
+ idGenerator,
spanProcessors: [new SimpleSpanProcessor(exporter)],
});
const agentOtel = createAgentOtelInstrumentation({
frameworkVersion: "test",
+ idGenerator,
stateStore,
tracer: provider.getTracer("eve.agent"),
});
@@ -200,10 +204,24 @@ async function publishTurnStarted(input: {
});
}
+/** The turn span itself is emitted at the turn's session transition. */
+async function completeTurn(
+ hooks: InstrumentationHooks,
+ sessionId: string,
+ turnId: string,
+): Promise<void> {
+ await hooks.publish({ sessionId, turnId, type: "turn.completed" });
+ await hooks.publish({ sessionId, turnId, type: "session.waiting" });
+}
+
function byName(spans: readonly ReadableSpan[], name: string): ReadableSpan[] {
return spans.filter((span) => span.name === name);
}
+function nanos(hrTime: readonly [number, number]): bigint {
+ return BigInt(hrTime[0]) * 1_000_000_000n + BigInt(hrTime[1]);
+}
+
describe("createAgentOtelInstrumentation", () => {
it("emits the agent hierarchy in one session trace", async () => {
const runtime = createRuntime();
@@ -218,7 +236,6 @@ describe("createAgentOtelInstrumentation", () => {
const spans = runtime.exporter.getFinishedSpans();
const turn = byName(spans, "agent.turn")[0]!;
- const turnTerminal = byName(spans, "agent.turn.terminal")[0]!;
const step = byName(spans, "agent.step")[0]!;
const operation = byName(spans, "ai.streamText")[0]!;
const model = byName(spans, "ai.streamText.doStream")[0]!;
@@ -233,18 +250,22 @@ describe("createAgentOtelInstrumentation", () => {
"agent.session.window": 0,
});
expect(turn.parentSpanContext?.spanId).toBe(session.spanContext().spanId);
- expect(turnTerminal.parentSpanContext?.spanId).toBe(turn.spanContext().spanId);
expect(step.parentSpanContext?.spanId).toBe(turn.spanContext().spanId);
expect(operation.parentSpanContext?.spanId).toBe(step.spanContext().spanId);
expect(model.parentSpanContext?.spanId).toBe(operation.spanContext().spanId);
expect(action.parentSpanContext?.spanId).toBe(step.spanContext().spanId);
expect(tool.parentSpanContext?.spanId).toBe(action.spanContext().spanId);
expect(new Set(spans.map((span) => span.spanContext().traceId))).toHaveLength(1);
- expect(turn.events.map((event) => event.name)).toEqual(["turn.started"]);
- expect(turnTerminal.events.map((event) => event.name)).toEqual([
+ expect(turn.events.map((event) => event.name)).toEqual([
+ "turn.started",
"turn.completed",
"session.waiting",
]);
+ // Turn timestamps are millisecond-quantized (`Date.now`), so the end
+ // comparison against the step's sub-millisecond clock gets 1ms of slack.
+ expect(turn.attributes).toMatchObject({ "agent.name": "weather", "agent.session.window": 0 });
+ expect(nanos(turn.startTime)).toBeLessThanOrEqual(nanos(step.startTime));
+ expect(nanos(turn.endTime)).toBeGreaterThanOrEqual(nanos(step.endTime) - 1_000_000n);
expect(step.attributes).toMatchObject({
"agent.framework.name": "eve",
"agent.model.id": "claude-test",
@@ -372,12 +393,15 @@ describe("createAgentOtelInstrumentation", () => {
it("captures nothing when content capture is off", async () => {
const exporter = new InMemorySpanExporter();
+ const idGenerator = new AgentSpanIdGenerator();
const provider = new BasicTracerProvider({
+ idGenerator,
spanProcessors: [new SimpleSpanProcessor(exporter)],
});
const agentOtel = createAgentOtelInstrumentation({
captureContent: false,
frameworkVersion: "test",
+ idGenerator,
stateStore: new InMemoryAgentTraceStateStore(),
tracer: provider.getTracer("eve.agent"),
});
@@ -506,8 +530,11 @@ describe("createAgentOtelInstrumentation", () => {
});
await replacementRuntime.provider.forceFlush();
- const turn = byName(firstRuntime.exporter.getFinishedSpans(), "agent.turn")[0]!;
+ // The replacement's session transition emits the turn span with the span
+ // id the first worker allocated, so descendants from both workers attach.
+ expect(byName(firstRuntime.exporter.getFinishedSpans(), "agent.turn")).toHaveLength(0);
const replacementSpans = replacementRuntime.exporter.getFinishedSpans();
+ const turn = byName(replacementSpans, "agent.turn")[0]!;
const step = byName(replacementSpans, "agent.step")[0]!;
const action = byName(replacementSpans, "agent.action")[0]!;
@@ -554,6 +581,7 @@ describe("createAgentOtelInstrumentation", () => {
turnId: `turn-${sequence}`,
turnSequence: sequence,
});
+ await completeTurn(runtime.hooks, "session-1", `turn-${sequence}`);
}
await runtime.provider.forceFlush();
@@ -596,6 +624,7 @@ describe("createAgentOtelInstrumentation", () => {
turnId: "child-turn-1",
turnSequence: 0,
});
+ await completeTurn(runtime.hooks, "child-1", "child-turn-1");
await runtime.provider.forceFlush();
const spans = runtime.exporter.getFinishedSpans();
@@ -626,6 +655,7 @@ describe("createAgentOtelInstrumentation", () => {
turnId: "child-turn-1",
turnSequence: 0,
});
+ await completeTurn(runtime.hooks, "child-1", "child-turn-1");
await runtime.provider.forceFlush();
const spans = runtime.exporter.getFinishedSpans();
@@ -648,6 +678,7 @@ describe("createAgentOtelInstrumentation", () => {
turnId: "child-turn-1",
turnSequence: 0,
});
+ await completeTurn(runtime.hooks, "child-1", "child-turn-1");
await runtime.provider.forceFlush();
const childTurn = byName(runtime.exporter.getFinishedSpans(), "agent.turn").find(
@@ -665,6 +696,7 @@ describe("createAgentOtelInstrumentation", () => {
turnId: "turn-1",
turnSequence: 0,
});
+ await completeTurn(runtime.hooks, "session-1", "turn-1");
await runtime.provider.forceFlush();
const turn = byName(runtime.exporter.getFinishedSpans(), "agent.turn")[0]!;
diff --git a/packages/eve/src/tracing/agent-otel-provider.ts b/packages/eve/src/tracing/agent-otel-provider.ts
index 73ac55810..548bc237c 100644
--- a/packages/eve/src/tracing/agent-otel-provider.ts
+++ b/packages/eve/src/tracing/agent-otel-provider.ts
@@ -16,6 +16,8 @@ import {
textContentAttribute,
toolResultsContentAttribute,
} from "#tracing/agent-otel-content.js";
+import type { AgentSpanIdGenerator } from "#tracing/agent-span-id-generator.js";
+import type { AgentSessionTraceState, AgentTraceStateStore } from "#tracing/agent-trace-state.js";
import type {
InstrumentationAttemptMetadataEvent,
InstrumentationAttemptScope,
@@ -52,70 +54,6 @@ interface ToolSpanState extends SpanState {
/** Sized so an ordinary session stays one trace and only an outsized one rolls. */
export const SESSION_WINDOW_TURN_LIMIT = 200;
-export interface AgentSessionTraceState {
- readonly agentName?: string;
- readonly channelKind?: string;
- readonly context: SpanContext;
- readonly rootSessionId: string;
- readonly turnsInWindow: number;
- readonly window: number;
-}
-
-export interface AgentTurnTraceState {
- readonly context: SpanContext;
- readonly rootSessionId: string;
- readonly sequence: number;
- readonly terminal?: {
- readonly error?: unknown;
- readonly type: InstrumentationTurnTerminalEvent["type"];
- };
-}
-
-/** Provider-owned serializable storage for durable agent trace state. */
-export interface AgentTraceStateStore {
- deleteSession(sessionId: string): void | PromiseLike<void>;
- deleteTurn(sessionId: string, turnId: string): void | PromiseLike<void>;
- getSession(
- sessionId: string,
- ): AgentSessionTraceState | undefined | PromiseLike<AgentSessionTraceState | undefined>;
- getTurn(
- sessionId: string,
- turnId: string,
- ): AgentTurnTraceState | undefined | PromiseLike<AgentTurnTraceState | undefined>;
- setSession(sessionId: string, state: AgentSessionTraceState): void | PromiseLike<void>;
- setTurn(sessionId: string, turnId: string, state: AgentTurnTraceState): void | PromiseLike<void>;
-}
-
-/** In-memory trace state used by tests and non-durable runtimes. */
-export class InMemoryAgentTraceStateStore implements AgentTraceStateStore {
- readonly #sessions = new Map<string, AgentSessionTraceState>();
- readonly #turns = new Map<string, AgentTurnTraceState>();
-
- deleteSession(sessionId: string): void {
- this.#sessions.delete(sessionId);
- }
-
- deleteTurn(sessionId: string, turnId: string): void {
- this.#turns.delete(turnKey(sessionId, turnId));
- }
-
- getSession(sessionId: string): AgentSessionTraceState | undefined {
- return this.#sessions.get(sessionId);
- }
-
- getTurn(sessionId: string, turnId: string): AgentTurnTraceState | undefined {
- return this.#turns.get(turnKey(sessionId, turnId));
- }
-
- setSession(sessionId: string, state: AgentSessionTraceState): void {
- this.#sessions.set(sessionId, state);
- }
-
- setTurn(sessionId: string, turnId: string, state: AgentTurnTraceState): void {
- this.#turns.set(turnKey(sessionId, turnId), state);
- }
-}
-
export interface AgentOtelInstrumentationInput {
/**
* Capture model prompts/responses and tool call inputs/outputs as span
@@ -124,6 +62,7 @@ export interface AgentOtelInstrumentationInput {
*/
readonly captureContent?: boolean;
readonly frameworkVersion: string;
+ readonly idGenerator: AgentSpanIdGenerator;
readonly stateStore: AgentTraceStateStore;
readonly tracer: Tracer;
}
@@ -164,34 +103,27 @@ export function createAgentOtelInstrumentation(
type: "session.started",
}),
);
- const span = input.tracer.startSpan(
- "agent.turn",
- {
- attributes: {
- "agent.framework.name": "eve",
- "agent.framework.version": input.frameworkVersion,
- "agent.name": session.agentName,
- ...parentLineageAttributes(event.parentLineage),
- "agent.root.session.id": event.rootSessionId,
- "agent.session.id": event.sessionId,
- "agent.session.window": session.window,
- "agent.turn.id": event.turnId,
- "agent.turn.sequence": event.sequence,
- },
- },
- contextFromSpanContext(session.context),
- );
- span.addEvent("turn.started");
+ // The turn outlives this worker, so no live span can cover it: the span
+ // id is allocated now for descendants to parent through, and the span
+ // itself is emitted at the turn's session transition.
+ const turnContext: SpanContext = {
+ isRemote: false,
+ spanId: input.idGenerator.allocateSpanId(),
+ traceFlags: session.context.traceFlags,
+ traceId: session.context.traceId,
+ };
await input.stateStore.setSession(event.sessionId, {
...session,
turnsInWindow: session.turnsInWindow + 1,
});
await input.stateStore.setTurn(event.sessionId, event.turnId, {
- context: span.spanContext(),
+ context: turnContext,
+ lineage: event.parentLineage,
+ parentSpanId: session.context.spanId,
rootSessionId: event.rootSessionId,
sequence: event.sequence,
+ startTimeMs: Date.now(),
});
- span.end();
};
const onAttemptStarted = async (event: InstrumentationAttemptStartedEvent): Promise<void> => {
@@ -269,18 +201,33 @@ export function createAgentOtelInstrumentation(
if (event.turnId !== undefined) {
const turn = await input.stateStore.getTurn(event.sessionId, event.turnId);
if (turn !== undefined) {
- const span = input.tracer.startSpan(
- "agent.turn.terminal",
- {
- attributes: {
- "agent.root.session.id": turn.rootSessionId,
- "agent.session.id": event.sessionId,
- "agent.turn.id": event.turnId,
- "agent.turn.sequence": turn.sequence,
+ const session = await input.stateStore.getSession(event.sessionId);
+ const span = input.idGenerator.withSpanId(turn.context.spanId, () =>
+ input.tracer.startSpan(
+ "agent.turn",
+ {
+ attributes: {
+ "agent.framework.name": "eve",
+ "agent.framework.version": input.frameworkVersion,
+ "agent.name": session?.agentName,
+ ...parentLineageAttributes(turn.lineage),
+ "agent.root.session.id": turn.rootSessionId,
+ "agent.session.id": event.sessionId,
+ "agent.session.window": session?.window,
+ "agent.turn.id": event.turnId,
+ "agent.turn.sequence": turn.sequence,
+ },
+ startTime: turn.startTimeMs,
},
- },
- contextFromSpanContext(turn.context),
+ contextFromSpanContext({
+ isRemote: false,
+ spanId: turn.parentSpanId,
+ traceFlags: turn.context.traceFlags,
+ traceId: turn.context.traceId,
+ }),
+ ),
);
+ span.addEvent("turn.started", undefined, turn.startTimeMs);
if (turn.terminal !== undefined) {
span.addEvent(turn.terminal.type);
if (turn.terminal.type === "turn.failed") {
@@ -466,8 +413,9 @@ export function createAgentOtelInstrumentation(
root: true,
});
span.addEvent(window.index === 0 ? "session.started" : "session.window.opened");
- // The window outlives this worker, so the root is recorded as a marker and
- // later spans parent through its persisted context, as `agent.turn` does.
+ // The window outlives this worker and has no guaranteed close — an idle
+ // session never ends — so the root is recorded as a zero-duration marker
+ // and later spans parent through its persisted context.
span.end();
return span.spanContext();
};
@@ -573,10 +521,6 @@ export function createAgentOtelInstrumentation(
}
}
-function turnKey(sessionId: string, turnId: string): string {
- return `${sessionId}:${turnId}`;
-}
-
function parentLineageAttributes(
lineage: InstrumentationParentLineage | undefined,
): Record<string, string> {
diff --git a/packages/eve/src/tracing/agent-span-id-generator.ts b/packages/eve/src/tracing/agent-span-id-generator.ts
new file mode 100644
index 000000000..740ea8477
--- /dev/null
+++ b/packages/eve/src/tracing/agent-span-id-generator.ts
@@ -0,0 +1,50 @@
+/**
+ * Id generator shared by `registerOTel` and the agent OTel provider. A span
+ * whose lifetime crosses durable worker boundaries (`agent.turn`) is emitted
+ * at its terminal; priming the next span id lets that span carry the
+ * pre-allocated id its descendants already parented to.
+ */
+export class AgentSpanIdGenerator {
+ #primedSpanId: string | undefined;
+
+ /** Reserves a span id for a span emitted later via {@link withSpanId}. */
+ allocateSpanId(): string {
+ return randomHexId(16);
+ }
+
+ generateSpanId(): string {
+ const primed = this.#primedSpanId;
+ if (primed !== undefined) {
+ this.#primedSpanId = undefined;
+ return primed;
+ }
+ return randomHexId(16);
+ }
+
+ generateTraceId(): string {
+ return randomHexId(32);
+ }
+
+ /** Runs `startSpan` so the next span started carries `spanId`. */
+ withSpanId<T>(spanId: string, startSpan: () => T): T {
+ this.#primedSpanId = spanId;
+ try {
+ return startSpan();
+ } finally {
+ this.#primedSpanId = undefined;
+ }
+ }
+}
+
+const HEX_DIGITS = "0123456789abcdef";
+
+function randomHexId(length: number): string {
+ let id: string;
+ do {
+ id = "";
+ for (let index = 0; index < length; index += 1) {
+ id += HEX_DIGITS[Math.trunc(Math.random() * 16)];
+ }
+ } while (!/[1-9a-f]/u.test(id));
+ return id;
+}
diff --git a/packages/eve/src/tracing/agent-trace-context-store.test.ts b/packages/eve/src/tracing/agent-trace-context-store.test.ts
index 66e2d5d2d..9357f1446 100644
--- a/packages/eve/src/tracing/agent-trace-context-store.test.ts
+++ b/packages/eve/src/tracing/agent-trace-context-store.test.ts
@@ -22,8 +22,16 @@ describe("ContextAgentTraceStateStore", () => {
});
store.setTurn("session-1", "turn-1", {
context: spanContext("1", "3"),
+ lineage: {
+ callId: "call-1",
+ sessionId: "parent-session",
+ subagentName: "researcher",
+ turnId: "parent-turn",
+ },
+ parentSpanId: "2".repeat(16),
rootSessionId: "session-1",
sequence: 0,
+ startTimeMs: 1_700_000_000_000,
terminal: { error: new Error("failed"), type: "turn.failed" },
});
});
@@ -35,6 +43,16 @@ describe("ContextAgentTraceStateStore", () => {
expect(store.getSession("session-1")?.context).toEqual(spanContext("1", "2"));
expect(store.getSession("session-1")).toMatchObject({ turnsInWindow: 3, window: 1 });
expect(store.getTurn("session-1", "turn-1")?.context).toEqual(spanContext("1", "3"));
+ expect(store.getTurn("session-1", "turn-1")).toMatchObject({
+ lineage: {
+ callId: "call-1",
+ sessionId: "parent-session",
+ subagentName: "researcher",
+ turnId: "parent-turn",
+ },
+ parentSpanId: "2".repeat(16),
+ startTimeMs: 1_700_000_000_000,
+ });
expect(store.getTurn("session-1", "turn-1")?.terminal?.error).toMatchObject({
message: "failed",
});
@@ -52,8 +70,10 @@ describe("ContextAgentTraceStateStore", () => {
});
store.setTurn("session-1", "turn-1", {
context: spanContext("1", "3"),
+ parentSpanId: "2".repeat(16),
rootSessionId: "session-1",
sequence: 0,
+ startTimeMs: 1_700_000_000_000,
});
store.deleteTurn("session-1", "turn-1");
diff --git a/packages/eve/src/tracing/agent-trace-context-store.ts b/packages/eve/src/tracing/agent-trace-context-store.ts
index 1efee94d8..16fb85431 100644
--- a/packages/eve/src/tracing/agent-trace-context-store.ts
+++ b/packages/eve/src/tracing/agent-trace-context-store.ts
@@ -2,11 +2,12 @@ import type { SpanContext } from "#compiled/@opentelemetry/api/index.js";
import { contextStorage, loadContext } from "#context/container.js";
import { ContextKey } from "#context/key.js";
+import type { InstrumentationParentLineage } from "#harness/instrumentation-lifecycle.js";
import type {
AgentSessionTraceState,
AgentTraceStateStore,
AgentTurnTraceState,
-} from "#tracing/agent-otel-provider.js";
+} from "#tracing/agent-trace-state.js";
interface AgentTraceContextState {
readonly sessions: Readonly<Record<string, AgentSessionTraceState>>;
@@ -136,10 +137,16 @@ function deserializeState(data: unknown): AgentTraceContextState {
});
const turns = deserializeRecord(data.turns, (value) => {
if (!isRecord(value) || !isSpanContext(value.context)) return undefined;
+ if (typeof value.parentSpanId !== "string" || typeof value.startTimeMs !== "number") {
+ return undefined;
+ }
return {
context: value.context,
+ lineage: deserializeLineage(value.lineage),
+ parentSpanId: value.parentSpanId,
rootSessionId: typeof value.rootSessionId === "string" ? value.rootSessionId : "",
sequence: typeof value.sequence === "number" ? value.sequence : 0,
+ startTimeMs: value.startTimeMs,
terminal: deserializeTerminal(value.terminal),
} satisfies AgentTurnTraceState;
});
@@ -159,6 +166,23 @@ function deserializeRecord<T>(
return result;
}
+function deserializeLineage(value: unknown): InstrumentationParentLineage | undefined {
+ if (
+ !isRecord(value) ||
+ typeof value.callId !== "string" ||
+ typeof value.sessionId !== "string" ||
+ typeof value.turnId !== "string"
+ ) {
+ return undefined;
+ }
+ return {
+ callId: value.callId,
+ sessionId: value.sessionId,
+ subagentName: typeof value.subagentName === "string" ? value.subagentName : undefined,
+ turnId: value.turnId,
+ };
+}
+
function deserializeTerminal(value: unknown): AgentTurnTraceState["terminal"] {
if (!isRecord(value) || typeof value.type !== "string") return undefined;
const type = value.type;
diff --git a/packages/eve/src/tracing/agent-trace-state.ts b/packages/eve/src/tracing/agent-trace-state.ts
new file mode 100644
index 000000000..6564b9685
--- /dev/null
+++ b/packages/eve/src/tracing/agent-trace-state.ts
@@ -0,0 +1,77 @@
+import type { SpanContext } from "#compiled/@opentelemetry/api/index.js";
+
+import type {
+ InstrumentationParentLineage,
+ InstrumentationTurnTerminalEvent,
+} from "#harness/instrumentation-lifecycle.js";
+
+export interface AgentSessionTraceState {
+ readonly agentName?: string;
+ readonly channelKind?: string;
+ readonly context: SpanContext;
+ readonly rootSessionId: string;
+ readonly turnsInWindow: number;
+ readonly window: number;
+}
+
+export interface AgentTurnTraceState {
+ readonly context: SpanContext;
+ readonly lineage?: InstrumentationParentLineage;
+ readonly parentSpanId: string;
+ readonly rootSessionId: string;
+ readonly sequence: number;
+ readonly startTimeMs: number;
+ readonly terminal?: {
+ readonly error?: unknown;
+ readonly type: InstrumentationTurnTerminalEvent["type"];
+ };
+}
+
+/** Provider-owned serializable storage for durable agent trace state. */
+export interface AgentTraceStateStore {
+ deleteSession(sessionId: string): void | PromiseLike<void>;
+ deleteTurn(sessionId: string, turnId: string): void | PromiseLike<void>;
+ getSession(
+ sessionId: string,
+ ): AgentSessionTraceState | undefined | PromiseLike<AgentSessionTraceState | undefined>;
+ getTurn(
+ sessionId: string,
+ turnId: string,
+ ): AgentTurnTraceState | undefined | PromiseLike<AgentTurnTraceState | undefined>;
+ setSession(sessionId: string, state: AgentSessionTraceState): void | PromiseLike<void>;
+ setTurn(sessionId: string, turnId: string, state: AgentTurnTraceState): void | PromiseLike<void>;
+}
+
+/** In-memory trace state used by tests and non-durable runtimes. */
+export class InMemoryAgentTraceStateStore implements AgentTraceStateStore {
+ readonly #sessions = new Map<string, AgentSessionTraceState>();
+ readonly #turns = new Map<string, AgentTurnTraceState>();
+
+ deleteSession(sessionId: string): void {
+ this.#sessions.delete(sessionId);
+ }
+
+ deleteTurn(sessionId: string, turnId: string): void {
+ this.#turns.delete(turnKey(sessionId, turnId));
+ }
+
+ getSession(sessionId: string): AgentSessionTraceState | undefined {
+ return this.#sessions.get(sessionId);
+ }
+
+ getTurn(sessionId: string, turnId: string): AgentTurnTraceState | undefined {
+ return this.#turns.get(turnKey(sessionId, turnId));
+ }
+
+ setSession(sessionId: string, state: AgentSessionTraceState): void {
+ this.#sessions.set(sessionId, state);
+ }
+
+ setTurn(sessionId: string, turnId: string, state: AgentTurnTraceState): void {
+ this.#turns.set(turnKey(sessionId, turnId), state);
+ }
+}
+
+function turnKey(sessionId: string, turnId: string): string {
+ return `${sessionId}:${turnId}`;
+}
diff --git a/packages/eve/src/tracing/local-instrumentation-runtime.scenario.test.ts b/packages/eve/src/tracing/local-instrumentation-runtime.scenario.test.ts
index 2e655a620..f90ab4fa0 100644
--- a/packages/eve/src/tracing/local-instrumentation-runtime.scenario.test.ts
+++ b/packages/eve/src/tracing/local-instrumentation-runtime.scenario.test.ts
@@ -110,6 +110,17 @@ describe("local instrumentation runtime", () => {
},
]);
await runtime.hooks.publish({ scope, type: "attempt.completed" });
+ await runtime.hooks.publish({
+ sessionId: "session-1",
+ turnId: "turn-1",
+ type: "turn.completed",
+ });
+ // Settling the turn emits the turn span with the pre-allocated id.
+ await runtime.hooks.publish({
+ sessionId: "session-1",
+ turnId: "turn-1",
+ type: "session.waiting",
+ });
});
await runtime.forceFlush();
@@ -139,6 +150,7 @@ describe("local instrumentation runtime", () => {
"user.tool-work",
]),
);
+ expect(span(spans, "agent.step").parentSpanId).toBe(span(spans, "agent.turn").spanId);
expect(span(spans, "ai.streamText").parentSpanId).toBe(span(spans, "agent.step").spanId);
expect(span(spans, "ai.streamText.doStream").parentSpanId).toBe(
span(spans, "ai.streamText").spanId,
diff --git a/packages/eve/src/tracing/local-instrumentation-runtime.ts b/packages/eve/src/tracing/local-instrumentation-runtime.ts
index e0fb3a2ff..47201de28 100644
--- a/packages/eve/src/tracing/local-instrumentation-runtime.ts
+++ b/packages/eve/src/tracing/local-instrumentation-runtime.ts
@@ -3,6 +3,7 @@ import { registerOTel } from "#compiled/@vercel/otel/index.js";
import { ContextAgentTraceStateStore } from "#tracing/agent-trace-context-store.js";
import { createAgentOtelInstrumentation } from "#tracing/agent-otel-provider.js";
+import { AgentSpanIdGenerator } from "#tracing/agent-span-id-generator.js";
import { AgentTraceSpanProcessor } from "#tracing/agent-trace-span-processor.js";
import {
createInstrumentationHooks,
@@ -34,8 +35,10 @@ export function installLocalInstrumentationRuntime(input: {
const processor = new AgentTraceSpanProcessor(
retention.enabled ? [new LocalTraceSpanProcessor(input.appRoot)] : [],
);
+ const idGenerator = new AgentSpanIdGenerator();
registerOTel({
autoDetectResources: false,
+ idGenerator,
instrumentations: [],
propagators: ["none"],
serviceName: input.serviceName,
@@ -51,6 +54,7 @@ export function installLocalInstrumentationRuntime(input: {
const agentOtel = createAgentOtelInstrumentation({
captureContent: process.env.EVE_TRACES_CONTENT !== "off",
frameworkVersion: input.frameworkVersion,
+ idGenerator,
stateStore: new ContextAgentTraceStateStore(),
tracer: trace.getTracer("eve.agent", input.frameworkVersion),
});