The Open Framework for Building Agents
feat(eve): trace durable runtime actions (#1862)
Chad Hietala29313bee252c
diff --git a/.changeset/durable-action-tracing.md b/.changeset/durable-action-tracing.mdnew file mode 100644index 000000000..91222f207--- /dev/null+++ b/.changeset/durable-action-tracing.md@@ -0,0 +1,5 @@+---+"eve": patch+---++Reconstruct durable `agent.action` spans when runtime actions settle, including across worker replacement, and record each action's exact caller-accepted duration, kind, outcome, stable error code, and subagent usage. Remote eve sessions join the caller action trace through W3C `traceparent`; older receivers may ignore the header. Human approval waits appear as durable `agent.approval` child spans, while chat spans use standard self-contained model input and output attributes.diff --git a/packages/eve/scripts/vendor-compiled/declarations/@opentelemetry/api.d.ts b/packages/eve/scripts/vendor-compiled/declarations/@opentelemetry/api.d.tsindex d7168a656..62d05fac5 100644--- a/packages/eve/scripts/vendor-compiled/declarations/@opentelemetry/api.d.ts+++ b/packages/eve/scripts/vendor-compiled/declarations/@opentelemetry/api.d.ts@@ -1,14 +1,21 @@+export interface TraceState {+ get(key: string): string | undefined;+ serialize(): string;+ set(key: string, value: string): TraceState;+ unset(key: string): TraceState;+}+export interface SpanContext {isRemote?: boolean;spanId: string;traceFlags: number;traceId: string;- traceState?: unknown;+ traceState?: TraceState;}export interface Span {addEvent(name: string, attributes?: Attributes, timestamp?: number): this;- end(): void;+ end(endTime?: number): void;recordException(exception: Error | string | { message?: string; name?: string; stack?: string },): void;@@ -17,12 +24,18 @@ export interface Span {spanContext(): SpanContext;}+export interface Link {+ context: SpanContext;+ attributes?: Attributes;+}+export interface Tracer {startSpan(name: string,options?: {attributes?: Attributes | undefined;kind?: SpanKind | undefined;+ links?: Link[] | undefined;root?: boolean | undefined;startTime?: number | undefined;},@@ -75,6 +88,7 @@ export declare const propagation: {export declare const trace: {getActiveSpan(): Span | undefined;+ getSpan(context: Context): Span | undefined;getTracer(name: string, version?: string): Tracer;getTracerProvider(): unknown;setSpan(context: Context, span: Span): Context;diff --git a/packages/eve/src/cli/commands/trace.integration.test.ts b/packages/eve/src/cli/commands/trace.integration.test.tsindex 5b60f0feb..25851183a 100644--- a/packages/eve/src/cli/commands/trace.integration.test.ts+++ b/packages/eve/src/cli/commands/trace.integration.test.ts@@ -164,7 +164,7 @@ describe("eve traces", () => {root,TRACE_ONE,span(action, "agent.action", 30, 80, step, {- "agent.action.kind": "tool",+ "agent.action.kind": "tool-call","agent.action.name": "\u001B[31mweather\u001B[0m","agent.session.id": "session-one",}),@@ -179,7 +179,7 @@ describe("eve traces", () => {expect(output.out[0]).toContain("agent.turn [turn-1]");expect(output.out[0]).toContain("└─ agent.step [step 0, attempt 0]");- expect(output.out[0]).toContain("└─ agent.action [tool: weather]");+ expect(output.out[0]).toContain("└─ agent.action [tool-call: weather]");expect(output.out[0]).toContain("failed 10ms ERROR");expect(output.out[0]).not.toContain("\u001B");});diff --git a/packages/eve/src/cli/dev/tui/traces/trace-conversation.ts b/packages/eve/src/cli/dev/tui/traces/trace-conversation.tsindex fbd94f8b3..8a977ce3b 100644--- a/packages/eve/src/cli/dev/tui/traces/trace-conversation.ts+++ b/packages/eve/src/cli/dev/tui/traces/trace-conversation.ts@@ -563,7 +563,10 @@ function unwrapJsonString(raw: string | undefined): string | undefined {}function isModelSpan(span: LocalTraceSpan): boolean {- return span.name.startsWith("ai.") && span.name.includes("do");+ return (+ (span.name.startsWith("ai.") && span.name.includes("do")) ||+ span.attributes["gen_ai.operation.name"] === "chat"+ );}/** Extracts tool names from the `ai.response.tool_calls` JSON array. */diff --git a/packages/eve/src/execution/dispatch-runtime-actions-step.integration.test.ts b/packages/eve/src/execution/dispatch-runtime-actions-step.integration.test.tsindex 550fcc2c8..2b271a6fc 100644--- a/packages/eve/src/execution/dispatch-runtime-actions-step.integration.test.ts+++ b/packages/eve/src/execution/dispatch-runtime-actions-step.integration.test.ts@@ -154,16 +154,43 @@ describe("dispatchRuntimeActionsStep child starts", () => {nodeId: "subagents/research",});const writes: Uint8Array[] = [];+ const traceId = "1".repeat(32);+ const actionSpanId = "3".repeat(16);const result = await dispatchRuntimeActionsStep({parentContinuationToken: "turn-inbox",parentWritable: createWritable(writes),- serializedContext: {},+ serializedContext: {+ "eve.harness.agentTrace": {+ actions: {+ "action-1": {+ attemptIndex: 0,+ callId: "call-1",+ kind: "subagent-call",+ name: "research",+ parent: { spanId: "2".repeat(16), traceFlags: 1, traceId },+ rootSessionId: "parent-session",+ sessionId: "parent-session",+ spanId: actionSpanId,+ startTimeMs: 1_700_000_000_000,+ stepIndex: 0,+ turnId: "turn-1",+ },+ },+ sessions: {},+ turns: {},+ },+ },sessionState: BASE_STATE,});expect(result.results).toEqual([]);expect(mocks.createSession).toHaveBeenCalledTimes(1);+ expect(mocks.createSession).toHaveBeenCalledWith(+ expect.objectContaining({+ parentTraceContext: { isRemote: false, spanId: actionSpanId, traceFlags: 1, traceId },+ }),+ );expect(getAgentHandleStore(readResultSessionState(result, session))).toEqual({handles: [{@@ -277,6 +304,8 @@ describe("dispatchRuntimeActionsStep child starts", () => {it("owns a remote child with its confirmed remote address", async () => {const session = createStartSession({ kind: "remote" });+ const traceId = "4".repeat(32);+ const actionSpanId = "5".repeat(16);installContext(session, {definition: REMOTE_REGISTRY_DEFINITION,nodeId: "remote/research",@@ -286,11 +315,36 @@ describe("dispatchRuntimeActionsStep child starts", () => {callbackBaseUrl: "https://caller.example.com",parentContinuationToken: "turn-inbox",parentWritable: createWritable(),- serializedContext: {},+ serializedContext: {+ "eve.harness.agentTrace": {+ actions: {+ "action-1": {+ attemptIndex: 0,+ callId: "call-1",+ kind: "remote-agent-call",+ name: "research",+ parent: { spanId: "2".repeat(16), traceFlags: 1, traceId },+ rootSessionId: "parent-session",+ sessionId: "parent-session",+ spanId: actionSpanId,+ startTimeMs: 1_700_000_000_000,+ stepIndex: 0,+ turnId: "turn-1",+ },+ },+ sessions: {},+ turns: {},+ },+ },sessionState: BASE_STATE,});expect(result.results).toEqual([]);+ expect(mocks.startRemoteAgentSession).toHaveBeenCalledWith(+ expect.objectContaining({+ parentTraceContext: { isRemote: false, spanId: actionSpanId, traceFlags: 1, traceId },+ }),+ );expect(getAgentHandleStore(readResultSessionState(result, session))).toEqual({handles: [expect.objectContaining({diff --git a/packages/eve/src/execution/dispatch-runtime-actions-step.ts b/packages/eve/src/execution/dispatch-runtime-actions-step.tsindex 1d61e38f4..c11c803a2 100644--- a/packages/eve/src/execution/dispatch-runtime-actions-step.ts+++ b/packages/eve/src/execution/dispatch-runtime-actions-step.ts@@ -1,8 +1,4 @@-/**- * Dispatches every pending runtime action for a parked parent session.- * Starts commit ownership before their side effect, so the returned state- * owns every child it may have created.- */+/** Starts or continues pending actions, committing ownership before side effects. */import { buildAdapterContext } from "#channel/adapter-context.js";import { callAdapterEventHandler } from "#channel/adapter.js";@@ -70,7 +66,7 @@ import { buildSubagentRunInput, type SubagentInputSource } from "#execution/subaimport { createWorkflowRuntime, workflowEntryReference } from "#execution/workflow-runtime.js";import { createLogger, logError } from "#internal/logging.js";import { toErrorMessage } from "#shared/errors.js";-import { readSessionTraceContext } from "#tracing/agent-trace-context-store.js";+import * as traceState from "#tracing/agent-trace-context-store.js";import { resolveSubagentDepth } from "#harness/subagent-depth.js";import { getDynamicSubagentSelection } from "#context/dynamic-subagent-lifecycle.js";import { resolveEffectiveAgentRuntime } from "#execution/effective-agent-config.js";@@ -157,24 +153,19 @@ export async function dispatchRuntimeActionsStep(input: {const initiatorAuth = ctx.get(InitiatorAuthKey) ?? null;const adapterCtx = buildAdapterContext(adapter, ctx);- // Read here, not in the child: trace state is scoped to one session's- // context, so this is the last place the parent's window is visible.- const parentTraceContext = readSessionTraceContext(input.serializedContext, session.sessionId);+ // This is the last place the parent's session-scoped trace state is visible.+ const sessionTraceContext = traceState.readSessionTraceContext(+ input.serializedContext,+ session.sessionId,+ );const persistentSessions =bundle.resolvedAgent.config?.experimental?.subagentPersistentSessions === true;- // A corrupt handle store throws; surface that before anything dispatches.- // A mid-loop throw after a sibling started would durably replay the whole- // batch and re-dispatch that sibling.+ // Fail before dispatch; replaying after a sibling starts would duplicate it.getAgentHandleStore(durableSession.state);const plan = planDispatch({ actions: batch.actions, bundle, ctx, session });- // Acquired only once preflight can no longer throw, so a planning failure- // never leaks the writer lock.const writer = input.parentWritable.getWriter();- // Split the parent's remaining token quota across the batch's freshly- // started local subagents, the children that actually receive an enforced- // cap. Continuations already run under their own budget, and remote agents- // run on their own deployment under their own limits, so neither dilutes- // the local shares.+ // Only fresh local starts share the remaining quota; continuations and+ // remote agents already enforce their own budgets.const fanoutSize = plan.filter((entry) => entry.kind === "start" && entry.target.kind === "local",).length;@@ -217,7 +208,13 @@ export async function dispatchRuntimeActionsStep(input: {fanoutSize,initiatorAuth,parentContinuationToken: input.parentContinuationToken,- parentTraceContext,+ parentTraceContext:+ traceState.readActionTraceContext(+ input.serializedContext,+ session.sessionId,+ batch.event.turnId,+ entry.target.action.callId,+ ) ?? sessionTraceContext,persistentSessions,session,target: entry.target,@@ -470,6 +467,7 @@ async function startSubagent(input: {dynamicRemoteAgent: input.target.dynamicRemoteAgent,initiatorAuth: input.initiatorAuth,parentContinuationToken: input.parentContinuationToken,+ parentTraceContext: input.parentTraceContext,persistentSessions: input.persistentSessions,session: input.session,});@@ -595,6 +593,7 @@ async function startRemoteSubagent(input: {readonly dynamicRemoteAgent?: DynamicRemoteAgentConfig;readonly initiatorAuth: Parameters<typeof startRemoteAgentSession>[0]["initiatorAuth"];readonly parentContinuationToken: string | undefined;+ readonly parentTraceContext: Parameters<typeof startRemoteAgentSession>[0]["parentTraceContext"];readonly persistentSessions: boolean;readonly session: RuntimeSession;}): Promise<DispatchOutcome> {@@ -648,6 +647,7 @@ async function startRemoteSubagent(input: {callbackBaseUrl,callbackToken: input.parentContinuationToken,initiatorAuth: input.initiatorAuth,+ parentTraceContext: input.parentTraceContext,persistentSessions: input.persistentSessions,remote: resolvedRemote,session: input.session,diff --git a/packages/eve/src/execution/remote-agent-dispatch.test.ts b/packages/eve/src/execution/remote-agent-dispatch.test.tsindex d20a35eca..1cf7aaca0 100644--- a/packages/eve/src/execution/remote-agent-dispatch.test.ts+++ b/packages/eve/src/execution/remote-agent-dispatch.test.ts@@ -92,7 +92,15 @@ describe("startRemoteAgentSession", () => {const childSessionId = await startRemoteAgentSession({action: createAction(),callbackBaseUrl: "https://caller.example.com",- remote: createRemoteAgent(),+ parentTraceContext: {+ spanId: "2".repeat(16),+ traceFlags: 1,+ traceId: "1".repeat(32),+ },+ remote: {+ ...createRemoteAgent(),+ headers: { Traceparent: "00-authored", "x-static": "yes" },+ },session: {agent: {modelReference: { id: "mock/test" },@@ -116,6 +124,7 @@ describe("startRemoteAgentSession", () => {headers: {authorization: "Bearer remote-token","content-type": "application/json",+ traceparent: `00-${"1".repeat(32)}-${"2".repeat(16)}-01`,"x-static": "yes",},method: "POST",diff --git a/packages/eve/src/execution/remote-agent-dispatch.ts b/packages/eve/src/execution/remote-agent-dispatch.tsindex e315d3c7a..c36537f0e 100644--- a/packages/eve/src/execution/remote-agent-dispatch.ts+++ b/packages/eve/src/execution/remote-agent-dispatch.ts@@ -6,10 +6,11 @@ import {createEveSessionCancelRoutePath,createEveSessionRoutePath,} from "#protocol/routes.js";-import type { CancelTurnResult, SessionAuthContext } from "#channel/types.js";+import type { CancelTurnResult, SessionAuthContext, SessionTraceContext } from "#channel/types.js";import type { ForwardedPrincipal } from "#channel/forwarded-principal.js";import type { HeadersValue } from "#client/types.js";import { createWorkflowCallbackUrl } from "#execution/workflow-callback-url.js";+import { formatTraceparent } from "#protocol/traceparent.js";import {formatSubagentInput,normalizeRequestedOutputSchema,@@ -56,6 +57,7 @@ export async function startRemoteAgentSession(input: {* in conversation mode so their sessions accept follow-up messages.*/readonly persistentSessions?: boolean;+ readonly parentTraceContext?: SessionTraceContext;readonly remote: ResolvedRuntimeRemoteAgentNode;readonly session: HarnessSession;}): Promise<RemoteAgentSessionCoordinates> {@@ -105,6 +107,13 @@ export async function startRemoteAgentSession(input: {}const headers = await resolveRemoteAgentRequestHeaders(input.remote);+ const traceparent = formatTraceparent(input.parentTraceContext);+ if (traceparent !== undefined) {+ for (const key of Object.keys(headers)) {+ if (key.toLowerCase() === "traceparent") delete headers[key];+ }+ headers["traceparent"] = traceparent;+ }const response = await fetch(createRemoteAgentSessionUrl(input.remote), {body: JSON.stringify(requestBody),headers: {diff --git a/packages/eve/src/protocol/traceparent.test.ts b/packages/eve/src/protocol/traceparent.test.tsnew file mode 100644index 000000000..bc5c20189--- /dev/null+++ b/packages/eve/src/protocol/traceparent.test.ts@@ -0,0 +1,40 @@+import { describe, expect, it } from "vitest";++import { formatTraceparent, parseTraceparent } from "#protocol/traceparent.js";++const TRACE_ID = "1".repeat(32);+const SPAN_ID = "2".repeat(16);++describe("traceparent", () => {+ it("round-trips a remote sampled context", () => {+ const header = formatTraceparent({ spanId: SPAN_ID, traceFlags: 1, traceId: TRACE_ID });+ expect(header).toBe(`00-${TRACE_ID}-${SPAN_ID}-01`);+ expect(parseTraceparent(header!)).toEqual({+ isRemote: true,+ spanId: SPAN_ID,+ traceFlags: 1,+ traceId: TRACE_ID,+ });+ });++ it.each([+ null,+ "",+ `01-${TRACE_ID}-${SPAN_ID}-01`,+ `00-${"0".repeat(32)}-${SPAN_ID}-01`,+ `00-${TRACE_ID}-${"0".repeat(16)}-01`,+ `00-${TRACE_ID}-${SPAN_ID}-0g`,+ `00-${TRACE_ID}-${SPAN_ID}-01,00-${TRACE_ID}-${SPAN_ID}-01`,+ ])("ignores malformed input %p", (value) => {+ expect(parseTraceparent(value)).toBeUndefined();+ });++ it("omits invalid outbound contexts", () => {+ expect(+ formatTraceparent({ spanId: "0".repeat(16), traceFlags: 1, traceId: TRACE_ID }),+ ).toBeUndefined();+ expect(+ formatTraceparent({ spanId: SPAN_ID, traceFlags: 256, traceId: TRACE_ID }),+ ).toBeUndefined();+ });+});diff --git a/packages/eve/src/protocol/traceparent.ts b/packages/eve/src/protocol/traceparent.tsnew file mode 100644index 000000000..26debde84--- /dev/null+++ b/packages/eve/src/protocol/traceparent.ts@@ -0,0 +1,43 @@+export interface TraceparentContext {+ readonly isRemote?: boolean;+ readonly spanId: string;+ readonly traceFlags: number;+ readonly traceId: string;+}++const TRACEPARENT_PATTERN = /^00-([0-9a-f]{32})-([0-9a-f]{16})-([0-9a-f]{2})$/iu;++/** Formats a W3C version-00 traceparent, or omits an invalid context. */+export function formatTraceparent(context: TraceparentContext | undefined): string | undefined {+ if (+ context === undefined ||+ !validId(context.traceId, 32) ||+ !validId(context.spanId, 16) ||+ !Number.isInteger(context.traceFlags) ||+ context.traceFlags < 0 ||+ context.traceFlags > 0xff+ ) {+ return undefined;+ }+ return `00-${context.traceId.toLowerCase()}-${context.spanId.toLowerCase()}-${context.traceFlags.toString(16).padStart(2, "0")}`;+}++/** Parses a W3C version-00 traceparent without rejecting the containing request. */+export function parseTraceparent(value: string | null): TraceparentContext | undefined {+ if (value === null) return undefined;+ const match = TRACEPARENT_PATTERN.exec(value.trim());+ if (match === null) return undefined;+ const traceId = match[1]!.toLowerCase();+ const spanId = match[2]!.toLowerCase();+ if (!validId(traceId, 32) || !validId(spanId, 16)) return undefined;+ return {+ isRemote: true,+ spanId,+ traceFlags: Number.parseInt(match[3]!, 16),+ traceId,+ };+}++function validId(value: string, length: number): boolean {+ return value.length === length && /^[0-9a-f]+$/iu.test(value) && !/^0+$/u.test(value);+}diff --git a/packages/eve/src/public/channels/eve-session-routes.test.ts b/packages/eve/src/public/channels/eve-session-routes.test.tsindex 51c2f4019..9ca305b54 100644--- a/packages/eve/src/public/channels/eve-session-routes.test.ts+++ b/packages/eve/src/public/channels/eve-session-routes.test.ts@@ -52,7 +52,10 @@ describe("eve ID-addressed session routes", () => {const response = await route("POST", "/eve/v1/session")(new Request("https://eve.test/eve/v1/session", {body: JSON.stringify({ message: "hello" }),- headers: { "content-type": "application/json" },+ headers: {+ "content-type": "application/json",+ traceparent: `00-${"1".repeat(32)}-${"2".repeat(16)}-01`,+ },method: "POST",}),args,@@ -64,6 +67,16 @@ describe("eve ID-addressed session routes", () => {sessionId: "wrun_A",status: "accepted",});+ expect(createSession).toHaveBeenCalledWith(+ expect.objectContaining({+ parentTraceContext: {+ isRemote: true,+ spanId: "2".repeat(16),+ traceFlags: 1,+ traceId: "1".repeat(32),+ },+ }),+ );expect(createSession).toHaveBeenCalledWith(expect.not.objectContaining({ continuationToken: expect.anything() }),);diff --git a/packages/eve/src/public/channels/eve.ts b/packages/eve/src/public/channels/eve.tsindex 2a90d3626..6ee754b44 100644--- a/packages/eve/src/public/channels/eve.ts+++ b/packages/eve/src/public/channels/eve.ts@@ -28,6 +28,7 @@ import {EVE_STREAM_TAIL_INDEX_HEADER,EVE_STREAM_VERSION_HEADER,} from "#protocol/message.js";+import { parseTraceparent } from "#protocol/traceparent.js";import {EVE_INFO_ROUTE_PATH,EVE_SESSION_ROUTE_PATH,@@ -246,6 +247,7 @@ export function eveChannel(input: EveChannelInput): EveChannel {const body = parseCreateBody(payload);if (body instanceof Response) return body;+ const parentTraceContext = parseTraceparent(req.headers.get("traceparent"));const policyRejection = checkUploadPolicy(body, uploadPolicy);if (policyRejection !== null) return policyRejection;@@ -279,6 +281,7 @@ export function eveChannel(input: EveChannelInput): EveChannel {outputSchema: body.outputSchema,},mode: body.mode ?? "conversation",+ parentTraceContext,});} catch (error) {const errorId = logError(log, "session-create request failed", error);diff --git a/packages/eve/src/runtime/attributes/emit.ts b/packages/eve/src/runtime/attributes/emit.tsindex c9f53784d..e219484f5 100644--- a/packages/eve/src/runtime/attributes/emit.ts+++ b/packages/eve/src/runtime/attributes/emit.ts@@ -68,6 +68,7 @@ export async function setEveAttributes(attrs: Record<string, EveAttributeValue>)const { setAttributes } = await import("#compiled/@workflow/core/index.js");await setAttributes(normalized, { allowReservedAttributes: true });} catch (error) {+ if (isTerminalRunAttributeError(error)) return;if (!WARNED_ABOUT_TAG_FAILURE) {WARNED_ABOUT_TAG_FAILURE = true;console.warn("[eve] setEveAttributes failed; suppressing further warnings this process.", {@@ -77,3 +78,10 @@ export async function setEveAttributes(attrs: Record<string, EveAttributeValue>)}}}++function isTerminalRunAttributeError(error: unknown): boolean {+ return (+ error instanceof Error &&+ error.message.startsWith("Cannot set attributes on run in terminal state")+ );+}diff --git a/packages/eve/src/tracing/agent-action-instrumentation.ts b/packages/eve/src/tracing/agent-action-instrumentation.tsnew file mode 100644index 000000000..905546258--- /dev/null+++ b/packages/eve/src/tracing/agent-action-instrumentation.ts@@ -0,0 +1,205 @@+import {+ ROOT_CONTEXT,+ SpanStatusCode,+ type Context,+ type Span,+ type SpanContext,+ type Tracer,+ trace,+} from "#compiled/@opentelemetry/api/index.js";++import type {+ InstrumentationActionStartedEvent,+ InstrumentationActionTerminalEvent,+ InstrumentationAttemptScope,+ InstrumentationProviderDefinition,+} from "#harness/instrumentation-lifecycle.js";+import { actionIdempotencyKey } from "#harness/instrumentation-lifecycle.js";+import { contentAttribute } from "#tracing/agent-otel-content.js";+import { setAgentUsage } from "#tracing/agent-otel-usage.js";+import type { AgentSpanIdGenerator } from "#tracing/agent-span-id-generator.js";+import type { AgentActionTraceState, AgentTraceStateStore } from "#tracing/agent-trace-state.js";++export interface AgentActionInstrumentation {+ readonly events: Pick<+ NonNullable<InstrumentationProviderDefinition["events"]>,+ "action.completed" | "action.failed" | "action.started"+ >;+ deleteForSession(sessionId: string): void | PromiseLike<void>;+ deleteForTurn(sessionId: string, turnId: string): void | PromiseLike<void>;+ failForAttempt(scope: InstrumentationAttemptScope, error: unknown): Promise<void>;+ contextFor(+ sessionId: string,+ turnId: string,+ callId: string,+ ): Promise<AgentActionContext | undefined>;+}++export interface AgentActionContext {+ readonly context: Context;+ readonly spanContext: SpanContext;+}++/** Builds durable `agent.action` spans around eve's runtime dispatch boundary. */+export function createAgentActionInstrumentation(input: {+ readonly frameworkVersion: string;+ readonly idGenerator: AgentSpanIdGenerator;+ readonly recordInputs: boolean;+ readonly recordOutputs: boolean;+ readonly resolveParent: (+ event: InstrumentationActionStartedEvent,+ ) =>+ | { readonly context: Context; readonly spanContext: SpanContext }+ | undefined+ | PromiseLike<{ readonly context: Context; readonly spanContext: SpanContext } | undefined>;+ readonly stateStore: AgentTraceStateStore;+ readonly tracer: Tracer;+}): AgentActionInstrumentation {+ const byAttempt = new Map<string, Set<string>>();++ const onStarted = async (event: InstrumentationActionStartedEvent): Promise<void> => {+ const parent = await input.resolveParent(event);+ if (parent === undefined) return;++ const existing = await input.stateStore.getAction(event.idempotencyKey);+ const state: AgentActionTraceState = existing ?? {+ attemptIndex: event.scope.attemptIndex,+ callId: event.callId,+ inputAttribute: input.recordInputs ? contentAttribute(event.input, false) : undefined,+ kind: event.kind,+ name: event.name,+ parent: {+ spanId: parent.spanContext.spanId,+ traceFlags: parent.spanContext.traceFlags,+ traceId: parent.spanContext.traceId,+ },+ rootSessionId: event.scope.rootSessionId ?? event.scope.sessionId,+ sessionId: event.scope.sessionId,+ spanId: input.idGenerator.deriveSpanId(`action:${event.idempotencyKey}`),+ startTimeMs: Date.now(),+ stepIndex: event.scope.stepIndex,+ turnId: event.scope.turnId,+ };+ await input.stateStore.setAction(event.idempotencyKey, state);+ const keys = byAttempt.get(event.scope.attemptId) ?? new Set<string>();+ keys.add(event.idempotencyKey);+ byAttempt.set(event.scope.attemptId, keys);+ };++ const onTerminal = async (event: InstrumentationActionTerminalEvent): Promise<void> => {+ const state = await input.stateStore.getAction(event.idempotencyKey);+ if (state === undefined) return;+ try {+ const span = startSpan(state);+ span.setAttribute("agent.action.outcome", event.outcome);+ if (event.type === "action.failed") {+ if (event.errorCode !== undefined) {+ span.setAttribute("agent.action.error.code", event.errorCode);+ span.setAttribute("error.type", event.errorCode);+ }+ recordError(span, event.error);+ } else if (event.output.type === "error") {+ recordError(span, event.output.error);+ } else {+ if (event.usage !== undefined) setAgentUsage(span, event.usage);+ if (input.recordOutputs) {+ const result = contentAttribute(event.output.output, false);+ if (result !== undefined) span.setAttribute("gen_ai.tool.call.result", result);+ }+ }+ span.end(event.acceptedAtMs);+ } finally {+ await input.stateStore.deleteAction(event.idempotencyKey);+ forget(event.idempotencyKey);+ }+ };++ const startSpan = (state: AgentActionTraceState): Span => {+ const span = input.idGenerator.withSpanId(state.spanId, () =>+ input.tracer.startSpan(+ "agent.action",+ {+ attributes: {+ "agent.action.call_id": state.callId,+ "agent.action.kind": state.kind,+ "agent.action.name": state.name,+ "agent.framework.name": "eve",+ "agent.framework.version": input.frameworkVersion,+ "agent.root.session.id": state.rootSessionId,+ "agent.session.id": state.sessionId,+ "agent.step.attempt": state.attemptIndex,+ "agent.step.index": state.stepIndex,+ "agent.turn.id": state.turnId,+ },+ startTime: state.startTimeMs,+ },+ contextFromActionState(state),+ ),+ );+ if (state.inputAttribute !== undefined) {+ span.setAttribute("gen_ai.tool.call.arguments", state.inputAttribute);+ }+ return span;+ };++ return {+ async contextFor(sessionId, turnId, callId) {+ const directKey = actionIdempotencyKey(sessionId, turnId, callId);+ const direct = await input.stateStore.getAction(directKey);+ if (direct !== undefined) return actionContext(direct);+ const state = await input.stateStore.findAction(sessionId, callId);+ return state === undefined ? undefined : actionContext(state);+ },+ deleteForSession: (sessionId) => input.stateStore.deleteActions(sessionId),+ deleteForTurn: (sessionId, turnId) => input.stateStore.deleteActions(sessionId, turnId),+ async failForAttempt(scope, error) {+ const keys = byAttempt.get(scope.attemptId);+ if (keys === undefined) return;+ byAttempt.delete(scope.attemptId);+ for (const key of keys) {+ const state = await input.stateStore.getAction(key);+ if (state === undefined) continue;+ const span = startSpan(state);+ recordError(span, error);+ span.end();+ await input.stateStore.deleteAction(key);+ }+ },+ events: {+ "action.completed": onTerminal,+ "action.failed": onTerminal,+ "action.started": onStarted,+ },+ };++ function forget(idempotencyKey: string): void {+ for (const [attemptId, keys] of byAttempt) {+ keys.delete(idempotencyKey);+ if (keys.size === 0) byAttempt.delete(attemptId);+ }+ }+}++function actionContext(state: AgentActionTraceState): AgentActionContext {+ const spanContext = {+ isRemote: false,+ spanId: state.spanId,+ traceFlags: state.parent.traceFlags,+ traceId: state.parent.traceId,+ };+ return {+ context: trace.setSpan(ROOT_CONTEXT, trace.wrapSpanContext(spanContext)),+ spanContext,+ };+}++function contextFromActionState(state: AgentActionTraceState): Context {+ return trace.setSpan(ROOT_CONTEXT, trace.wrapSpanContext({ ...state.parent, isRemote: false }));+}++function recordError(span: Span, error: unknown): void {+ if (error instanceof Error) {+ span.recordException(error);+ span.setStatus({ code: SpanStatusCode.ERROR, message: error.message });+ } else span.setStatus({ code: SpanStatusCode.ERROR });+}diff --git a/packages/eve/src/tracing/agent-approval-instrumentation.ts b/packages/eve/src/tracing/agent-approval-instrumentation.tsnew file mode 100644index 000000000..7ab1e2de6--- /dev/null+++ b/packages/eve/src/tracing/agent-approval-instrumentation.ts@@ -0,0 +1,182 @@+import {+ ROOT_CONTEXT,+ SpanStatusCode,+ type Span,+ type SpanContext,+ type Tracer,+ trace,+} from "#compiled/@opentelemetry/api/index.js";++import type {+ InstrumentationHandlerContext,+ InstrumentationInputRequestedEvent,+ InstrumentationInputResolvedEvent,+ InstrumentationProviderDefinition,+} from "#harness/instrumentation-lifecycle.js";+import type { JsonValue } from "#shared/json.js";+import { contentAttribute } from "#tracing/agent-otel-content.js";+import type { AgentActionContext } from "#tracing/agent-action-instrumentation.js";+import type { AgentSpanIdGenerator } from "#tracing/agent-span-id-generator.js";++interface AgentApprovalSpanState {+ readonly actionCallId: string;+ readonly actionName: string;+ readonly attemptIndex: number;+ readonly parent: SpanContext;+ readonly requestAttribute?: string;+ readonly requestId: string;+ readonly rootSessionId: string;+ readonly sessionId: string;+ readonly startTimeMs: number;+ readonly stepIndex: number;+ readonly turnId: string;+}++/** Builds durable approval wait spans under their originating runtime action. */+export function createAgentApprovalInstrumentation(input: {+ readonly actionContextFor: (+ sessionId: string,+ turnId: string,+ callId: string,+ ) => Promise<AgentActionContext | undefined>;+ readonly frameworkVersion: string;+ readonly idGenerator: AgentSpanIdGenerator;+ readonly recordInputs: boolean;+ readonly recordOutputs: boolean;+ readonly tracer: Tracer;+}): Pick<+ NonNullable<InstrumentationProviderDefinition["events"]>,+ "input.requested" | "input.resolved"+> {+ const onRequested = async (+ event: InstrumentationInputRequestedEvent,+ ctx: InstrumentationHandlerContext,+ ): Promise<void> => {+ if (event.kind !== "tool-approval" || readState(ctx.state.get()) !== undefined) return;+ const parent = await input.actionContextFor(+ event.scope.sessionId,+ event.scope.turnId,+ event.action.callId,+ );+ if (parent === undefined) return;+ const state: Record<string, JsonValue> = {+ actionCallId: event.action.callId,+ actionName: event.action.name,+ attemptIndex: event.scope.attemptIndex,+ parent: {+ spanId: parent.spanContext.spanId,+ traceFlags: parent.spanContext.traceFlags,+ traceId: parent.spanContext.traceId,+ },+ requestId: event.requestId,+ rootSessionId: event.scope.rootSessionId ?? event.scope.sessionId,+ sessionId: event.scope.sessionId,+ startTimeMs: Date.now(),+ stepIndex: event.scope.stepIndex,+ turnId: event.scope.turnId,+ };+ const requestAttribute = input.recordInputs+ ? contentAttribute(event.request, false)+ : undefined;+ if (requestAttribute !== undefined) state["requestAttribute"] = requestAttribute;+ ctx.state.set(state);+ };++ const onResolved = (+ event: InstrumentationInputResolvedEvent,+ ctx: InstrumentationHandlerContext,+ ): void => {+ const state = readState(ctx.state.get());+ if (state === undefined) return;+ const span = input.idGenerator.withSpanId(+ input.idGenerator.deriveSpanId(`approval:${event.idempotencyKey}`),+ () =>+ input.tracer.startSpan(+ "agent.approval",+ {+ attributes: {+ "agent.action.call_id": state.actionCallId,+ "agent.action.name": state.actionName,+ "agent.approval.kind": "tool-approval",+ "agent.approval.outcome": event.outcome,+ "agent.approval.request_id": state.requestId,+ "agent.framework.name": "eve",+ "agent.framework.version": input.frameworkVersion,+ "agent.root.session.id": state.rootSessionId,+ "agent.session.id": state.sessionId,+ "agent.step.attempt": state.attemptIndex,+ "agent.step.index": state.stepIndex,+ "agent.turn.id": state.turnId,+ },+ startTime: state.startTimeMs,+ },+ trace.setSpan(ROOT_CONTEXT, trace.wrapSpanContext({ ...state.parent, isRemote: false })),+ ),+ );+ if (state.requestAttribute !== undefined) {+ span.setAttribute("agent.approval.request", state.requestAttribute);+ }+ if (input.recordOutputs && event.response !== undefined) {+ const response = contentAttribute(event.response, false);+ if (response !== undefined) span.setAttribute("agent.approval.response", response);+ }+ if (event.outcome === "failed") recordError(span, event.error);+ span.end();+ };++ return {+ "input.requested": onRequested,+ "input.resolved": onResolved,+ };+}++function readState(value: unknown): AgentApprovalSpanState | undefined {+ if (typeof value !== "object" || value === null || Array.isArray(value)) return undefined;+ const state = value as Record<string, unknown>;+ const parent = state["parent"];+ if (typeof parent !== "object" || parent === null || Array.isArray(parent)) return undefined;+ const parentRecord = parent as Record<string, unknown>;+ if (+ typeof state["actionCallId"] !== "string" ||+ typeof state["actionName"] !== "string" ||+ typeof state["attemptIndex"] !== "number" ||+ typeof parentRecord["spanId"] !== "string" ||+ typeof parentRecord["traceFlags"] !== "number" ||+ typeof parentRecord["traceId"] !== "string" ||+ typeof state["requestId"] !== "string" ||+ typeof state["rootSessionId"] !== "string" ||+ typeof state["sessionId"] !== "string" ||+ typeof state["startTimeMs"] !== "number" ||+ typeof state["stepIndex"] !== "number" ||+ typeof state["turnId"] !== "string"+ ) {+ return undefined;+ }+ const requestAttribute = state["requestAttribute"];+ if (requestAttribute !== undefined && typeof requestAttribute !== "string") return undefined;+ return {+ actionCallId: state["actionCallId"],+ actionName: state["actionName"],+ attemptIndex: state["attemptIndex"],+ parent: {+ isRemote: false,+ spanId: parentRecord["spanId"],+ traceFlags: parentRecord["traceFlags"],+ traceId: parentRecord["traceId"],+ },+ requestAttribute,+ requestId: state["requestId"],+ rootSessionId: state["rootSessionId"],+ sessionId: state["sessionId"],+ startTimeMs: state["startTimeMs"],+ stepIndex: state["stepIndex"],+ turnId: state["turnId"],+ };+}++function recordError(span: Span, error: unknown): void {+ if (error instanceof Error) {+ span.recordException(error);+ span.setStatus({ code: SpanStatusCode.ERROR, message: error.message });+ } else span.setStatus({ code: SpanStatusCode.ERROR });+}diff --git a/packages/eve/src/tracing/agent-otel-content.test.ts b/packages/eve/src/tracing/agent-otel-content.test.tsindex 0984747d6..b81c3a695 100644--- a/packages/eve/src/tracing/agent-otel-content.test.ts+++ b/packages/eve/src/tracing/agent-otel-content.test.ts@@ -2,9 +2,54 @@ import { describe, expect, it } from "vitest";import {CONTENT_ATTRIBUTE_LIMIT,+ genAiInputMessagesAttribute,+ genAiOutputMessagesAttribute,+ genAiSystemInstructionsAttribute,toolResultsContentAttribute,} from "#tracing/agent-otel-content.js";+describe("GenAI message attributes", () => {+ it("formats model input, output, and system instructions for inspectors", () => {+ expect(+ genAiInputMessagesAttribute([+ { content: "hello", role: "user" },+ {+ content: [+ {+ input: { message: "echo" },+ toolCallId: "call-1",+ toolName: "delegate",+ type: "tool-call",+ },+ ],+ role: "assistant",+ },+ ]),+ ).toBe(+ '[{"parts":[{"content":"hello","type":"text"}],"role":"user"},{"parts":[{"arguments":{"message":"echo"},"id":"call-1","name":"delegate","type":"tool_call"}],"role":"assistant"}]',+ );+ expect(genAiSystemInstructionsAttribute("Be concise.")).toBe(+ '[{"content":"Be concise.","type":"text"}]',+ );+ expect(+ genAiOutputMessagesAttribute(+ [+ { text: "Working.", type: "text" },+ {+ callId: "call-1",+ input: { message: "echo" },+ toolName: "delegate",+ type: "tool-call",+ },+ ],+ "tool-calls",+ ),+ ).toBe(+ '[{"finish_reason":"tool_call","parts":[{"content":"Working.","type":"text"},{"arguments":{"message":"echo"},"id":"call-1","name":"delegate","type":"tool_call"}],"role":"assistant"}]',+ );+ });+});+describe("toolResultsContentAttribute", () => {it("returns the full payload when it fits", () => {const json = toolResultsContentAttribute([diff --git a/packages/eve/src/tracing/agent-otel-content.ts b/packages/eve/src/tracing/agent-otel-content.tsindex 12559c47e..9737f2c93 100644--- a/packages/eve/src/tracing/agent-otel-content.ts+++ b/packages/eve/src/tracing/agent-otel-content.ts@@ -53,6 +53,44 @@ export function messagesContentAttribute(messages: unknown): string | undefinedreturn truncateSingleMessage(stripped);}+/** Serializes model messages using the OpenTelemetry GenAI message schema. */+export function genAiInputMessagesAttribute(messages: unknown): string | undefined {+ if (!Array.isArray(messages)) return undefined;+ const formatted = messages.flatMap((message) => {+ if (!isRecord(message) || message.role === "system" || typeof message.role !== "string") {+ return [];+ }+ const parts = semanticParts(message.content);+ return parts.length === 0 ? [] : [{ parts, role: message.role }];+ });+ for (let start = 0; start < formatted.length; start += 1) {+ const json = semanticJsonAttribute(formatted.slice(start));+ if (json !== undefined) return json;+ }+ return semanticJsonAttribute([]);+}++/** Serializes the system prompt using the OpenTelemetry GenAI instruction schema. */+export function genAiSystemInstructionsAttribute(instructions: unknown): string | undefined {+ const text = systemPromptAttribute(instructions);+ return text === undefined ? undefined : semanticJsonAttribute([{ content: text, type: "text" }]);+}++/** Serializes one model response using the OpenTelemetry GenAI message schema. */+export function genAiOutputMessagesAttribute(+ content: readonly unknown[],+ finishReason: string,+): string | undefined {+ const parts = semanticParts(content);+ return semanticJsonAttribute([+ {+ finish_reason: finishReason === "tool-calls" ? "tool_call" : finishReason,+ parts,+ role: "assistant",+ },+ ]);+}+/*** Serializes provider-executed tool results, keeping the attribute valid* JSON under the cap: when the full payload is too big, each entry's input@@ -146,6 +184,75 @@ function stringifyContent(value: unknown): string | undefined {}}+function semanticJsonAttribute(value: unknown): string | undefined {+ const json = stringifyContent(value);+ return json !== undefined && json.length <= CONTENT_ATTRIBUTE_LIMIT ? json : undefined;+}++function semanticParts(content: unknown): Record<string, unknown>[] {+ if (typeof content === "string") return [{ content, type: "text" }];+ if (!Array.isArray(content)) return [];+ const parts: Record<string, unknown>[] = [];+ for (const part of content) {+ const formatted = semanticPart(part);+ if (formatted !== undefined) parts.push(formatted);+ }+ return parts;+}++function semanticPart(value: unknown): Record<string, unknown> | undefined {+ if (!isRecord(value) || typeof value.type !== "string") return undefined;+ switch (value.type) {+ case "text":+ case "reasoning": {+ const content = typeof value.text === "string" ? value.text : value.content;+ return typeof content === "string" ? { content, type: value.type } : undefined;+ }+ case "tool-call":+ return {+ arguments: value.input,+ id: stringValue(value.toolCallId) ?? stringValue(value.callId) ?? null,+ name: value.toolName,+ type: "tool_call",+ };+ case "tool-result":+ return {+ id: stringValue(value.toolCallId) ?? stringValue(value.callId) ?? null,+ response: toolResponse(value.output),+ type: "tool_call_response",+ };+ case "tool-error":+ return {+ id: stringValue(value.toolCallId) ?? stringValue(value.callId) ?? null,+ response: value.error instanceof Error ? value.error.message : value.error,+ type: "tool_call_response",+ };+ default:+ return { type: value.type };+ }+}++function toolResponse(output: unknown): unknown {+ if (!isRecord(output)) return output;+ if (+ (output.type === "text" ||+ output.type === "error-text" ||+ output.type === "json" ||+ output.type === "error-json") &&+ "value" in output+ ) {+ return output.value;+ }+ if (output.type === "execution-denied") {+ return { denied: true, reason: output.reason };+ }+ return output;+}++function stringValue(value: unknown): string | undefined {+ return typeof value === "string" ? value : undefined;+}+function isRecord(value: unknown): value is Record<string, unknown> {return typeof value === "object" && value !== null && !Array.isArray(value);}diff --git a/packages/eve/src/tracing/agent-otel-provider.test.ts b/packages/eve/src/tracing/agent-otel-provider.test.tsindex a5b920303..e5611b1cc 100644--- a/packages/eve/src/tracing/agent-otel-provider.test.ts+++ b/packages/eve/src/tracing/agent-otel-provider.test.ts@@ -7,24 +7,39 @@ import {} from "@opentelemetry/sdk-trace-base";import { describe, expect, it, vi } from "vitest";-import { context } from "#compiled/@opentelemetry/api/index.js";+import {+ ROOT_CONTEXT,+ context,+ trace as runtimeTrace,+} from "#compiled/@opentelemetry/api/index.js";+import { ContextContainer, contextStorage } from "#context/container.js";+import { deserializeContext, serializeContext } from "#context/serialize.js";import { createAiSdkHookBridge } from "#harness/ai-sdk-hook-bridge.js";import { createAgentOtelInstrumentation } from "#tracing/agent-otel-provider.js";import { AgentSpanIdGenerator } from "#tracing/agent-span-id-generator.js";+import { ContextAgentTraceStateStore } from "#tracing/agent-trace-context-store.js";import {+ type AgentTraceStateStore,InMemoryAgentTraceStateStore,SESSION_WINDOW_TURN_LIMIT,} from "#tracing/agent-trace-state.js";import {- attemptIdempotencyKey,createInstrumentationHooks,- sessionIdempotencyKey,- turnIdempotencyKey,+ type InstrumentationActionKind,type InstrumentationAttemptScope,type InstrumentationContextRunner,type InstrumentationHooks,type InstrumentationParentLineage,type InstrumentationTraceContext,+ type InstrumentationUsage,+} from "#harness/instrumentation-lifecycle.js";+import {+ actionIdempotencyKey,+ attemptIdempotencyKey,+ inputIdempotencyKey,+ modelCallIdempotencyKey,+ sessionIdempotencyKey,+ turnIdempotencyKey,} from "#harness/instrumentation-lifecycle.js";interface TestRuntime {@@ -32,31 +47,38 @@ interface TestRuntime {readonly hooks: InstrumentationHooks;readonly provider: BasicTracerProvider;readonly runInContext: InstrumentationContextRunner;+ readonly tracer: ReturnType<BasicTracerProvider["getTracer"]>;}-function createRuntime(stateStore = new InMemoryAgentTraceStateStore()): TestRuntime {+function createRuntime(+ stateStore: AgentTraceStateStore = new InMemoryAgentTraceStateStore(),+): TestRuntime {const exporter = new InMemorySpanExporter();const idGenerator = new AgentSpanIdGenerator();const provider = new BasicTracerProvider({idGenerator,spanProcessors: [new SimpleSpanProcessor(exporter)],});+ const tracer = provider.getTracer("eve.agent");const agentOtel = createAgentOtelInstrumentation({frameworkVersion: "test",idGenerator,stateStore,- tracer: provider.getTracer("eve.agent"),+ tracer,});const hooks = createInstrumentationHooks([agentOtel.hook]);- return { exporter, hooks, provider, runInContext: agentOtel.runInContext };+ return { exporter, hooks, provider, runInContext: agentOtel.runInContext, tracer };}async function emitAttempt(input: {+ readonly actionUsage?: InstrumentationUsage;readonly attemptIndex?: number;readonly attemptError?: Error;readonly hooks: InstrumentationHooks;+ readonly parentTraceContext?: InstrumentationTraceContext;readonly runInContext: InstrumentationContextRunner;readonly providerMetadata?: Readonly<Record<string, unknown>>;+ readonly actionKind?: InstrumentationActionKind;readonly sessionId: string;readonly skipModelTerminal?: boolean;readonly skipToolTerminal?: boolean;@@ -139,6 +161,16 @@ async function emitAttempt(input: {},]);}+ const actionKey = actionIdempotencyKey(input.sessionId, input.turnId, "tool-1");+ await input.hooks.publish({+ callId: "tool-1",+ idempotencyKey: actionKey,+ input: { secret: "value" },+ kind: input.actionKind ?? "tool-call",+ name: "weather",+ scope,+ type: "action.started",+ });await Reflect.apply(bridge.onToolExecutionStart!, bridge, [{callId: "call-1",@@ -163,6 +195,25 @@ async function emitAttempt(input: {: { error: input.toolError, type: "tool-error" },},]);+ await input.hooks.publish(+ input.toolError === undefined+ ? {+ idempotencyKey: actionKey,+ outcome: "completed",+ output: { output: { temperature: 72 }, type: "result" },+ scope,+ type: "action.completed",+ usage: input.actionUsage,+ }+ : {+ error: input.toolError,+ errorCode: "TOOL_CALL_FAILED",+ idempotencyKey: actionKey,+ outcome: "failed",+ scope,+ type: "action.failed",+ },+ );}if (input.providerMetadata !== undefined) {@@ -264,14 +315,21 @@ function nanos(hrTime: readonly [number, number]): bigint {describe("createAgentOtelInstrumentation", () => {it("emits the agent hierarchy in one session trace", async () => {const runtime = createRuntime();+ const delivery = runtime.tracer.startSpan("workflow.delivery");+ const activeContext = runtimeTrace.setSpan(ROOT_CONTEXT, delivery);+ const contextActive = vi.spyOn(context, "active").mockReturnValue(activeContext);const contextWith = vi.spyOn(context, "with");- await emitAttempt({- hooks: runtime.hooks,- runInContext: runtime.runInContext,- sessionId: "session-1",- turnId: "turn-1",- turnSequence: 0,- });+ await context.with(activeContext, () =>+ emitAttempt({+ hooks: runtime.hooks,+ runInContext: runtime.runInContext,+ sessionId: "session-1",+ turnId: "turn-1",+ turnSequence: 0,+ }),+ );+ contextActive.mockRestore();+ delivery.end();const executionParents = contextWith.mock.calls.map(([parent]) => parent);contextWith.mockRestore();await runtime.provider.forceFlush();@@ -280,7 +338,7 @@ describe("createAgentOtelInstrumentation", () => {const turn = byName(spans, "agent.turn")[0]!;const step = byName(spans, "agent.step")[0]!;const operation = byName(spans, "ai.streamText")[0]!;- const model = byName(spans, "ai.streamText.doStream")[0]!;+ const model = byName(spans, "chat claude-test")[0]!;const action = byName(spans, "agent.action")[0]!;const tool = byName(spans, "ai.toolCall")[0]!;@@ -295,6 +353,12 @@ describe("createAgentOtelInstrumentation", () => {expect(turn.parentSpanContext?.spanId).toBe(session.spanContext().spanId);expect(step.parentSpanContext?.spanId).toBe(turn.spanContext().spanId);expect(operation.parentSpanContext?.spanId).toBe(step.spanContext().spanId);+ expect(step.links).toEqual([+ expect.objectContaining({+ attributes: { "eve.link.type": "workflow.delivery" },+ context: delivery.spanContext(),+ }),+ ]);expect(model.parentSpanContext?.spanId).toBe(operation.spanContext().spanId);expect(executionParents.some(@@ -302,14 +366,16 @@ describe("createAgentOtelInstrumentation", () => {apiTrace.getSpan(parent as never)?.spanContext().spanId === model.spanContext().spanId,),).toBe(true);- expect(action.parentSpanContext?.spanId).toBe(step.spanContext().spanId);+ expect(action.parentSpanContext?.spanId).toBe(turn.spanContext().spanId);expect(tool.parentSpanContext?.spanId).toBe(action.spanContext().spanId);- expect(new Set(spans.map((span) => span.spanContext().traceId))).toHaveLength(1);- expect(turn.events.map((event) => event.name)).toEqual([- "turn.started",- "turn.completed",- "session.waiting",- ]);+ expect(+ new Set(+ spans+ .filter((span) => span.name !== "workflow.delivery")+ .map((span) => span.spanContext().traceId),+ ),+ ).toHaveLength(1);+ expect(turn.events.map((event) => event.name)).toEqual(["turn.started", "turn.completed"]);// Turn timestamps are millisecond-quantized (`Date.now`), so the end// comparison against the step's sub-millisecond clock gets 1ms of slack.expect(turn.attributes).toMatchObject({ "agent.name": "weather", "agent.session.window": 0 });@@ -326,14 +392,346 @@ describe("createAgentOtelInstrumentation", () => {"gen_ai.usage.cache_read.input_tokens": 4,});expect(action.attributes).toMatchObject({- "agent.action.kind": "tool",+ "agent.action.kind": "tool-call","agent.action.name": "weather","agent.framework.name": "eve","agent.root.session.id": "session-1",});});- it("ends model and tool spans still open when the step attempt terminates", async () => {+ it("keeps logical ids stable but separates physical redeliveries", async () => {+ const first = createRuntime();+ const redelivery = createRuntime();+ const parentTraceContext: InstrumentationTraceContext = {+ spanId: "b".repeat(16),+ traceFlags: 1,+ traceId: "a".repeat(32),+ };++ for (const runtime of [first, redelivery]) {+ await emitAttempt({+ hooks: runtime.hooks,+ parentTraceContext,+ runInContext: runtime.runInContext,+ sessionId: "session-1",+ turnId: "turn-1",+ turnSequence: 0,+ });+ await runtime.provider.forceFlush();+ }++ const firstSpans = first.exporter.getFinishedSpans();+ const redeliverySpans = redelivery.exporter.getFinishedSpans();+ for (const name of ["agent.turn", "agent.action", "ai.toolCall"]) {+ expect(byName(redeliverySpans, name)[0]!.spanContext().spanId).toBe(+ byName(firstSpans, name)[0]!.spanContext().spanId,+ );+ }+ for (const name of ["agent.step", "ai.streamText", "chat claude-test"]) {+ expect(byName(redeliverySpans, name)[0]!.spanContext().spanId).not.toBe(+ byName(firstSpans, name)[0]!.spanContext().spanId,+ );+ }+ });++ it("parents a tool to its action when SDK telemetry arrives first", async () => {+ const runtime = createRuntime();+ const scope: InstrumentationAttemptScope = {+ attemptId: "session-1:turn-1:0:0",+ attemptIndex: 0,+ sessionId: "session-1",+ stepIndex: 0,+ turnId: "turn-1",+ };+ const actionKey = actionIdempotencyKey(scope.sessionId, scope.turnId, "tool-1");+ const toolKey = `tool:${scope.attemptId}:tool-1:0`;++ await publishTurnStarted({+ hooks: runtime.hooks,+ sessionId: scope.sessionId,+ turnId: scope.turnId,+ turnSequence: 0,+ });+ await runtime.hooks.publish({+ idempotencyKey: attemptIdempotencyKey(scope),+ operation: { modelId: "model", operationId: "ai.streamText", provider: "test" },+ scope,+ type: "step.attempt.started",+ });+ await runtime.hooks.publish({+ callId: "tool-1",+ idempotencyKey: toolKey,+ input: { secret: "value" },+ scope,+ toolName: "weather",+ type: "tool.call.started",+ });+ await runtime.hooks.publish({+ idempotencyKey: toolKey,+ output: { output: "sunny", type: "result" },+ scope,+ type: "tool.call.completed",+ });+ await runtime.hooks.publish({+ callId: "tool-1",+ idempotencyKey: actionKey,+ input: { secret: "value" },+ kind: "tool-call",+ name: "weather",+ scope,+ type: "action.started",+ });+ await runtime.hooks.publish({+ idempotencyKey: actionKey,+ outcome: "completed",+ output: { output: "sunny", type: "result" },+ scope,+ type: "action.completed",+ });+ const uncorrelatedToolKey = `tool:${scope.attemptId}:tool-2:0`;+ await runtime.hooks.publish({+ callId: "tool-2",+ idempotencyKey: uncorrelatedToolKey,+ input: {},+ scope,+ toolName: "final_output",+ type: "tool.call.started",+ });+ await runtime.hooks.publish({+ idempotencyKey: uncorrelatedToolKey,+ output: { output: "done", type: "result" },+ scope,+ type: "tool.call.completed",+ });+ await runtime.hooks.publish({+ idempotencyKey: attemptIdempotencyKey(scope),+ scope,+ type: "step.attempt.completed",+ });+ await runtime.provider.forceFlush();++ const spans = runtime.exporter.getFinishedSpans();+ const action = byName(spans, "agent.action")[0]!;+ const [tool, uncorrelatedTool] = byName(spans, "ai.toolCall");+ expect(tool!.parentSpanContext?.spanId).toBe(action.spanContext().spanId);+ expect(uncorrelatedTool!.parentSpanContext?.spanId).toBe(+ byName(spans, "agent.step")[0]!.spanContext().spanId,+ );+ });++ it("reconstructs a durable action span in a replacement worker", async () => {+ const first = createRuntime(new ContextAgentTraceStateStore());+ const context = new ContextContainer();+ const scope: InstrumentationAttemptScope = {+ attemptId: "session-1:turn-1:0:0",+ attemptIndex: 0,+ sessionId: "session-1",+ stepIndex: 0,+ turnId: "turn-1",+ };+ const actionKey = actionIdempotencyKey(scope.sessionId, scope.turnId, "tool-1");+ const toolKey = `tool:${scope.attemptId}:tool-1:0`;++ await contextStorage.run(context, async () => {+ await publishTurnStarted({+ hooks: first.hooks,+ sessionId: scope.sessionId,+ turnId: scope.turnId,+ turnSequence: 0,+ });+ await first.hooks.publish({+ idempotencyKey: attemptIdempotencyKey(scope),+ operation: { modelId: "model", operationId: "ai.streamText", provider: "test" },+ scope,+ type: "step.attempt.started",+ });+ await first.hooks.publish({+ callId: "tool-1",+ idempotencyKey: actionKey,+ input: { secret: "value" },+ kind: "tool-call",+ name: "weather",+ scope,+ type: "action.started",+ });+ await first.hooks.publish({+ idempotencyKey: attemptIdempotencyKey(scope),+ scope,+ type: "step.attempt.completed",+ });+ });+ await first.provider.forceFlush();+ const firstSpans = first.exporter.getFinishedSpans();+ const step = byName(firstSpans, "agent.step")[0]!;++ await new Promise((resolve) => setTimeout(resolve, 2));+ const acceptedAtMs = Date.now() + 1_000;+ const restored = await deserializeContext(await serializeContext(context));+ const replacement = createRuntime(new ContextAgentTraceStateStore());+ const replacementScope = {+ ...scope,+ attemptId: "session-1:turn-2:0:0",+ turnId: "turn-2",+ };+ await contextStorage.run(restored, async () => {+ // Approval-resumed tools can execute before the replacement AI SDK emits+ // a new step start. The persisted action context is still their parent.+ await replacement.hooks.publish({+ callId: "tool-1",+ idempotencyKey: toolKey,+ input: {},+ scope: replacementScope,+ toolName: "weather",+ type: "tool.call.started",+ });+ await replacement.hooks.publish({+ idempotencyKey: toolKey,+ output: { output: "ok", type: "result" },+ scope: replacementScope,+ type: "tool.call.completed",+ });+ await replacement.hooks.publish({+ acceptedAtMs,+ idempotencyKey: actionKey,+ outcome: "completed",+ output: { output: { temperature: 72 }, type: "result" },+ scope,+ type: "action.completed",+ });+ });+ await replacement.provider.forceFlush();++ const replacementSpans = replacement.exporter.getFinishedSpans();+ const action = byName(replacementSpans, "agent.action")[0]!;+ const tool = byName(replacementSpans, "ai.toolCall")[0]!;+ expect(action.spanContext().spanId).toBe(tool.parentSpanContext?.spanId);+ expect(action.parentSpanContext?.spanId).toBe(step.parentSpanContext?.spanId);+ expect(action.attributes).toMatchObject({+ "agent.action.kind": "tool-call",+ "agent.action.name": "weather",+ "gen_ai.tool.call.arguments": expect.stringContaining("secret"),+ "gen_ai.tool.call.result": expect.stringContaining("temperature"),+ });+ expect(nanos(action.duration)).toBeGreaterThan(0n);+ expect(nanos(action.endTime)).toBe(BigInt(acceptedAtMs) * 1_000_000n);+ });++ it("reconstructs a durable approval span in a replacement worker", async () => {+ const first = createRuntime(new ContextAgentTraceStateStore());+ const context = new ContextContainer();+ const scope: InstrumentationAttemptScope = {+ attemptId: "session-1:turn-1:0:0",+ attemptIndex: 0,+ sessionId: "session-1",+ stepIndex: 0,+ turnId: "turn-1",+ };+ const actionKey = actionIdempotencyKey(scope.sessionId, scope.turnId, "tool-1");+ const inputKey = inputIdempotencyKey(scope.sessionId, scope.turnId, "approval-1");++ await contextStorage.run(context, async () => {+ await publishTurnStarted({+ hooks: first.hooks,+ sessionId: scope.sessionId,+ turnId: scope.turnId,+ turnSequence: 0,+ });+ await first.hooks.publish({+ callId: "tool-1",+ idempotencyKey: actionKey,+ input: { city: "SF" },+ kind: "tool-call",+ name: "weather",+ scope,+ type: "action.started",+ });+ await first.hooks.publish({+ action: { callId: "tool-1", name: "weather" },+ idempotencyKey: inputKey,+ kind: "tool-approval",+ request: { prompt: "Approve weather?" },+ requestId: "approval-1",+ scope,+ type: "input.requested",+ });+ });++ await new Promise((resolve) => setTimeout(resolve, 2));+ const restored = await deserializeContext(await serializeContext(context));+ const replacement = createRuntime(new ContextAgentTraceStateStore());+ await contextStorage.run(restored, async () => {+ await replacement.hooks.publish({+ idempotencyKey: inputKey,+ kind: "tool-approval",+ outcome: "approved",+ requestId: "approval-1",+ response: { optionId: "approve" },+ scope,+ type: "input.resolved",+ });+ await replacement.hooks.publish({+ idempotencyKey: actionKey,+ outcome: "completed",+ output: { output: { temperature: 72 }, type: "result" },+ scope,+ type: "action.completed",+ });+ });+ await replacement.provider.forceFlush();++ const spans = replacement.exporter.getFinishedSpans();+ const approval = byName(spans, "agent.approval")[0]!;+ const action = byName(spans, "agent.action")[0]!;+ expect(approval.parentSpanContext?.spanId).toBe(action.spanContext().spanId);+ expect(approval.status.code).toBe(SpanStatusCode.UNSET);+ expect(approval.attributes).toMatchObject({+ "agent.action.call_id": "tool-1",+ "agent.action.name": "weather",+ "agent.approval.kind": "tool-approval",+ "agent.approval.outcome": "approved",+ "agent.approval.request": expect.stringContaining("Approve weather?"),+ "agent.approval.request_id": "approval-1",+ "agent.approval.response": expect.stringContaining("approve"),+ "agent.session.id": "session-1",+ "agent.step.index": 0,+ "agent.turn.id": "turn-1",+ });+ expect(nanos(approval.duration)).toBeGreaterThan(0n);+ });++ it("does not create approval spans for other input requests", async () => {+ const runtime = createRuntime();+ const scope: InstrumentationAttemptScope = {+ attemptId: "session-1:turn-1:0:0",+ attemptIndex: 0,+ sessionId: "session-1",+ stepIndex: 0,+ turnId: "turn-1",+ };+ const key = inputIdempotencyKey(scope.sessionId, scope.turnId, "question-1");+ await runtime.hooks.publish({+ action: { callId: "question-1", name: "ask_question" },+ idempotencyKey: key,+ kind: "question",+ request: { prompt: "Which region?" },+ requestId: "question-1",+ scope,+ type: "input.requested",+ });+ await runtime.hooks.publish({+ idempotencyKey: key,+ kind: "question",+ outcome: "answered",+ requestId: "question-1",+ response: { text: "west" },+ scope,+ type: "input.resolved",+ });+ await runtime.provider.forceFlush();+ expect(byName(runtime.exporter.getFinishedSpans(), "agent.approval")).toHaveLength(0);+ });++ it("ends SDK spans but leaves durable actions for their own terminal", async () => {const runtime = createRuntime();await emitAttempt({hooks: runtime.hooks,@@ -347,8 +745,8 @@ describe("createAgentOtelInstrumentation", () => {await runtime.provider.forceFlush();const spans = runtime.exporter.getFinishedSpans();- expect(byName(spans, "ai.streamText.doStream")).toHaveLength(1);- expect(byName(spans, "agent.action")).toHaveLength(1);+ expect(byName(spans, "chat claude-test")).toHaveLength(1);+ expect(byName(spans, "agent.action")).toHaveLength(0);expect(byName(spans, "ai.toolCall")).toHaveLength(1);expect(byName(spans, "agent.step")).toHaveLength(1);});@@ -369,7 +767,7 @@ describe("createAgentOtelInstrumentation", () => {await runtime.provider.forceFlush();const spans = runtime.exporter.getFinishedSpans();- for (const name of ["ai.streamText.doStream", "agent.action", "ai.toolCall"]) {+ for (const name of ["chat claude-test", "agent.action", "ai.toolCall"]) {const span = byName(spans, name)[0]!;expect(span.status).toEqual({ code: SpanStatusCode.ERROR, message: error.message });expect(span.events).toContainEqual(@@ -381,6 +779,35 @@ describe("createAgentOtelInstrumentation", () => {}});+ it("labels a subagent action by its kind, not as a plain tool", async () => {+ const runtime = createRuntime();+ await emitAttempt({+ actionUsage: {+ inputTokenDetails: { cacheReadTokens: 3, cacheWriteTokens: 4 },+ inputTokens: 10,+ outputTokens: 5,+ },+ hooks: runtime.hooks,+ actionKind: "subagent-call",+ runInContext: runtime.runInContext,+ sessionId: "session-1",+ turnId: "turn-1",+ turnSequence: 0,+ });+ await runtime.provider.forceFlush();++ const action = runtime.exporter.getFinishedSpans().find((span) => span.name === "agent.action");+ expect(action?.attributes).toMatchObject({+ "agent.action.kind": "subagent-call",+ "agent.action.name": "weather",+ "agent.action.outcome": "completed",+ "agent.usage.input_tokens": 10,+ "agent.usage.output_tokens": 5,+ "gen_ai.usage.cache_creation.input_tokens": 4,+ "gen_ai.usage.cache_read.input_tokens": 3,+ });+ });+it("captures model and tool inputs/outputs on the operation spans", async () => {const runtime = createRuntime();await emitAttempt({@@ -393,7 +820,7 @@ describe("createAgentOtelInstrumentation", () => {await runtime.provider.forceFlush();const spans = runtime.exporter.getFinishedSpans();- const model = byName(spans, "ai.streamText.doStream")[0]!;+ const model = byName(spans, "chat claude-test")[0]!;const tool = byName(spans, "ai.toolCall")[0]!;// Provider transport noise (signatures et al.) is stripped at capture time.expect(model.attributes["ai.prompt.messages"]).toBe(@@ -405,6 +832,17 @@ describe("createAgentOtelInstrumentation", () => {expect(model.attributes["ai.response.finish_reason"]).toBe("tool-calls");expect(model.attributes["ai.response.reasoning"]).toBe("thinking about weather");expect(model.attributes["ai.response.text"]).toBe("Checking the weather.");+ expect(model.attributes).toMatchObject({+ "gen_ai.agent.name": "weather",+ "gen_ai.input.messages":+ '[{"parts":[{"content":"real user text","type":"text"}],"role":"user"}]',+ "gen_ai.operation.name": "chat",+ "gen_ai.output.messages": expect.stringContaining('"finish_reason":"tool_call"'),+ "gen_ai.response.finish_reasons": ["tool-calls"],+ "gen_ai.system_instructions":+ '[{"content":"You are a weather assistant (system prompt).","type":"text"}]',+ });+ expect(model.attributes["agent.input.messages.delta"]).toBeUndefined();// Provider-executed tools never reach the tool loop; their calls and// results are captured off the model response content.expect(model.attributes["ai.response.tool_calls"]).toBe(@@ -415,13 +853,13 @@ describe("createAgentOtelInstrumentation", () => {);expect(tool.attributes["gen_ai.tool.call.arguments"]).toBe('{"secret":"value"}');expect(tool.attributes["gen_ai.tool.call.result"]).toBe('{"temperature":72}');- // Structural spans stay structural: content lives only on the operation spans.- const structural = byName(spans, "agent.action")[0]!;- expect(JSON.stringify(structural.attributes)).not.toContain("secret");- expect(JSON.stringify(structural.attributes)).not.toContain("temperature");+ // Runtime action spans carry content for dispatches that have no SDK tool boundary.+ const action = byName(spans, "agent.action")[0]!;+ expect(action.attributes["gen_ai.tool.call.arguments"]).toContain("secret");+ expect(action.attributes["gen_ai.tool.call.result"]).toContain("temperature");});- it("truncates long conversations from the front, keeping valid JSON and recent messages", async () => {+ it("caps full model input while keeping valid message JSON", async () => {const runtime = createRuntime();const manyMessages = Array.from({ length: 200 }, (_, index) => ({content: `message ${index} ${"x".repeat(200)}`,@@ -478,16 +916,18 @@ describe("createAgentOtelInstrumentation", () => {]);await runtime.provider.forceFlush();- const model = byName(runtime.exporter.getFinishedSpans(), "ai.streamText.doStream")[0]!;+ const model = byName(runtime.exporter.getFinishedSpans(), "chat claude-test")[0]!;const raw = model.attributes["ai.prompt.messages"];expect(typeof raw).toBe("string");expect((raw as string).length).toBeLessThanOrEqual(32 * 1024);const parsed = JSON.parse(raw as string) as Array<Record<string, unknown>>;- const marker = parsed[0]!["eve.truncated"] as { omittedMessages: number };- expect(marker.omittedMessages).toBeGreaterThan(0);- expect(marker.omittedMessages).toBeLessThan(200);+ expect(parsed.length).toBeGreaterThan(1);+ expect(parsed[0]).toMatchObject({+ "eve.truncated": { omittedMessages: expect.any(Number) },+ });expect(JSON.stringify(parsed)).toContain("message 199");expect(JSON.stringify(parsed)).not.toContain("message 0 ");+ expect(model.attributes["agent.input.messages.delta"]).toBeUndefined();});it("captures nothing when content capture is off", async () => {@@ -523,6 +963,56 @@ describe("createAgentOtelInstrumentation", () => {expect(JSON.stringify(spans.map((span) => span.attributes))).not.toContain("system prompt");});+ it("resolves execution context by attempt identity across a scope snapshot", async () => {+ const runtime = createRuntime();+ const scope: InstrumentationAttemptScope = {+ attemptId: "session-1:turn-1:0:0",+ attemptIndex: 0,+ functionId: "weather",+ sessionId: "session-1",+ stepIndex: 0,+ turnId: "turn-1",+ };+ await runtime.hooks.publish({+ agentName: "weather",+ idempotencyKey: sessionIdempotencyKey("session-1"),+ rootSessionId: "session-1",+ sessionId: "session-1",+ type: "session.started",+ });+ await runtime.hooks.publish({+ idempotencyKey: turnIdempotencyKey("session-1", "turn-1"),+ rootSessionId: "session-1",+ sequence: 0,+ sessionId: "session-1",+ turnId: "turn-1",+ type: "turn.started",+ });+ await runtime.hooks.publish({+ idempotencyKey: attemptIdempotencyKey(scope),+ operation: { modelId: "model", operationId: "ai.streamText", provider: "test" },+ scope,+ type: "step.attempt.started",+ });+ const idempotencyKey = modelCallIdempotencyKey(scope, 0);+ await runtime.hooks.publish({+ idempotencyKey,+ input: { messages: [] },+ model: { modelId: "model", provider: "test" },+ scope,+ type: "model.call.started",+ });++ const withSpy = vi.spyOn(context, "with");+ await runtime.runInContext(+ { idempotencyKey, scope: { ...scope }, type: "model.call" },+ async () => undefined,+ );++ expect(withSpy).toHaveBeenCalledOnce();+ withSpy.mockRestore();+ });+it("writes gateway cost attributes on the step span when the gateway reports them", async () => {const runtime = createRuntime();await emitAttempt({@@ -602,6 +1092,21 @@ describe("createAgentOtelInstrumentation", () => {expect(turns).toHaveLength(2);expect(turns[0]!.spanContext().traceId).toBe(turns[1]!.spanContext().traceId);expect(turns.every((turn) => turn.attributes["agent.session.window"] === 0)).toBe(true);+ const firstModel = byName(firstRuntime.exporter.getFinishedSpans(), "chat claude-test")[0]!;+ const secondModel = byName(secondRuntime.exporter.getFinishedSpans(), "chat claude-test")[0]!;+ expect(firstModel.attributes["ai.prompt.system"]).toBe(+ "You are a weather assistant (system prompt).",+ );+ expect(secondModel.attributes["ai.prompt.system"]).toBe(+ "You are a weather assistant (system prompt).",+ );+ expect(secondModel.attributes["gen_ai.system_instructions"]).toBe(+ '[{"content":"You are a weather assistant (system prompt).","type":"text"}]',+ );+ expect(secondModel.attributes["agent.input.messages.delta"]).toBeUndefined();+ expect(secondModel.attributes["ai.prompt.messages"]).toBe(+ '[{"content":"real user text","role":"user"}]',+ );expect([...byName(firstRuntime.exporter.getFinishedSpans(), "agent.session"),...byName(secondRuntime.exporter.getFinishedSpans(), "agent.session"),@@ -639,7 +1144,7 @@ describe("createAgentOtelInstrumentation", () => {const action = byName(replacementSpans, "agent.action")[0]!;expect(step.parentSpanContext?.spanId).toBe(turn.spanContext().spanId);- expect(action.parentSpanContext?.spanId).toBe(step.spanContext().spanId);+ expect(action.parentSpanContext?.spanId).toBe(turn.spanContext().spanId);expect(step.spanContext().traceId).toBe(turn.spanContext().traceId);});@@ -740,6 +1245,31 @@ describe("createAgentOtelInstrumentation", () => {expect(byName(spans, "agent.session")).toHaveLength(1);});+ it("preserves a remote action as the parent of a remote child turn", async () => {+ const runtime = createRuntime();+ const parentTraceContext: InstrumentationTraceContext & { readonly isRemote: true } = {+ isRemote: true,+ spanId: "2".repeat(16),+ traceFlags: 1,+ traceId: "1".repeat(32),+ };+ await publishTurnStarted({+ hooks: runtime.hooks,+ parentTraceContext,+ rootSessionId: "parent-session",+ sessionId: "remote-child",+ turnId: "child-turn-1",+ turnSequence: 0,+ });+ await completeTurn(runtime.hooks, "remote-child", "child-turn-1");+ await runtime.provider.forceFlush();++ const childTurn = byName(runtime.exporter.getFinishedSpans(), "agent.turn")[0]!;+ expect(childTurn.spanContext().traceId).toBe(parentTraceContext.traceId);+ expect(childTurn.parentSpanContext).toMatchObject(parentTraceContext);+ expect(childTurn.attributes["agent.root.session.id"]).toBe("parent-session");+ });+it("attributes a child turn to the exact call that dispatched it", async () => {const runtime = createRuntime();await publishTurnStarted({@@ -869,7 +1399,13 @@ describe("createAgentOtelInstrumentation", () => {await runtime.provider.forceFlush();const spans = runtime.exporter.getFinishedSpans();- expect(byName(spans, "agent.action")[0]!.status.code).toBe(SpanStatusCode.ERROR);+ const action = byName(spans, "agent.action")[0]!;+ expect(action.status.code).toBe(SpanStatusCode.ERROR);+ expect(action.attributes).toMatchObject({+ "agent.action.error.code": "TOOL_CALL_FAILED",+ "agent.action.outcome": "failed",+ "error.type": "TOOL_CALL_FAILED",+ });expect(byName(spans, "agent.turn")[0]!.status.code).toBe(SpanStatusCode.UNSET);});});diff --git a/packages/eve/src/tracing/agent-otel-provider.ts b/packages/eve/src/tracing/agent-otel-provider.tsindex 245f576a7..9ff4b7280 100644--- a/packages/eve/src/tracing/agent-otel-provider.ts+++ b/packages/eve/src/tracing/agent-otel-provider.ts@@ -16,12 +16,19 @@ import {} from "#tracing/agent-trace-state.js";import {contentAttribute,+ genAiInputMessagesAttribute,+ genAiOutputMessagesAttribute,+ genAiSystemInstructionsAttribute,messagesContentAttribute,systemPromptAttribute,textContentAttribute,toolResultsContentAttribute,} from "#tracing/agent-otel-content.js";import type { AgentSpanIdGenerator } from "#tracing/agent-span-id-generator.js";+import { createAgentActionInstrumentation } from "#tracing/agent-action-instrumentation.js";+import { createAgentApprovalInstrumentation } from "#tracing/agent-approval-instrumentation.js";+import { createAgentToolInstrumentation } from "#tracing/agent-tool-instrumentation.js";+import { setAgentUsage } from "#tracing/agent-otel-usage.js";import type {InstrumentationStepAttemptMetadataEvent,InstrumentationAttemptScope,@@ -35,11 +42,8 @@ import type {InstrumentationParentLineage,InstrumentationTraceContext,InstrumentationSessionTransitionEvent,- InstrumentationToolCallStartedEvent,- InstrumentationToolCallTerminalEvent,InstrumentationTurnStartedEvent,InstrumentationTurnTerminalEvent,- InstrumentationUsage,} from "#harness/instrumentation-lifecycle.js";import { sessionIdempotencyKey } from "#harness/instrumentation-lifecycle.js";@@ -53,10 +57,6 @@ interface AttemptSpanState {readonly step: SpanState;}-interface ToolSpanState extends SpanState {- readonly toolSpan: Span;-}-export interface AgentOtelInstrumentationInput {/** Whether any destination requested model prompts and tool inputs. */readonly recordInputs?: boolean;@@ -80,17 +80,49 @@ export function createAgentOtelInstrumentation(): AgentOtelInstrumentation {const recordInputs = input.recordInputs ?? true;const recordOutputs = input.recordOutputs ?? true;- const executionContexts = new WeakMap<- InstrumentationAttemptScope,- { readonly models: Map<string, Context>; readonly tools: Map<string, Context> }- >();+ const executionContexts = new WeakMap<InstrumentationAttemptScope, Map<string, Context>>();const attemptScopes = new Map<string, InstrumentationAttemptScope>();// A serverless turn runs inside one `turnStep` "use step" invocation. If// that worker is lost, Workflow retries the whole step from entry rather// than resuming this callback sequence in a replacement process.const steps = new WeakMap<InstrumentationAttemptScope, AttemptSpanState>();const modelSpans = new WeakMap<InstrumentationAttemptScope, Map<string, SpanState>>();- const toolSpans = new WeakMap<InstrumentationAttemptScope, Map<string, ToolSpanState>>();+ const actions = createAgentActionInstrumentation({+ frameworkVersion: input.frameworkVersion,+ idGenerator: input.idGenerator,+ recordInputs,+ recordOutputs,+ resolveParent: async (event) => {+ const turn = await input.stateStore.getTurn(event.scope.sessionId, event.scope.turnId);+ return turn === undefined+ ? undefined+ : { context: contextFromSpanContext(turn.context), spanContext: turn.context };+ },+ stateStore: input.stateStore,+ tracer: input.tracer,+ });+ const approvals = createAgentApprovalInstrumentation({+ actionContextFor: actions.contextFor,+ frameworkVersion: input.frameworkVersion,+ idGenerator: input.idGenerator,+ recordInputs,+ recordOutputs,+ tracer: input.tracer,+ });+ const tools = createAgentToolInstrumentation({+ actionContextFor: actions.contextFor,+ idGenerator: input.idGenerator,+ recordInputs,+ recordOutputs,+ resolveFallback: (event) => {+ const scope = attemptScopes.get(event.scope.attemptId) ?? event.scope;+ const step = steps.get(scope)?.step;+ return step === undefined+ ? undefined+ : { context: step.context, spanContext: step.span.spanContext() };+ },+ tracer: input.tracer,+ });const onSessionStarted = async (event: InstrumentationSessionStartedEvent): Promise<void> => {await ensureSessionContext(event);@@ -114,7 +146,7 @@ export function createAgentOtelInstrumentation(// itself is emitted at the turn's session transition.const turnContext: SpanContext = {isRemote: false,- spanId: input.idGenerator.allocateSpanId(),+ spanId: input.idGenerator.deriveSpanId(`turn:${event.idempotencyKey}`),traceFlags: session.context.traceFlags,traceId: session.context.traceId,};@@ -125,6 +157,7 @@ export function createAgentOtelInstrumentation(await input.stateStore.setTurn(event.sessionId, event.turnId, {context: turnContext,lineage: event.parentLineage,+ parentIsRemote: session.context.isRemote,parentSpanId: session.context.spanId,rootSessionId: event.rootSessionId,sequence: event.sequence,@@ -136,6 +169,7 @@ export function createAgentOtelInstrumentation(const turn = await input.stateStore.getTurn(event.scope.sessionId, event.scope.turnId);if (turn === undefined) return;const turnContext = contextFromSpanContext(turn.context);+ const activeSpanContext = trace.getSpan(context.active())?.spanContext();const stepSpan = input.tracer.startSpan("agent.step",{@@ -149,6 +183,15 @@ export function createAgentOtelInstrumentation("agent.turn.id": event.scope.turnId,"agent.name": event.scope.functionId,},+ links:+ activeSpanContext === undefined || activeSpanContext.traceId === turn.context.traceId+ ? undefined+ : [+ {+ attributes: { "eve.link.type": "workflow.delivery" },+ context: activeSpanContext,+ },+ ],},turnContext,);@@ -159,9 +202,9 @@ export function createAgentOtelInstrumentation(operationName,{attributes: {- "gen_ai.operation.name": operationName,- "gen_ai.provider.name": event.operation.provider,- "gen_ai.request.model": event.operation.modelId,+ "ai.operation.name": operationName,+ "ai.provider.name": event.operation.provider,+ "ai.request.model": event.operation.modelId,},},stepContext,@@ -177,10 +220,17 @@ export function createAgentOtelInstrumentation(attemptScopes.set(event.scope.attemptId, event.scope);};- const onStepTerminal = (event: InstrumentationStepAttemptTerminalEvent): void => {+ const onStepTerminal = async (event: InstrumentationStepAttemptTerminalEvent): Promise<void> => {const scope = attemptScopes.get(event.scope.attemptId) ?? event.scope;executionContexts.delete(scope);drainOpenSpans({ ...event, scope });+ tools.drain(+ event.scope.attemptId,+ event.type === "step.attempt.failed" ? { error: event.error } : undefined,+ );+ if (event.type === "step.attempt.failed") {+ await actions.failForAttempt(scope, event.error);+ }attemptScopes.delete(event.scope.attemptId);const attempt = steps.get(scope);if (attempt === undefined) return;@@ -199,6 +249,9 @@ export function createAgentOtelInstrumentation(};const onTurnTerminal = async (event: InstrumentationTurnTerminalEvent): Promise<void> => {+ if (event.type === "turn.cancelled" || event.type === "turn.failed") {+ await actions.deleteForTurn(event.sessionId, event.turnId);+ }const turn = await input.stateStore.getTurn(event.sessionId, event.turnId);if (turn === undefined) return;await input.stateStore.setTurn(event.sessionId, event.turnId, {@@ -235,7 +288,7 @@ export function createAgentOtelInstrumentation(startTime: turn.startTimeMs,},contextFromSpanContext({- isRemote: false,+ isRemote: turn.parentIsRemote ?? false,spanId: turn.parentSpanId,traceFlags: turn.context.traceFlags,traceId: turn.context.traceId,@@ -249,7 +302,6 @@ export function createAgentOtelInstrumentation(recordError(span, turn.terminal.error);}}- span.addEvent(event.type);span.end();await input.stateStore.deleteTurn(event.sessionId, event.turnId);}@@ -258,6 +310,7 @@ export function createAgentOtelInstrumentation(// turn that still needs its metadata — so only release session-scoped// state on terminal transitions.if (event.type === "session.completed" || event.type === "session.failed") {+ await actions.deleteForSession(event.sessionId);await input.stateStore.deleteSession(event.sessionId);}};@@ -268,10 +321,11 @@ export function createAgentOtelInstrumentation(attempt.step.span.setAttribute("agent.model.id", event.model.modelId);attempt.step.span.setAttribute("agent.model.provider", event.model.provider);const span = input.tracer.startSpan(- modelSpanName(attempt.operation.name),+ modelSpanName(event.model.modelId),{attributes: {- "gen_ai.operation.name": attempt.operation.name,+ "gen_ai.agent.name": event.scope.functionId,+ "gen_ai.operation.name": "chat","gen_ai.provider.name": event.model.provider,"gen_ai.request.model": event.model.modelId,},@@ -281,26 +335,38 @@ export function createAgentOtelInstrumentation(if (recordInputs) {const messages = messagesContentAttribute(event.input.messages);if (messages !== undefined) span.setAttribute("ai.prompt.messages", messages);+ const genAiMessages = genAiInputMessagesAttribute(event.input.messages);+ if (genAiMessages !== undefined) span.setAttribute("gen_ai.input.messages", genAiMessages);const system = systemPromptAttribute(event.input.instructions);if (system !== undefined) span.setAttribute("ai.prompt.system", system);+ const genAiSystem = genAiSystemInstructionsAttribute(event.input.instructions);+ if (genAiSystem !== undefined) {+ span.setAttribute("gen_ai.system_instructions", genAiSystem);+ }}const state = { context: trace.setSpan(attempt.operation.context, span), span };- getExecutionContexts(event.scope).models.set(event.idempotencyKey, state.context);+ getExecutionContexts(event.scope).set(event.idempotencyKey, state.context);getSpanStates(modelSpans, event.scope).set(event.idempotencyKey, state);};const onModelCallTerminal = (event: InstrumentationModelCallTerminalEvent): void => {- executionContexts.get(event.scope)?.models.delete(event.idempotencyKey);+ executionContexts.get(event.scope)?.delete(event.idempotencyKey);const state = takeSpanState(modelSpans, event.scope, event.idempotencyKey);if (state === undefined) return;if (event.type === "model.call.failed") {recordError(state.span, event.error);} else {- setUsage(state.span, event.usage);+ setAgentUsage(state.span, event.usage);+ state.span.setAttribute("gen_ai.response.finish_reasons", [event.finishReason]);const attempt = steps.get(event.scope);- if (attempt !== undefined) setUsage(attempt.step.span, event.usage);+ if (attempt !== undefined) setAgentUsage(attempt.step.span, event.usage);if (recordOutputs) {state.span.setAttribute("ai.response.finish_reason", event.finishReason);+ const content = event.content;+ const outputMessages = genAiOutputMessagesAttribute(content, event.finishReason);+ if (outputMessages !== undefined) {+ state.span.setAttribute("gen_ai.output.messages", outputMessages);+ }const reasoning = textContentAttribute(event.content.filter((part) => part.type === "reasoning")@@ -352,70 +418,6 @@ export function createAgentOtelInstrumentation(state.span.end();};- const onToolCallStarted = (event: InstrumentationToolCallStartedEvent): void => {- const attempt = steps.get(event.scope);- if (attempt === undefined) return;- const actionSpan = input.tracer.startSpan(- "agent.action",- {- attributes: {- "agent.action.call_id": event.callId,- "agent.action.kind": "tool",- "agent.action.name": event.toolName,- "agent.framework.name": "eve",- "agent.framework.version": input.frameworkVersion,- "agent.root.session.id": event.scope.rootSessionId ?? event.scope.sessionId,- "agent.session.id": event.scope.sessionId,- "agent.step.attempt": event.scope.attemptIndex,- "agent.step.index": event.scope.stepIndex,- "agent.turn.id": event.scope.turnId,- },- },- attempt.step.context,- );- const actionContext = trace.setSpan(attempt.step.context, actionSpan);- const toolSpan = input.tracer.startSpan(- "ai.toolCall",- {- attributes: {- "gen_ai.operation.name": "execute_tool",- "gen_ai.tool.call.id": event.callId,- "gen_ai.tool.name": event.toolName,- },- },- actionContext,- );- if (recordInputs) {- const args = contentAttribute(event.input, false);- if (args !== undefined) toolSpan.setAttribute("gen_ai.tool.call.arguments", args);- }- const state: ToolSpanState = {- context: trace.setSpan(actionContext, toolSpan),- span: actionSpan,- toolSpan,- };- getExecutionContexts(event.scope).tools.set(event.idempotencyKey, state.context);- getSpanStates(toolSpans, event.scope).set(event.idempotencyKey, state);- };-- const onToolCallTerminal = (event: InstrumentationToolCallTerminalEvent): void => {- executionContexts.get(event.scope)?.tools.delete(event.idempotencyKey);- const state = takeSpanState(toolSpans, event.scope, event.idempotencyKey);- if (state === undefined) return;- if (event.type === "tool.call.failed") {- recordError(state.toolSpan, event.error);- recordError(state.span, event.error);- } else if (event.output.type === "error") {- recordError(state.toolSpan, event.output.error);- recordError(state.span, event.output.error);- } else if (recordOutputs) {- const result = contentAttribute(event.output.output, false);- if (result !== undefined) state.toolSpan.setAttribute("gen_ai.tool.call.result", result);- }- state.toolSpan.end();- state.span.end();- };-const openSessionWindow = (window: {readonly agentName?: string;readonly index: number;@@ -509,6 +511,13 @@ export function createAgentOtelInstrumentation(hook: {name: "eve.otel",events: {+ "action.completed": actions.events["action.completed"],+ "action.failed": actions.events["action.failed"],+ async "action.started"(event, ctx) {+ await actions.events["action.started"]!(event, ctx);+ await tools.actionStarted(event);+ },+ ...approvals,"step.attempt.completed": onStepTerminal,"step.attempt.failed": onStepTerminal,"step.attempt.metadata": onStepMetadata,@@ -520,9 +529,7 @@ export function createAgentOtelInstrumentation("session.failed": onSessionTransition,"session.started": onSessionStarted,"session.waiting": onSessionTransition,- "tool.call.completed": onToolCallTerminal,- "tool.call.failed": onToolCallTerminal,- "tool.call.started": onToolCallStarted,+ ...tools.events,"turn.cancelled": onTurnTerminal,"turn.completed": onTurnTerminal,"turn.failed": onTurnTerminal,@@ -534,19 +541,16 @@ export function createAgentOtelInstrumentation(const contexts = executionContexts.get(scope);const parent =operation.type === "model.call"- ? contexts?.models.get(operation.idempotencyKey)- : contexts?.tools.get(operation.idempotencyKey);+ ? contexts?.get(operation.idempotencyKey)+ : tools.contextFor(operation.scope.attemptId, operation.idempotencyKey);return parent === undefined ? execute() : context.with(parent, execute);},};- function getExecutionContexts(scope: InstrumentationAttemptScope): {- readonly models: Map<string, Context>;- readonly tools: Map<string, Context>;- } {+ function getExecutionContexts(scope: InstrumentationAttemptScope): Map<string, Context> {let state = executionContexts.get(scope);if (state === undefined) {- state = { models: new Map(), tools: new Map() };+ state = new Map();executionContexts.set(scope, state);}return state;@@ -558,15 +562,6 @@ export function createAgentOtelInstrumentation(state.span.end();}modelSpans.delete(event.scope);- for (const state of toolSpans.get(event.scope)?.values() ?? []) {- if (event.type === "step.attempt.failed") {- recordError(state.toolSpan, event.error);- recordError(state.span, event.error);- }- state.toolSpan.end();- state.span.end();- }- toolSpans.delete(event.scope);}}@@ -612,9 +607,7 @@ function parentLineageAttributes(function adoptedSpanContext(handed: InstrumentationTraceContext): SpanContext {return {- // Not remote: a subagent crosses the same worker boundary every restored- // session context does, not the wire.- isRemote: false,+ isRemote: "isRemote" in handed && handed.isRemote === true,spanId: handed.spanId,traceFlags: handed.traceFlags,traceId: handed.traceId,@@ -662,29 +655,8 @@ function isRecord(value: unknown): value is Record<string, unknown> {return typeof value === "object" && value !== null && !Array.isArray(value);}-function modelSpanName(operationName: string): string {- return operationName === "ai.generateText"- ? "ai.generateText.doGenerate"- : "ai.streamText.doStream";-}--function setUsage(span: Span, usage: InstrumentationUsage): void {- if (usage.inputTokens !== undefined) {- span.setAttribute("agent.usage.input_tokens", usage.inputTokens);- }- if (usage.outputTokens !== undefined) {- span.setAttribute("agent.usage.output_tokens", usage.outputTokens);- }- // Cached tokens price differently from plain input, so keep the split.- // Named for the OTel GenAI semantic conventions; present only when the- // provider reports details — others emit nothing.- const details = usage.inputTokenDetails;- if (details?.cacheReadTokens !== undefined) {- span.setAttribute("gen_ai.usage.cache_read.input_tokens", details.cacheReadTokens);- }- if (details?.cacheWriteTokens !== undefined) {- span.setAttribute("gen_ai.usage.cache_creation.input_tokens", details.cacheWriteTokens);- }+function modelSpanName(modelId: string): string {+ return `chat ${modelId}`;}function errorText(error: unknown): unknown {diff --git a/packages/eve/src/tracing/agent-otel-usage.ts b/packages/eve/src/tracing/agent-otel-usage.tsnew file mode 100644index 000000000..b6889a9f2--- /dev/null+++ b/packages/eve/src/tracing/agent-otel-usage.ts@@ -0,0 +1,20 @@+import type { Span } from "#compiled/@opentelemetry/api/index.js";++import type { InstrumentationUsage } from "#harness/instrumentation-lifecycle.js";++/** Applies eve's structural token usage attributes to an agent span. */+export function setAgentUsage(span: Span, usage: InstrumentationUsage): void {+ if (usage.inputTokens !== undefined) {+ span.setAttribute("agent.usage.input_tokens", usage.inputTokens);+ }+ if (usage.outputTokens !== undefined) {+ span.setAttribute("agent.usage.output_tokens", usage.outputTokens);+ }+ const details = usage.inputTokenDetails;+ if (details?.cacheReadTokens !== undefined) {+ span.setAttribute("gen_ai.usage.cache_read.input_tokens", details.cacheReadTokens);+ }+ if (details?.cacheWriteTokens !== undefined) {+ span.setAttribute("gen_ai.usage.cache_creation.input_tokens", details.cacheWriteTokens);+ }+}diff --git a/packages/eve/src/tracing/agent-span-id-generator.ts b/packages/eve/src/tracing/agent-span-id-generator.tsindex 740ea8477..9e7ad0c92 100644--- a/packages/eve/src/tracing/agent-span-id-generator.ts+++ b/packages/eve/src/tracing/agent-span-id-generator.ts@@ -1,3 +1,5 @@+import { createHash } from "node:crypto";+/*** Id generator shared by `registerOTel` and the agent OTel provider. A span* whose lifetime crosses durable worker boundaries (`agent.turn`) is emitted@@ -12,6 +14,12 @@ export class AgentSpanIdGenerator {return randomHexId(16);}+ /** Derives one span id for a replay-stable instrumentation event. */+ deriveSpanId(key: string): string {+ const spanId = createHash("sha256").update(key).digest("hex").slice(0, 16);+ return /^0+$/u.test(spanId) ? "0000000000000001" : spanId;+ }+generateSpanId(): string {const primed = this.#primedSpanId;if (primed !== undefined) {diff --git a/packages/eve/src/tracing/agent-tool-instrumentation.ts b/packages/eve/src/tracing/agent-tool-instrumentation.tsnew file mode 100644index 000000000..9c865e220--- /dev/null+++ b/packages/eve/src/tracing/agent-tool-instrumentation.ts@@ -0,0 +1,226 @@+import {+ ROOT_CONTEXT,+ SpanStatusCode,+ type Context,+ type Span,+ type SpanContext,+ type Tracer,+ trace,+} from "#compiled/@opentelemetry/api/index.js";++import type {+ InstrumentationActionStartedEvent,+ InstrumentationToolCallStartedEvent,+ InstrumentationToolCallTerminalEvent,+} from "#harness/instrumentation-lifecycle.js";+import { actionIdempotencyKey } from "#harness/instrumentation-lifecycle.js";+import { contentAttribute } from "#tracing/agent-otel-content.js";+import type { AgentSpanIdGenerator } from "#tracing/agent-span-id-generator.js";+import type { AgentActionContext } from "#tracing/agent-action-instrumentation.js";++interface ToolSpanState {+ readonly actionKey: string;+ readonly attemptId: string;+ readonly context: Context;+ readonly event: InstrumentationToolCallStartedEvent;+ readonly fallbackParent: Context;+ readonly idempotencyKey: string;+ readonly spanId: string;+ readonly startTimeMs: number;+ finished?: true;+ span?: Span;+ terminal?: InstrumentationToolCallTerminalEvent;+}++export interface AgentToolInstrumentation {+ actionStarted(event: InstrumentationActionStartedEvent): Promise<void>;+ contextFor(attemptId: string, idempotencyKey: string): Context | undefined;+ drain(attemptId: string, failure?: { readonly error: unknown }): void;+ readonly events: {+ readonly "tool.call.completed": (event: InstrumentationToolCallTerminalEvent) => Promise<void>;+ readonly "tool.call.failed": (event: InstrumentationToolCallTerminalEvent) => Promise<void>;+ readonly "tool.call.started": (event: InstrumentationToolCallStartedEvent) => Promise<void>;+ };+}++/** Keeps SDK tool spans parented to actions even when SDK telemetry wins the event race. */+export function createAgentToolInstrumentation(input: {+ readonly actionContextFor: (+ sessionId: string,+ turnId: string,+ callId: string,+ ) => Promise<AgentActionContext | undefined>;+ readonly idGenerator: AgentSpanIdGenerator;+ readonly recordInputs: boolean;+ readonly recordOutputs: boolean;+ readonly resolveFallback: (+ event: InstrumentationToolCallStartedEvent,+ ) => { readonly context: Context; readonly spanContext: SpanContext } | undefined;+ readonly tracer: Tracer;+}): AgentToolInstrumentation {+ const byAction = new Map<string, ToolSpanState>();+ const byAttempt = new Map<string, Map<string, ToolSpanState>>();++ const onStarted = async (event: InstrumentationToolCallStartedEvent): Promise<void> => {+ const actionKey = actionIdempotencyKey(event.scope.sessionId, event.scope.turnId, event.callId);+ const fallback = input.resolveFallback(event);+ let state = fallback === undefined ? undefined : reserve(event, actionKey, fallback);+ const actionParent = await input.actionContextFor(+ event.scope.sessionId,+ event.scope.turnId,+ event.callId,+ );+ if (state === undefined) {+ if (actionParent === undefined) return;+ state = reserve(event, actionKey, actionParent);+ }+ if (actionParent !== undefined) startSpan(state, actionParent.context);+ };++ const onTerminal = async (event: InstrumentationToolCallTerminalEvent): Promise<void> => {+ const state = byAttempt.get(event.scope.attemptId)?.get(event.idempotencyKey);+ if (state === undefined) return;+ state.terminal = event;++ if (state.span === undefined) {+ const actionParent = await input.actionContextFor(+ state.event.scope.sessionId,+ state.event.scope.turnId,+ state.event.callId,+ );+ if (actionParent !== undefined) startSpan(state, actionParent.context);+ }+ finishIfReady(state);+ };++ return {+ async actionStarted(event) {+ const state = byAction.get(event.idempotencyKey);+ if (state === undefined || state.span !== undefined || state.finished === true) return;+ const actionParent = await input.actionContextFor(+ event.scope.sessionId,+ event.scope.turnId,+ event.callId,+ );+ if (actionParent === undefined) return;+ startSpan(state, actionParent.context);+ finishIfReady(state);+ },+ contextFor: (attemptId, idempotencyKey) =>+ byAttempt.get(attemptId)?.get(idempotencyKey)?.context,+ drain(attemptId, failure) {+ const states = byAttempt.get(attemptId);+ if (states === undefined) return;+ for (const state of states.values()) {+ if (state.finished === true) continue;+ if (state.span === undefined) startSpan(state, state.fallbackParent);+ finish(state, failure);+ }+ byAttempt.delete(attemptId);+ },+ events: {+ "tool.call.completed": onTerminal,+ "tool.call.failed": onTerminal,+ "tool.call.started": onStarted,+ },+ };++ function getAttemptStates(attemptId: string): Map<string, ToolSpanState> {+ let states = byAttempt.get(attemptId);+ if (states === undefined) {+ states = new Map();+ byAttempt.set(attemptId, states);+ }+ return states;+ }++ function reserve(+ event: InstrumentationToolCallStartedEvent,+ actionKey: string,+ parent: { readonly context: Context; readonly spanContext: SpanContext },+ ): ToolSpanState {+ const spanId = input.idGenerator.deriveSpanId(`tool:${event.idempotencyKey}`);+ const state: ToolSpanState = {+ actionKey,+ attemptId: event.scope.attemptId,+ context: contextFromSpanContext({+ isRemote: false,+ spanId,+ traceFlags: parent.spanContext.traceFlags,+ traceId: parent.spanContext.traceId,+ }),+ event,+ fallbackParent: parent.context,+ idempotencyKey: event.idempotencyKey,+ spanId,+ startTimeMs: Date.now(),+ };+ getAttemptStates(event.scope.attemptId).set(event.idempotencyKey, state);+ byAction.set(actionKey, state);+ return state;+ }++ function startSpan(state: ToolSpanState, parent: Context): void {+ if (state.span !== undefined || state.finished === true) return;+ state.span = input.idGenerator.withSpanId(state.spanId, () =>+ input.tracer.startSpan(+ "ai.toolCall",+ {+ attributes: toolAttributes(state.event),+ startTime: state.startTimeMs,+ },+ parent,+ ),+ );+ if (input.recordInputs) {+ const args = contentAttribute(state.event.input, false);+ if (args !== undefined) state.span.setAttribute("gen_ai.tool.call.arguments", args);+ }+ }++ function finishIfReady(state: ToolSpanState): void {+ if (state.span === undefined || state.terminal === undefined) return;+ finish(state);+ }++ function finish(state: ToolSpanState, failure?: { readonly error: unknown }): void {+ const span = state.span;+ if (span === undefined || state.finished === true) return;+ state.finished = true;+ const terminal = state.terminal;+ if (failure !== undefined) {+ recordError(span, failure.error);+ } else if (terminal?.type === "tool.call.failed") {+ recordError(span, terminal.error);+ } else if (terminal?.output.type === "error") {+ recordError(span, terminal.output.error);+ } else if (terminal !== undefined && input.recordOutputs) {+ const result = contentAttribute(terminal.output.output, false);+ if (result !== undefined) span.setAttribute("gen_ai.tool.call.result", result);+ }+ span.end();+ byAction.delete(state.actionKey);+ const states = byAttempt.get(state.attemptId);+ states?.delete(state.idempotencyKey);+ if (states?.size === 0) byAttempt.delete(state.attemptId);+ }+}++function toolAttributes(event: InstrumentationToolCallStartedEvent): Record<string, string> {+ return {+ "gen_ai.operation.name": "execute_tool",+ "gen_ai.tool.call.id": event.callId,+ "gen_ai.tool.name": event.toolName,+ };+}++function contextFromSpanContext(spanContext: SpanContext): Context {+ return trace.setSpan(ROOT_CONTEXT, trace.wrapSpanContext(spanContext));+}++function recordError(span: Span, error: unknown): void {+ if (error instanceof Error) {+ span.recordException(error);+ span.setStatus({ code: SpanStatusCode.ERROR, message: error.message });+ } else span.setStatus({ code: SpanStatusCode.ERROR });+}diff --git a/packages/eve/src/tracing/agent-trace-context-store.test.ts b/packages/eve/src/tracing/agent-trace-context-store.test.tsindex 51c36eb09..01c77b1d4 100644--- a/packages/eve/src/tracing/agent-trace-context-store.test.ts+++ b/packages/eve/src/tracing/agent-trace-context-store.test.ts@@ -5,6 +5,7 @@ import { deserializeContext, serializeContext } from "#context/serialize.js";import {ContextAgentTraceStateStore,preserveSerializedAgentTraceState,+ readActionTraceContext,readSessionTraceContext,} from "#tracing/agent-trace-context-store.js";@@ -28,6 +29,7 @@ describe("ContextAgentTraceStateStore", () => {subagentName: "researcher",turnId: "parent-turn",},+ parentIsRemote: true,parentSpanId: "2".repeat(16),rootSessionId: "session-1",sequence: 0,@@ -50,6 +52,7 @@ describe("ContextAgentTraceStateStore", () => {subagentName: "researcher",turnId: "parent-turn",},+ parentIsRemote: true,parentSpanId: "2".repeat(16),startTimeMs: 1_700_000_000_000,});@@ -124,6 +127,36 @@ describe("readSessionTraceContext", () => {});});+describe("readActionTraceContext", () => {+ it("reads the invoking action span out of a serialized context", async () => {+ const context = new ContextContainer();+ await contextStorage.run(context, () => {+ new ContextAgentTraceStateStore().setAction("action-1", {+ attemptIndex: 0,+ callId: "call-1",+ kind: "subagent-call",+ name: "researcher",+ parent: spanContext("1", "2"),+ rootSessionId: "session-1",+ sessionId: "session-1",+ spanId: "3".repeat(16),+ startTimeMs: 1_700_000_000_000,+ stepIndex: 0,+ turnId: "turn-1",+ });+ });+ const serialized = await serializeContext(context);++ expect(readActionTraceContext(serialized, "session-1", "turn-1", "call-1")).toEqual({+ isRemote: false,+ spanId: "3".repeat(16),+ traceFlags: 1,+ traceId: "1".repeat(32),+ });+ expect(readActionTraceContext(serialized, "session-1", "turn-1", "missing")).toBeUndefined();+ });+});+function spanContext(traceId: string, spanId: string) {return { spanId: spanId.repeat(16), traceFlags: 1, traceId: traceId.repeat(32) };}diff --git a/packages/eve/src/tracing/agent-trace-context-store.ts b/packages/eve/src/tracing/agent-trace-context-store.tsindex 3c6111044..15c9c7d8b 100644--- a/packages/eve/src/tracing/agent-trace-context-store.ts+++ b/packages/eve/src/tracing/agent-trace-context-store.ts@@ -4,12 +4,14 @@ import { contextStorage, loadContext } from "#context/container.js";import { ContextKey } from "#context/key.js";import type { InstrumentationParentLineage } from "#harness/instrumentation-lifecycle.js";import type {+ AgentActionTraceState,AgentSessionTraceState,AgentTraceStateStore,AgentTurnTraceState,} from "#tracing/agent-trace-state.js";interface AgentTraceContextState {+ readonly actions: Readonly<Record<string, AgentActionTraceState>>;readonly sessions: Readonly<Record<string, AgentSessionTraceState>>;readonly turns: Readonly<Record<string, AgentTurnTraceState>>;}@@ -46,8 +48,49 @@ export function readSessionTraceContext(return deserializeState(raw).sessions[sessionId]?.context;}+/** Reads the durable action span that should parent a dispatched child agent. */+export function readActionTraceContext(+ serializedContext: Readonly<Record<string, unknown>>,+ sessionId: string,+ turnId: string,+ callId: string,+): SpanContext | undefined {+ const raw = serializedContext[AgentTraceContextKey.name];+ if (raw === undefined) return undefined;+ const action = Object.values(deserializeState(raw).actions).find(+ (state) => state.sessionId === sessionId && state.turnId === turnId && state.callId === callId,+ );+ if (action === undefined) return undefined;+ return {+ isRemote: false,+ spanId: action.spanId,+ traceFlags: action.parent.traceFlags,+ traceId: action.parent.traceId,+ };+}+/** Durable trace state backed by eve's serialized Workflow context. */export class ContextAgentTraceStateStore implements AgentTraceStateStore {+ deleteAction(idempotencyKey: string): void {+ updateState((state) => {+ const actions = { ...state.actions };+ delete actions[idempotencyKey];+ return { ...state, actions };+ });+ }++ deleteActions(sessionId: string, turnId?: string): void {+ updateState((state) => {+ const actions = { ...state.actions };+ for (const [key, action] of Object.entries(actions)) {+ if (action.sessionId === sessionId && (turnId === undefined || action.turnId === turnId)) {+ delete actions[key];+ }+ }+ return { ...state, actions };+ });+ }+deleteSession(sessionId: string): void {updateState((state) => {const sessions = { ...state.sessions };@@ -64,6 +107,16 @@ export class ContextAgentTraceStateStore implements AgentTraceStateStore {});}+ findAction(sessionId: string, callId: string): AgentActionTraceState | undefined {+ return Object.values(contextStorage.getStore()?.get(AgentTraceContextKey)?.actions ?? {}).find(+ (state) => state.sessionId === sessionId && state.callId === callId,+ );+ }++ getAction(idempotencyKey: string): AgentActionTraceState | undefined {+ return contextStorage.getStore()?.get(AgentTraceContextKey)?.actions[idempotencyKey];+ }+getSession(sessionId: string): AgentSessionTraceState | undefined {return contextStorage.getStore()?.get(AgentTraceContextKey)?.sessions[sessionId];}@@ -72,6 +125,13 @@ export class ContextAgentTraceStateStore implements AgentTraceStateStore {return contextStorage.getStore()?.get(AgentTraceContextKey)?.turns[turnKey(sessionId, turnId)];}+ setAction(idempotencyKey: string, value: AgentActionTraceState): void {+ updateState((state) => ({+ ...state,+ actions: { ...state.actions, [idempotencyKey]: value },+ }));+ }+setSession(sessionId: string, value: AgentSessionTraceState): void {updateState((state) => ({...state,@@ -88,7 +148,9 @@ export class ContextAgentTraceStateStore implements AgentTraceStateStore {}function updateState(update: (state: AgentTraceContextState) => AgentTraceContextState): void {- loadContext().set(AgentTraceContextKey, (state) => update(state ?? { sessions: {}, turns: {} }));+ loadContext().set(AgentTraceContextKey, (state) =>+ update(state ?? { actions: {}, sessions: {}, turns: {} }),+ );}function turnKey(sessionId: string, turnId: string): string {@@ -97,6 +159,7 @@ function turnKey(sessionId: string, turnId: string): string {function serializeState(state: AgentTraceContextState): unknown {return {+ actions: state.actions,sessions: Object.fromEntries(Object.entries(state.sessions).map(([id, value]) => [id,@@ -122,7 +185,8 @@ function serializeState(state: AgentTraceContextState): unknown {}function deserializeState(data: unknown): AgentTraceContextState {- if (!isRecord(data)) return { sessions: {}, turns: {} };+ if (!isRecord(data)) return { actions: {}, sessions: {}, turns: {} };+ const actions = deserializeRecord(data.actions, deserializeAction);const sessions = deserializeRecord(data.sessions, (value) => {if (!isRecord(value) || !isSpanContext(value.context)) return undefined;return {@@ -142,6 +206,7 @@ function deserializeState(data: unknown): AgentTraceContextState {return {context: value.context,lineage: deserializeLineage(value.lineage),+ parentIsRemote: typeof value.parentIsRemote === "boolean" ? value.parentIsRemote : undefined,parentSpanId: value.parentSpanId,rootSessionId: typeof value.rootSessionId === "string" ? value.rootSessionId : "",sequence: typeof value.sequence === "number" ? value.sequence : 0,@@ -149,7 +214,49 @@ function deserializeState(data: unknown): AgentTraceContextState {terminal: deserializeTerminal(value.terminal),} satisfies AgentTurnTraceState;});- return { sessions, turns };+ return { actions, sessions, turns };+}++function deserializeAction(value: unknown): AgentActionTraceState | undefined {+ if (+ !isRecord(value) ||+ typeof value.attemptIndex !== "number" ||+ typeof value.callId !== "string" ||+ !isActionKind(value.kind) ||+ typeof value.name !== "string" ||+ !isSpanContext(value.parent) ||+ typeof value.rootSessionId !== "string" ||+ typeof value.sessionId !== "string" ||+ typeof value.spanId !== "string" ||+ typeof value.startTimeMs !== "number" ||+ typeof value.stepIndex !== "number" ||+ typeof value.turnId !== "string"+ ) {+ return undefined;+ }+ return {+ attemptIndex: value.attemptIndex,+ callId: value.callId,+ inputAttribute: typeof value.inputAttribute === "string" ? value.inputAttribute : undefined,+ kind: value.kind,+ name: value.name,+ parent: value.parent,+ rootSessionId: value.rootSessionId,+ sessionId: value.sessionId,+ spanId: value.spanId,+ startTimeMs: value.startTimeMs,+ stepIndex: value.stepIndex,+ turnId: value.turnId,+ };+}++function isActionKind(value: unknown): value is AgentActionTraceState["kind"] {+ return (+ value === "load-skill" ||+ value === "remote-agent-call" ||+ value === "subagent-call" ||+ value === "tool-call"+ );}function deserializeRecord<T>(diff --git a/packages/eve/src/tracing/agent-trace-state.ts b/packages/eve/src/tracing/agent-trace-state.tsindex 93e9f20b6..641845db2 100644--- a/packages/eve/src/tracing/agent-trace-state.ts+++ b/packages/eve/src/tracing/agent-trace-state.ts@@ -1,7 +1,9 @@import type { SpanContext } from "#compiled/@opentelemetry/api/index.js";import type {+ InstrumentationActionKind,InstrumentationParentLineage,+ InstrumentationTraceContext,InstrumentationTurnFailedEvent,InstrumentationTurnSettledEvent,} from "#harness/instrumentation-lifecycle.js";@@ -21,6 +23,7 @@ export interface AgentSessionTraceState {export interface AgentTurnTraceState {readonly context: SpanContext;readonly lineage?: InstrumentationParentLineage;+ readonly parentIsRemote?: boolean;readonly parentSpanId: string;readonly rootSessionId: string;readonly sequence: number;@@ -30,10 +33,34 @@ export interface AgentTurnTraceState {| { readonly type: InstrumentationTurnSettledEvent["type"] };}+export interface AgentActionTraceState {+ readonly attemptIndex: number;+ readonly callId: string;+ readonly inputAttribute?: string;+ readonly kind: InstrumentationActionKind;+ readonly name: string;+ readonly parent: InstrumentationTraceContext;+ readonly rootSessionId: string;+ readonly sessionId: string;+ readonly spanId: string;+ readonly startTimeMs: number;+ readonly stepIndex: number;+ readonly turnId: string;+}+/** Provider-owned serializable storage for durable agent trace state. */export interface AgentTraceStateStore {+ deleteAction(idempotencyKey: string): void | PromiseLike<void>;+ deleteActions(sessionId: string, turnId?: string): void | PromiseLike<void>;deleteSession(sessionId: string): void | PromiseLike<void>;deleteTurn(sessionId: string, turnId: string): void | PromiseLike<void>;+ findAction(+ sessionId: string,+ callId: string,+ ): AgentActionTraceState | undefined | PromiseLike<AgentActionTraceState | undefined>;+ getAction(+ idempotencyKey: string,+ ): AgentActionTraceState | undefined | PromiseLike<AgentActionTraceState | undefined>;getSession(sessionId: string,): AgentSessionTraceState | undefined | PromiseLike<AgentSessionTraceState | undefined>;@@ -41,15 +68,29 @@ export interface AgentTraceStateStore {sessionId: string,turnId: string,): AgentTurnTraceState | undefined | PromiseLike<AgentTurnTraceState | undefined>;+ setAction(idempotencyKey: string, state: AgentActionTraceState): void | PromiseLike<void>;setSession(sessionId: string, state: AgentSessionTraceState): void | PromiseLike<void>;setTurn(sessionId: string, turnId: string, state: AgentTurnTraceState): void | PromiseLike<void>;}/** In-memory trace state used by tests and non-durable runtimes. */export class InMemoryAgentTraceStateStore implements AgentTraceStateStore {+ readonly #actions = new Map<string, AgentActionTraceState>();readonly #sessions = new Map<string, AgentSessionTraceState>();readonly #turns = new Map<string, AgentTurnTraceState>();+ deleteAction(idempotencyKey: string): void {+ this.#actions.delete(idempotencyKey);+ }++ deleteActions(sessionId: string, turnId?: string): void {+ for (const [key, state] of this.#actions) {+ if (state.sessionId === sessionId && (turnId === undefined || state.turnId === turnId)) {+ this.#actions.delete(key);+ }+ }+ }+deleteSession(sessionId: string): void {this.#sessions.delete(sessionId);}@@ -58,6 +99,16 @@ export class InMemoryAgentTraceStateStore implements AgentTraceStateStore {this.#turns.delete(turnKey(sessionId, turnId));}+ findAction(sessionId: string, callId: string): AgentActionTraceState | undefined {+ return [...this.#actions.values()].find(+ (state) => state.sessionId === sessionId && state.callId === callId,+ );+ }++ getAction(idempotencyKey: string): AgentActionTraceState | undefined {+ return this.#actions.get(idempotencyKey);+ }+getSession(sessionId: string): AgentSessionTraceState | undefined {return this.#sessions.get(sessionId);}@@ -66,6 +117,10 @@ export class InMemoryAgentTraceStateStore implements AgentTraceStateStore {return this.#turns.get(turnKey(sessionId, turnId));}+ setAction(idempotencyKey: string, state: AgentActionTraceState): void {+ this.#actions.set(idempotencyKey, state);+ }+setSession(sessionId: string, state: AgentSessionTraceState): void {this.#sessions.set(sessionId, state);}diff --git a/packages/eve/src/tracing/local-instrumentation-runtime.scenario.test.ts b/packages/eve/src/tracing/local-instrumentation-runtime.scenario.test.tsindex ec8755b3e..422920cd0 100644--- a/packages/eve/src/tracing/local-instrumentation-runtime.scenario.test.ts+++ b/packages/eve/src/tracing/local-instrumentation-runtime.scenario.test.ts@@ -3,15 +3,21 @@ import { createRequire } from "node:module";import { tmpdir } from "node:os";import { join } from "node:path";-import { ROOT_CONTEXT, trace as runtimeTrace } from "@opentelemetry/api";+import { ROOT_CONTEXT, trace as apiTrace } from "@opentelemetry/api";import { BasicTracerProvider } from "@opentelemetry/sdk-trace-base";-import { afterEach, describe, expect, it } from "vitest";+import { afterEach, describe, expect, it, vi } from "vitest";+import {+ ROOT_CONTEXT as COMPILED_ROOT_CONTEXT,+ context as runtimeContext,+ trace as runtimeTrace,+} from "#compiled/@opentelemetry/api/index.js";import { ContextContainer, contextStorage } from "#context/container.js";import { createAiSdkHookBridge } from "#harness/ai-sdk-hook-bridge.js";import { listLocalTraces } from "#tracing/local-trace-reader.js";import type { InstrumentationAttemptScope } from "#harness/instrumentation-lifecycle.js";import {+ actionIdempotencyKey,attemptIdempotencyKey,sessionIdempotencyKey,turnIdempotencyKey,@@ -50,6 +56,9 @@ describe("local instrumentation runtime", () => {stepIndex: 0,turnId: "turn-1",};+ const delivery = runtimeTrace.getTracer("workflow").startSpan("workflow.delivery");+ const activeContext = runtimeTrace.setSpan(COMPILED_ROOT_CONTEXT, delivery);+ const contextActive = vi.spyOn(runtimeContext, "active").mockReturnValue(activeContext);await contextStorage.run(new ContextContainer(), async () => {await runtime.hooks.publish({@@ -99,6 +108,16 @@ describe("local instrumentation runtime", () => {usage: { inputTokens: 1, outputTokens: 1 },},]);+ const actionKey = actionIdempotencyKey("session-1", "turn-1", "tool-1");+ await runtime.hooks.publish({+ callId: "tool-1",+ idempotencyKey: actionKey,+ input: {},+ kind: "tool-call",+ name: "weather",+ scope,+ type: "action.started",+ });await Reflect.apply(bridge.onToolExecutionStart!, bridge, [{callId: "call-1",@@ -124,6 +143,13 @@ describe("local instrumentation runtime", () => {toolOutput: { output: { temperature: 72 }, type: "tool-result" },},]);+ await runtime.hooks.publish({+ idempotencyKey: actionKey,+ outcome: "completed",+ output: { output: { temperature: 72 }, type: "result" },+ scope,+ type: "action.completed",+ });await runtime.hooks.publish({idempotencyKey: attemptIdempotencyKey(scope),scope,@@ -143,6 +169,8 @@ describe("local instrumentation runtime", () => {type: "session.waiting",});});+ contextActive.mockRestore();+ delivery.end();await runtime.forceFlush();const traceRoot = join(appRoot, ".eve", "traces", "v1");@@ -164,7 +192,7 @@ describe("local instrumentation runtime", () => {"agent.turn","agent.step","ai.streamText",- "ai.streamText.doStream",+ "chat model-1","agent.action","ai.toolCall","user.model-work",@@ -172,14 +200,22 @@ describe("local instrumentation runtime", () => {]),);expect(span(spans, "agent.step").parentSpanId).toBe(span(spans, "agent.turn").spanId);+ expect(span(spans, "agent.step").links).toEqual([+ expect.objectContaining({+ attributes: expect.arrayContaining([+ expect.objectContaining({+ key: "eve.link.type",+ value: { stringValue: "workflow.delivery" },+ }),+ ]),+ spanId: delivery.spanContext().spanId,+ traceId: delivery.spanContext().traceId,+ }),+ ]);expect(span(spans, "ai.streamText").parentSpanId).toBe(span(spans, "agent.step").spanId);- expect(span(spans, "ai.streamText.doStream").parentSpanId).toBe(- span(spans, "ai.streamText").spanId,- );- expect(span(spans, "user.model-work").parentSpanId).toBe(- span(spans, "ai.streamText.doStream").spanId,- );- expect(span(spans, "agent.action").parentSpanId).toBe(span(spans, "agent.step").spanId);+ expect(span(spans, "chat model-1").parentSpanId).toBe(span(spans, "ai.streamText").spanId);+ expect(span(spans, "user.model-work").parentSpanId).toBe(span(spans, "chat model-1").spanId);+ expect(span(spans, "agent.action").parentSpanId).toBe(span(spans, "agent.turn").spanId);expect(span(spans, "ai.toolCall").parentSpanId).toBe(span(spans, "agent.action").spanId);expect(span(spans, "user.tool-work").parentSpanId).toBe(span(spans, "ai.toolCall").spanId);const listed = await listLocalTraces(appRoot);@@ -194,9 +230,9 @@ describe("local instrumentation runtime", () => {const secondProcessor = new LocalTraceSpanProcessor(appRoot);const firstProvider = new BasicTracerProvider({ spanProcessors: [firstProcessor] });const secondProvider = new BasicTracerProvider({ spanProcessors: [secondProcessor] });- const parent = runtimeTrace.setSpan(+ const parent = apiTrace.setSpan(ROOT_CONTEXT,- runtimeTrace.wrapSpanContext({+ apiTrace.wrapSpanContext({spanId: "b".repeat(16),traceFlags: 1,traceId: "a".repeat(32),@@ -215,6 +251,11 @@ describe("local instrumentation runtime", () => {});interface OtlpSpan {+ readonly links?: ReadonlyArray<{+ readonly attributes: ReadonlyArray<{ readonly key: string; readonly value: unknown }>;+ readonly spanId: string;+ readonly traceId: string;+ }>;readonly name: string;readonly parentSpanId?: string;readonly spanId: string;diff --git a/packages/eve/src/tracing/otel-registration.test.ts b/packages/eve/src/tracing/otel-registration.test.tsindex 9e49ca976..cb53e16a1 100644--- a/packages/eve/src/tracing/otel-registration.test.ts+++ b/packages/eve/src/tracing/otel-registration.test.ts@@ -92,6 +92,68 @@ describe("registerOtelPipeline", () => {expect(configuration.spanProcessors[0]).toBe("auto");});+ it("exports only the first terminal for a replay-stable span", () => {+ const downstream = {+ forceFlush: vi.fn(async () => {}),+ onEnd: vi.fn(),+ onStart: vi.fn(),+ shutdown: vi.fn(async () => {}),+ };+ registerOTel.mockImplementation(() => undefined);++ expect(() =>+ registerOtelPipeline({+ pipeline: { spanProcessors: [downstream] },+ serviceName: "weather",+ }),+ ).toThrow();++ const processor = (+ registerOTel.mock.calls.at(-1)?.[0] as {+ spanProcessors: { onEnd(span: unknown): void }[];+ }+ ).spanProcessors[0]!;+ const first = replaySpan("first");+ const losingReplay = replaySpan("losing replay");+ processor.onEnd(first);+ processor.onEnd(losingReplay);++ expect(downstream.onEnd).toHaveBeenCalledExactlyOnceWith(first);+ });++ it("holds a physical child until its started parent completes", () => {+ const downstream = {+ forceFlush: vi.fn(async () => {}),+ onEnd: vi.fn(),+ onStart: vi.fn(),+ shutdown: vi.fn(async () => {}),+ };+ registerOTel.mockImplementation(() => undefined);+ expect(() =>+ registerOtelPipeline({+ pipeline: { spanProcessors: [downstream] },+ serviceName: "weather",+ }),+ ).toThrow();+ const processor = (+ registerOTel.mock.calls.at(-1)?.[0] as {+ spanProcessors: {+ onEnd(span: unknown): void;+ onStart(span: unknown, parentContext: unknown): void;+ }[];+ }+ ).spanProcessors[0]!;+ const parent = physicalSpan("2");+ const child = physicalSpan("3", "2");+ processor.onStart(parent, {});+ processor.onStart(child, {});+ processor.onEnd(child);+ expect(downstream.onEnd).not.toHaveBeenCalled();++ processor.onEnd(parent);+ expect(downstream.onEnd.mock.calls).toEqual([[parent], [child]]);+ });+it("throws when the registration never reached a processor", () => {registerOTel.mockImplementation(() => undefined);@@ -100,3 +162,19 @@ describe("registerOtelPipeline", () => {).toThrow(/another runtime already owns/u);});});++function replaySpan(marker: string) {+ return {+ marker,+ spanContext: () => ({ spanId: "2".repeat(16), traceId: "1".repeat(32) }),+ };+}++function physicalSpan(spanId: string, parentSpanId?: string) {+ const traceId = "1".repeat(32);+ return {+ parentSpanContext:+ parentSpanId === undefined ? undefined : { spanId: parentSpanId.repeat(16), traceId },+ spanContext: () => ({ spanId: spanId.repeat(16), traceId }),+ };+}diff --git a/packages/eve/src/tracing/otel-registration.ts b/packages/eve/src/tracing/otel-registration.tsindex e2337f4fb..eb54545a5 100644--- a/packages/eve/src/tracing/otel-registration.ts+++ b/packages/eve/src/tracing/otel-registration.ts@@ -10,6 +10,7 @@ import { AgentSpanIdGenerator } from "#tracing/agent-span-id-generator.js";import type { OtelPipeline } from "#tracing/otel-declaration.js";const REGISTRATION_SPAN_NAME = "eve.otel.registration";+const REPLAY_DEDUPLICATION_LIMIT = 100_000;class RegistrationMarkerPropagator {#injected = false;@@ -35,7 +36,11 @@ class RegistrationMarkerPropagator {/** Keeps eve's ownership check out of every authored destination. */class PrivateSpanFilteringProcessor implements SpanProcessor {+ private readonly endedSpans = new Set<string>();+ private readonly forwardedSpans = new Set<string>();+ private readonly pendingByParent = new Map<string, unknown[]>();private readonly processors: readonly SpanProcessor[];+ private readonly startedSpans = new Set<string>();constructor(processors: readonly SpanProcessor[]) {this.processors = processors;@@ -47,17 +52,45 @@ class PrivateSpanFilteringProcessor implements SpanProcessor {onEnd(span: unknown): void {if (isRegistrationSpan(span)) return;- for (const processor of this.processors) processor.onEnd(span);+ const identity = spanIdentity(span);+ if (identity !== undefined) {+ if (this.endedSpans.has(identity)) return;+ if (this.endedSpans.size >= REPLAY_DEDUPLICATION_LIMIT) {+ const oldest = this.endedSpans.values().next().value;+ if (oldest !== undefined) this.endedSpans.delete(oldest);+ }+ this.endedSpans.add(identity);+ }+ const parent = parentIdentity(span);+ if (parent !== undefined && this.startedSpans.has(parent) && !this.forwardedSpans.has(parent)) {+ const pending = this.pendingByParent.get(parent) ?? [];+ pending.push(span);+ this.pendingByParent.set(parent, pending);+ return;+ }+ this.forward(span, identity);}onStart(span: unknown, parentContext: unknown): void {if (isRegistrationSpan(span)) return;+ const identity = spanIdentity(span);+ if (identity !== undefined) addBounded(this.startedSpans, identity);for (const processor of this.processors) processor.onStart(span, parentContext);}async shutdown(): Promise<void> {await Promise.all(this.processors.map((processor) => processor.shutdown()));}++ private forward(span: unknown, identity: string | undefined): void {+ for (const processor of this.processors) processor.onEnd(span);+ if (identity === undefined) return;+ addBounded(this.forwardedSpans, identity);+ const children = this.pendingByParent.get(identity);+ if (children === undefined) return;+ this.pendingByParent.delete(identity);+ for (const child of children) this.forward(child, spanIdentity(child));+ }}/**@@ -181,6 +214,41 @@ function isRegistrationSpan(span: unknown): boolean {);}+function spanIdentity(span: unknown): string | undefined {+ if (+ typeof span !== "object" ||+ span === null ||+ !("spanContext" in span) ||+ typeof span.spanContext !== "function"+ ) {+ return undefined;+ }+ const context = span.spanContext() as { readonly spanId?: unknown; readonly traceId?: unknown };+ return typeof context.traceId === "string" && typeof context.spanId === "string"+ ? `${context.traceId}:${context.spanId}`+ : undefined;+}++function parentIdentity(span: unknown): string | undefined {+ if (typeof span !== "object" || span === null || !("parentSpanContext" in span)) {+ return undefined;+ }+ const parent = span.parentSpanContext as+ | { readonly spanId?: unknown; readonly traceId?: unknown }+ | undefined;+ return typeof parent?.traceId === "string" && typeof parent.spanId === "string"+ ? `${parent.traceId}:${parent.spanId}`+ : undefined;+}++function addBounded(values: Set<string>, value: string): void {+ if (values.size >= REPLAY_DEDUPLICATION_LIMIT) {+ const oldest = values.values().next().value;+ if (oldest !== undefined) values.delete(oldest);+ }+ values.add(value);+}+function isSpanProcessor(processor: SpanProcessorOrName): processor is SpanProcessor {return processor !== "auto";}