Skip to content

armgit / eve

Public Imported from GitHub
0

The Open Framework for Building Agents

feat(eve): trace durable runtime actions (#1862)

Chad Hietala29313bee252c
diff --git a/.changeset/durable-action-tracing.md b/.changeset/durable-action-tracing.md
new file mode 100644
index 000000000..91222f207
--- /dev/null
+++ b/.changeset/durable-action-tracing.md
@@ -0,0 +1,5 @@
+---
+"eve": patch
+---
+
+Reconstruct durable `agent.action` spans when runtime actions settle, including across worker replacement, and record each action's exact caller-accepted duration, kind, outcome, stable error code, and subagent usage. Remote eve sessions join the caller action trace through W3C `traceparent`; older receivers may ignore the header. Human approval waits appear as durable `agent.approval` child spans, while chat spans use standard self-contained model input and output attributes.
diff --git a/packages/eve/scripts/vendor-compiled/declarations/@opentelemetry/api.d.ts b/packages/eve/scripts/vendor-compiled/declarations/@opentelemetry/api.d.ts
index d7168a656..62d05fac5 100644
--- a/packages/eve/scripts/vendor-compiled/declarations/@opentelemetry/api.d.ts
+++ b/packages/eve/scripts/vendor-compiled/declarations/@opentelemetry/api.d.ts
@@ -1,14 +1,21 @@
+export interface TraceState {
+ get(key: string): string | undefined;
+ serialize(): string;
+ set(key: string, value: string): TraceState;
+ unset(key: string): TraceState;
+}
+
export interface SpanContext {
isRemote?: boolean;
spanId: string;
traceFlags: number;
traceId: string;
- traceState?: unknown;
+ traceState?: TraceState;
}
export interface Span {
addEvent(name: string, attributes?: Attributes, timestamp?: number): this;
- end(): void;
+ end(endTime?: number): void;
recordException(
exception: Error | string | { message?: string; name?: string; stack?: string },
): void;
@@ -17,12 +24,18 @@ export interface Span {
spanContext(): SpanContext;
}
+export interface Link {
+ context: SpanContext;
+ attributes?: Attributes;
+}
+
export interface Tracer {
startSpan(
name: string,
options?: {
attributes?: Attributes | undefined;
kind?: SpanKind | undefined;
+ links?: Link[] | undefined;
root?: boolean | undefined;
startTime?: number | undefined;
},
@@ -75,6 +88,7 @@ export declare const propagation: {
export declare const trace: {
getActiveSpan(): Span | undefined;
+ getSpan(context: Context): Span | undefined;
getTracer(name: string, version?: string): Tracer;
getTracerProvider(): unknown;
setSpan(context: Context, span: Span): Context;
diff --git a/packages/eve/src/cli/commands/trace.integration.test.ts b/packages/eve/src/cli/commands/trace.integration.test.ts
index 5b60f0feb..25851183a 100644
--- a/packages/eve/src/cli/commands/trace.integration.test.ts
+++ b/packages/eve/src/cli/commands/trace.integration.test.ts
@@ -164,7 +164,7 @@ describe("eve traces", () => {
root,
TRACE_ONE,
span(action, "agent.action", 30, 80, step, {
- "agent.action.kind": "tool",
+ "agent.action.kind": "tool-call",
"agent.action.name": "\u001B[31mweather\u001B[0m",
"agent.session.id": "session-one",
}),
@@ -179,7 +179,7 @@ describe("eve traces", () => {
expect(output.out[0]).toContain("agent.turn [turn-1]");
expect(output.out[0]).toContain("└─ agent.step [step 0, attempt 0]");
- expect(output.out[0]).toContain("└─ agent.action [tool: weather]");
+ expect(output.out[0]).toContain("└─ agent.action [tool-call: weather]");
expect(output.out[0]).toContain("failed 10ms ERROR");
expect(output.out[0]).not.toContain("\u001B");
});
diff --git a/packages/eve/src/cli/dev/tui/traces/trace-conversation.ts b/packages/eve/src/cli/dev/tui/traces/trace-conversation.ts
index fbd94f8b3..8a977ce3b 100644
--- a/packages/eve/src/cli/dev/tui/traces/trace-conversation.ts
+++ b/packages/eve/src/cli/dev/tui/traces/trace-conversation.ts
@@ -563,7 +563,10 @@ function unwrapJsonString(raw: string | undefined): string | undefined {
}
function isModelSpan(span: LocalTraceSpan): boolean {
- return span.name.startsWith("ai.") && span.name.includes("do");
+ return (
+ (span.name.startsWith("ai.") && span.name.includes("do")) ||
+ span.attributes["gen_ai.operation.name"] === "chat"
+ );
}
/** Extracts tool names from the `ai.response.tool_calls` JSON array. */
diff --git a/packages/eve/src/execution/dispatch-runtime-actions-step.integration.test.ts b/packages/eve/src/execution/dispatch-runtime-actions-step.integration.test.ts
index 550fcc2c8..2b271a6fc 100644
--- a/packages/eve/src/execution/dispatch-runtime-actions-step.integration.test.ts
+++ b/packages/eve/src/execution/dispatch-runtime-actions-step.integration.test.ts
@@ -154,16 +154,43 @@ describe("dispatchRuntimeActionsStep child starts", () => {
nodeId: "subagents/research",
});
const writes: Uint8Array[] = [];
+ const traceId = "1".repeat(32);
+ const actionSpanId = "3".repeat(16);
const result = await dispatchRuntimeActionsStep({
parentContinuationToken: "turn-inbox",
parentWritable: createWritable(writes),
- serializedContext: {},
+ serializedContext: {
+ "eve.harness.agentTrace": {
+ actions: {
+ "action-1": {
+ attemptIndex: 0,
+ callId: "call-1",
+ kind: "subagent-call",
+ name: "research",
+ parent: { spanId: "2".repeat(16), traceFlags: 1, traceId },
+ rootSessionId: "parent-session",
+ sessionId: "parent-session",
+ spanId: actionSpanId,
+ startTimeMs: 1_700_000_000_000,
+ stepIndex: 0,
+ turnId: "turn-1",
+ },
+ },
+ sessions: {},
+ turns: {},
+ },
+ },
sessionState: BASE_STATE,
});
expect(result.results).toEqual([]);
expect(mocks.createSession).toHaveBeenCalledTimes(1);
+ expect(mocks.createSession).toHaveBeenCalledWith(
+ expect.objectContaining({
+ parentTraceContext: { isRemote: false, spanId: actionSpanId, traceFlags: 1, traceId },
+ }),
+ );
expect(getAgentHandleStore(readResultSessionState(result, session))).toEqual({
handles: [
{
@@ -277,6 +304,8 @@ describe("dispatchRuntimeActionsStep child starts", () => {
it("owns a remote child with its confirmed remote address", async () => {
const session = createStartSession({ kind: "remote" });
+ const traceId = "4".repeat(32);
+ const actionSpanId = "5".repeat(16);
installContext(session, {
definition: REMOTE_REGISTRY_DEFINITION,
nodeId: "remote/research",
@@ -286,11 +315,36 @@ describe("dispatchRuntimeActionsStep child starts", () => {
callbackBaseUrl: "https://caller.example.com",
parentContinuationToken: "turn-inbox",
parentWritable: createWritable(),
- serializedContext: {},
+ serializedContext: {
+ "eve.harness.agentTrace": {
+ actions: {
+ "action-1": {
+ attemptIndex: 0,
+ callId: "call-1",
+ kind: "remote-agent-call",
+ name: "research",
+ parent: { spanId: "2".repeat(16), traceFlags: 1, traceId },
+ rootSessionId: "parent-session",
+ sessionId: "parent-session",
+ spanId: actionSpanId,
+ startTimeMs: 1_700_000_000_000,
+ stepIndex: 0,
+ turnId: "turn-1",
+ },
+ },
+ sessions: {},
+ turns: {},
+ },
+ },
sessionState: BASE_STATE,
});
expect(result.results).toEqual([]);
+ expect(mocks.startRemoteAgentSession).toHaveBeenCalledWith(
+ expect.objectContaining({
+ parentTraceContext: { isRemote: false, spanId: actionSpanId, traceFlags: 1, traceId },
+ }),
+ );
expect(getAgentHandleStore(readResultSessionState(result, session))).toEqual({
handles: [
expect.objectContaining({
diff --git a/packages/eve/src/execution/dispatch-runtime-actions-step.ts b/packages/eve/src/execution/dispatch-runtime-actions-step.ts
index 1d61e38f4..c11c803a2 100644
--- a/packages/eve/src/execution/dispatch-runtime-actions-step.ts
+++ b/packages/eve/src/execution/dispatch-runtime-actions-step.ts
@@ -1,8 +1,4 @@
-/**
- * Dispatches every pending runtime action for a parked parent session.
- * Starts commit ownership before their side effect, so the returned state
- * owns every child it may have created.
- */
+/** Starts or continues pending actions, committing ownership before side effects. */
import { buildAdapterContext } from "#channel/adapter-context.js";
import { callAdapterEventHandler } from "#channel/adapter.js";
@@ -70,7 +66,7 @@ import { buildSubagentRunInput, type SubagentInputSource } from "#execution/suba
import { createWorkflowRuntime, workflowEntryReference } from "#execution/workflow-runtime.js";
import { createLogger, logError } from "#internal/logging.js";
import { toErrorMessage } from "#shared/errors.js";
-import { readSessionTraceContext } from "#tracing/agent-trace-context-store.js";
+import * as traceState from "#tracing/agent-trace-context-store.js";
import { resolveSubagentDepth } from "#harness/subagent-depth.js";
import { getDynamicSubagentSelection } from "#context/dynamic-subagent-lifecycle.js";
import { resolveEffectiveAgentRuntime } from "#execution/effective-agent-config.js";
@@ -157,24 +153,19 @@ export async function dispatchRuntimeActionsStep(input: {
const initiatorAuth = ctx.get(InitiatorAuthKey) ?? null;
const adapterCtx = buildAdapterContext(adapter, ctx);
- // Read here, not in the child: trace state is scoped to one session's
- // context, so this is the last place the parent's window is visible.
- const parentTraceContext = readSessionTraceContext(input.serializedContext, session.sessionId);
+ // This is the last place the parent's session-scoped trace state is visible.
+ const sessionTraceContext = traceState.readSessionTraceContext(
+ input.serializedContext,
+ session.sessionId,
+ );
const persistentSessions =
bundle.resolvedAgent.config?.experimental?.subagentPersistentSessions === true;
- // A corrupt handle store throws; surface that before anything dispatches.
- // A mid-loop throw after a sibling started would durably replay the whole
- // batch and re-dispatch that sibling.
+ // Fail before dispatch; replaying after a sibling starts would duplicate it.
getAgentHandleStore(durableSession.state);
const plan = planDispatch({ actions: batch.actions, bundle, ctx, session });
- // Acquired only once preflight can no longer throw, so a planning failure
- // never leaks the writer lock.
const writer = input.parentWritable.getWriter();
- // Split the parent's remaining token quota across the batch's freshly
- // started local subagents, the children that actually receive an enforced
- // cap. Continuations already run under their own budget, and remote agents
- // run on their own deployment under their own limits, so neither dilutes
- // the local shares.
+ // Only fresh local starts share the remaining quota; continuations and
+ // remote agents already enforce their own budgets.
const fanoutSize = plan.filter(
(entry) => entry.kind === "start" && entry.target.kind === "local",
).length;
@@ -217,7 +208,13 @@ export async function dispatchRuntimeActionsStep(input: {
fanoutSize,
initiatorAuth,
parentContinuationToken: input.parentContinuationToken,
- parentTraceContext,
+ parentTraceContext:
+ traceState.readActionTraceContext(
+ input.serializedContext,
+ session.sessionId,
+ batch.event.turnId,
+ entry.target.action.callId,
+ ) ?? sessionTraceContext,
persistentSessions,
session,
target: entry.target,
@@ -470,6 +467,7 @@ async function startSubagent(input: {
dynamicRemoteAgent: input.target.dynamicRemoteAgent,
initiatorAuth: input.initiatorAuth,
parentContinuationToken: input.parentContinuationToken,
+ parentTraceContext: input.parentTraceContext,
persistentSessions: input.persistentSessions,
session: input.session,
});
@@ -595,6 +593,7 @@ async function startRemoteSubagent(input: {
readonly dynamicRemoteAgent?: DynamicRemoteAgentConfig;
readonly initiatorAuth: Parameters<typeof startRemoteAgentSession>[0]["initiatorAuth"];
readonly parentContinuationToken: string | undefined;
+ readonly parentTraceContext: Parameters<typeof startRemoteAgentSession>[0]["parentTraceContext"];
readonly persistentSessions: boolean;
readonly session: RuntimeSession;
}): Promise<DispatchOutcome> {
@@ -648,6 +647,7 @@ async function startRemoteSubagent(input: {
callbackBaseUrl,
callbackToken: input.parentContinuationToken,
initiatorAuth: input.initiatorAuth,
+ parentTraceContext: input.parentTraceContext,
persistentSessions: input.persistentSessions,
remote: resolvedRemote,
session: input.session,
diff --git a/packages/eve/src/execution/remote-agent-dispatch.test.ts b/packages/eve/src/execution/remote-agent-dispatch.test.ts
index d20a35eca..1cf7aaca0 100644
--- a/packages/eve/src/execution/remote-agent-dispatch.test.ts
+++ b/packages/eve/src/execution/remote-agent-dispatch.test.ts
@@ -92,7 +92,15 @@ describe("startRemoteAgentSession", () => {
const childSessionId = await startRemoteAgentSession({
action: createAction(),
callbackBaseUrl: "https://caller.example.com",
- remote: createRemoteAgent(),
+ parentTraceContext: {
+ spanId: "2".repeat(16),
+ traceFlags: 1,
+ traceId: "1".repeat(32),
+ },
+ remote: {
+ ...createRemoteAgent(),
+ headers: { Traceparent: "00-authored", "x-static": "yes" },
+ },
session: {
agent: {
modelReference: { id: "mock/test" },
@@ -116,6 +124,7 @@ describe("startRemoteAgentSession", () => {
headers: {
authorization: "Bearer remote-token",
"content-type": "application/json",
+ traceparent: `00-${"1".repeat(32)}-${"2".repeat(16)}-01`,
"x-static": "yes",
},
method: "POST",
diff --git a/packages/eve/src/execution/remote-agent-dispatch.ts b/packages/eve/src/execution/remote-agent-dispatch.ts
index e315d3c7a..c36537f0e 100644
--- a/packages/eve/src/execution/remote-agent-dispatch.ts
+++ b/packages/eve/src/execution/remote-agent-dispatch.ts
@@ -6,10 +6,11 @@ import {
createEveSessionCancelRoutePath,
createEveSessionRoutePath,
} from "#protocol/routes.js";
-import type { CancelTurnResult, SessionAuthContext } from "#channel/types.js";
+import type { CancelTurnResult, SessionAuthContext, SessionTraceContext } from "#channel/types.js";
import type { ForwardedPrincipal } from "#channel/forwarded-principal.js";
import type { HeadersValue } from "#client/types.js";
import { createWorkflowCallbackUrl } from "#execution/workflow-callback-url.js";
+import { formatTraceparent } from "#protocol/traceparent.js";
import {
formatSubagentInput,
normalizeRequestedOutputSchema,
@@ -56,6 +57,7 @@ export async function startRemoteAgentSession(input: {
* in conversation mode so their sessions accept follow-up messages.
*/
readonly persistentSessions?: boolean;
+ readonly parentTraceContext?: SessionTraceContext;
readonly remote: ResolvedRuntimeRemoteAgentNode;
readonly session: HarnessSession;
}): Promise<RemoteAgentSessionCoordinates> {
@@ -105,6 +107,13 @@ export async function startRemoteAgentSession(input: {
}
const headers = await resolveRemoteAgentRequestHeaders(input.remote);
+ const traceparent = formatTraceparent(input.parentTraceContext);
+ if (traceparent !== undefined) {
+ for (const key of Object.keys(headers)) {
+ if (key.toLowerCase() === "traceparent") delete headers[key];
+ }
+ headers["traceparent"] = traceparent;
+ }
const response = await fetch(createRemoteAgentSessionUrl(input.remote), {
body: JSON.stringify(requestBody),
headers: {
diff --git a/packages/eve/src/protocol/traceparent.test.ts b/packages/eve/src/protocol/traceparent.test.ts
new file mode 100644
index 000000000..bc5c20189
--- /dev/null
+++ b/packages/eve/src/protocol/traceparent.test.ts
@@ -0,0 +1,40 @@
+import { describe, expect, it } from "vitest";
+
+import { formatTraceparent, parseTraceparent } from "#protocol/traceparent.js";
+
+const TRACE_ID = "1".repeat(32);
+const SPAN_ID = "2".repeat(16);
+
+describe("traceparent", () => {
+ it("round-trips a remote sampled context", () => {
+ const header = formatTraceparent({ spanId: SPAN_ID, traceFlags: 1, traceId: TRACE_ID });
+ expect(header).toBe(`00-${TRACE_ID}-${SPAN_ID}-01`);
+ expect(parseTraceparent(header!)).toEqual({
+ isRemote: true,
+ spanId: SPAN_ID,
+ traceFlags: 1,
+ traceId: TRACE_ID,
+ });
+ });
+
+ it.each([
+ null,
+ "",
+ `01-${TRACE_ID}-${SPAN_ID}-01`,
+ `00-${"0".repeat(32)}-${SPAN_ID}-01`,
+ `00-${TRACE_ID}-${"0".repeat(16)}-01`,
+ `00-${TRACE_ID}-${SPAN_ID}-0g`,
+ `00-${TRACE_ID}-${SPAN_ID}-01,00-${TRACE_ID}-${SPAN_ID}-01`,
+ ])("ignores malformed input %p", (value) => {
+ expect(parseTraceparent(value)).toBeUndefined();
+ });
+
+ it("omits invalid outbound contexts", () => {
+ expect(
+ formatTraceparent({ spanId: "0".repeat(16), traceFlags: 1, traceId: TRACE_ID }),
+ ).toBeUndefined();
+ expect(
+ formatTraceparent({ spanId: SPAN_ID, traceFlags: 256, traceId: TRACE_ID }),
+ ).toBeUndefined();
+ });
+});
diff --git a/packages/eve/src/protocol/traceparent.ts b/packages/eve/src/protocol/traceparent.ts
new file mode 100644
index 000000000..26debde84
--- /dev/null
+++ b/packages/eve/src/protocol/traceparent.ts
@@ -0,0 +1,43 @@
+export interface TraceparentContext {
+ readonly isRemote?: boolean;
+ readonly spanId: string;
+ readonly traceFlags: number;
+ readonly traceId: string;
+}
+
+const TRACEPARENT_PATTERN = /^00-([0-9a-f]{32})-([0-9a-f]{16})-([0-9a-f]{2})$/iu;
+
+/** Formats a W3C version-00 traceparent, or omits an invalid context. */
+export function formatTraceparent(context: TraceparentContext | undefined): string | undefined {
+ if (
+ context === undefined ||
+ !validId(context.traceId, 32) ||
+ !validId(context.spanId, 16) ||
+ !Number.isInteger(context.traceFlags) ||
+ context.traceFlags < 0 ||
+ context.traceFlags > 0xff
+ ) {
+ return undefined;
+ }
+ return `00-${context.traceId.toLowerCase()}-${context.spanId.toLowerCase()}-${context.traceFlags.toString(16).padStart(2, "0")}`;
+}
+
+/** Parses a W3C version-00 traceparent without rejecting the containing request. */
+export function parseTraceparent(value: string | null): TraceparentContext | undefined {
+ if (value === null) return undefined;
+ const match = TRACEPARENT_PATTERN.exec(value.trim());
+ if (match === null) return undefined;
+ const traceId = match[1]!.toLowerCase();
+ const spanId = match[2]!.toLowerCase();
+ if (!validId(traceId, 32) || !validId(spanId, 16)) return undefined;
+ return {
+ isRemote: true,
+ spanId,
+ traceFlags: Number.parseInt(match[3]!, 16),
+ traceId,
+ };
+}
+
+function validId(value: string, length: number): boolean {
+ return value.length === length && /^[0-9a-f]+$/iu.test(value) && !/^0+$/u.test(value);
+}
diff --git a/packages/eve/src/public/channels/eve-session-routes.test.ts b/packages/eve/src/public/channels/eve-session-routes.test.ts
index 51c2f4019..9ca305b54 100644
--- a/packages/eve/src/public/channels/eve-session-routes.test.ts
+++ b/packages/eve/src/public/channels/eve-session-routes.test.ts
@@ -52,7 +52,10 @@ describe("eve ID-addressed session routes", () => {
const response = await route("POST", "/eve/v1/session")(
new Request("https://eve.test/eve/v1/session", {
body: JSON.stringify({ message: "hello" }),
- headers: { "content-type": "application/json" },
+ headers: {
+ "content-type": "application/json",
+ traceparent: `00-${"1".repeat(32)}-${"2".repeat(16)}-01`,
+ },
method: "POST",
}),
args,
@@ -64,6 +67,16 @@ describe("eve ID-addressed session routes", () => {
sessionId: "wrun_A",
status: "accepted",
});
+ expect(createSession).toHaveBeenCalledWith(
+ expect.objectContaining({
+ parentTraceContext: {
+ isRemote: true,
+ spanId: "2".repeat(16),
+ traceFlags: 1,
+ traceId: "1".repeat(32),
+ },
+ }),
+ );
expect(createSession).toHaveBeenCalledWith(
expect.not.objectContaining({ continuationToken: expect.anything() }),
);
diff --git a/packages/eve/src/public/channels/eve.ts b/packages/eve/src/public/channels/eve.ts
index 2a90d3626..6ee754b44 100644
--- a/packages/eve/src/public/channels/eve.ts
+++ b/packages/eve/src/public/channels/eve.ts
@@ -28,6 +28,7 @@ import {
EVE_STREAM_TAIL_INDEX_HEADER,
EVE_STREAM_VERSION_HEADER,
} from "#protocol/message.js";
+import { parseTraceparent } from "#protocol/traceparent.js";
import {
EVE_INFO_ROUTE_PATH,
EVE_SESSION_ROUTE_PATH,
@@ -246,6 +247,7 @@ export function eveChannel(input: EveChannelInput): EveChannel {
const body = parseCreateBody(payload);
if (body instanceof Response) return body;
+ const parentTraceContext = parseTraceparent(req.headers.get("traceparent"));
const policyRejection = checkUploadPolicy(body, uploadPolicy);
if (policyRejection !== null) return policyRejection;
@@ -279,6 +281,7 @@ export function eveChannel(input: EveChannelInput): EveChannel {
outputSchema: body.outputSchema,
},
mode: body.mode ?? "conversation",
+ parentTraceContext,
});
} catch (error) {
const errorId = logError(log, "session-create request failed", error);
diff --git a/packages/eve/src/runtime/attributes/emit.ts b/packages/eve/src/runtime/attributes/emit.ts
index c9f53784d..e219484f5 100644
--- a/packages/eve/src/runtime/attributes/emit.ts
+++ b/packages/eve/src/runtime/attributes/emit.ts
@@ -68,6 +68,7 @@ export async function setEveAttributes(attrs: Record<string, EveAttributeValue>)
const { setAttributes } = await import("#compiled/@workflow/core/index.js");
await setAttributes(normalized, { allowReservedAttributes: true });
} catch (error) {
+ if (isTerminalRunAttributeError(error)) return;
if (!WARNED_ABOUT_TAG_FAILURE) {
WARNED_ABOUT_TAG_FAILURE = true;
console.warn("[eve] setEveAttributes failed; suppressing further warnings this process.", {
@@ -77,3 +78,10 @@ export async function setEveAttributes(attrs: Record<string, EveAttributeValue>)
}
}
}
+
+function isTerminalRunAttributeError(error: unknown): boolean {
+ return (
+ error instanceof Error &&
+ error.message.startsWith("Cannot set attributes on run in terminal state")
+ );
+}
diff --git a/packages/eve/src/tracing/agent-action-instrumentation.ts b/packages/eve/src/tracing/agent-action-instrumentation.ts
new file mode 100644
index 000000000..905546258
--- /dev/null
+++ b/packages/eve/src/tracing/agent-action-instrumentation.ts
@@ -0,0 +1,205 @@
+import {
+ ROOT_CONTEXT,
+ SpanStatusCode,
+ type Context,
+ type Span,
+ type SpanContext,
+ type Tracer,
+ trace,
+} from "#compiled/@opentelemetry/api/index.js";
+
+import type {
+ InstrumentationActionStartedEvent,
+ InstrumentationActionTerminalEvent,
+ InstrumentationAttemptScope,
+ InstrumentationProviderDefinition,
+} from "#harness/instrumentation-lifecycle.js";
+import { actionIdempotencyKey } from "#harness/instrumentation-lifecycle.js";
+import { contentAttribute } from "#tracing/agent-otel-content.js";
+import { setAgentUsage } from "#tracing/agent-otel-usage.js";
+import type { AgentSpanIdGenerator } from "#tracing/agent-span-id-generator.js";
+import type { AgentActionTraceState, AgentTraceStateStore } from "#tracing/agent-trace-state.js";
+
+export interface AgentActionInstrumentation {
+ readonly events: Pick<
+ NonNullable<InstrumentationProviderDefinition["events"]>,
+ "action.completed" | "action.failed" | "action.started"
+ >;
+ deleteForSession(sessionId: string): void | PromiseLike<void>;
+ deleteForTurn(sessionId: string, turnId: string): void | PromiseLike<void>;
+ failForAttempt(scope: InstrumentationAttemptScope, error: unknown): Promise<void>;
+ contextFor(
+ sessionId: string,
+ turnId: string,
+ callId: string,
+ ): Promise<AgentActionContext | undefined>;
+}
+
+export interface AgentActionContext {
+ readonly context: Context;
+ readonly spanContext: SpanContext;
+}
+
+/** Builds durable `agent.action` spans around eve's runtime dispatch boundary. */
+export function createAgentActionInstrumentation(input: {
+ readonly frameworkVersion: string;
+ readonly idGenerator: AgentSpanIdGenerator;
+ readonly recordInputs: boolean;
+ readonly recordOutputs: boolean;
+ readonly resolveParent: (
+ event: InstrumentationActionStartedEvent,
+ ) =>
+ | { readonly context: Context; readonly spanContext: SpanContext }
+ | undefined
+ | PromiseLike<{ readonly context: Context; readonly spanContext: SpanContext } | undefined>;
+ readonly stateStore: AgentTraceStateStore;
+ readonly tracer: Tracer;
+}): AgentActionInstrumentation {
+ const byAttempt = new Map<string, Set<string>>();
+
+ const onStarted = async (event: InstrumentationActionStartedEvent): Promise<void> => {
+ const parent = await input.resolveParent(event);
+ if (parent === undefined) return;
+
+ const existing = await input.stateStore.getAction(event.idempotencyKey);
+ const state: AgentActionTraceState = existing ?? {
+ attemptIndex: event.scope.attemptIndex,
+ callId: event.callId,
+ inputAttribute: input.recordInputs ? contentAttribute(event.input, false) : undefined,
+ kind: event.kind,
+ name: event.name,
+ parent: {
+ spanId: parent.spanContext.spanId,
+ traceFlags: parent.spanContext.traceFlags,
+ traceId: parent.spanContext.traceId,
+ },
+ rootSessionId: event.scope.rootSessionId ?? event.scope.sessionId,
+ sessionId: event.scope.sessionId,
+ spanId: input.idGenerator.deriveSpanId(`action:${event.idempotencyKey}`),
+ startTimeMs: Date.now(),
+ stepIndex: event.scope.stepIndex,
+ turnId: event.scope.turnId,
+ };
+ await input.stateStore.setAction(event.idempotencyKey, state);
+ const keys = byAttempt.get(event.scope.attemptId) ?? new Set<string>();
+ keys.add(event.idempotencyKey);
+ byAttempt.set(event.scope.attemptId, keys);
+ };
+
+ const onTerminal = async (event: InstrumentationActionTerminalEvent): Promise<void> => {
+ const state = await input.stateStore.getAction(event.idempotencyKey);
+ if (state === undefined) return;
+ try {
+ const span = startSpan(state);
+ span.setAttribute("agent.action.outcome", event.outcome);
+ if (event.type === "action.failed") {
+ if (event.errorCode !== undefined) {
+ span.setAttribute("agent.action.error.code", event.errorCode);
+ span.setAttribute("error.type", event.errorCode);
+ }
+ recordError(span, event.error);
+ } else if (event.output.type === "error") {
+ recordError(span, event.output.error);
+ } else {
+ if (event.usage !== undefined) setAgentUsage(span, event.usage);
+ if (input.recordOutputs) {
+ const result = contentAttribute(event.output.output, false);
+ if (result !== undefined) span.setAttribute("gen_ai.tool.call.result", result);
+ }
+ }
+ span.end(event.acceptedAtMs);
+ } finally {
+ await input.stateStore.deleteAction(event.idempotencyKey);
+ forget(event.idempotencyKey);
+ }
+ };
+
+ const startSpan = (state: AgentActionTraceState): Span => {
+ const span = input.idGenerator.withSpanId(state.spanId, () =>
+ input.tracer.startSpan(
+ "agent.action",
+ {
+ attributes: {
+ "agent.action.call_id": state.callId,
+ "agent.action.kind": state.kind,
+ "agent.action.name": state.name,
+ "agent.framework.name": "eve",
+ "agent.framework.version": input.frameworkVersion,
+ "agent.root.session.id": state.rootSessionId,
+ "agent.session.id": state.sessionId,
+ "agent.step.attempt": state.attemptIndex,
+ "agent.step.index": state.stepIndex,
+ "agent.turn.id": state.turnId,
+ },
+ startTime: state.startTimeMs,
+ },
+ contextFromActionState(state),
+ ),
+ );
+ if (state.inputAttribute !== undefined) {
+ span.setAttribute("gen_ai.tool.call.arguments", state.inputAttribute);
+ }
+ return span;
+ };
+
+ return {
+ async contextFor(sessionId, turnId, callId) {
+ const directKey = actionIdempotencyKey(sessionId, turnId, callId);
+ const direct = await input.stateStore.getAction(directKey);
+ if (direct !== undefined) return actionContext(direct);
+ const state = await input.stateStore.findAction(sessionId, callId);
+ return state === undefined ? undefined : actionContext(state);
+ },
+ deleteForSession: (sessionId) => input.stateStore.deleteActions(sessionId),
+ deleteForTurn: (sessionId, turnId) => input.stateStore.deleteActions(sessionId, turnId),
+ async failForAttempt(scope, error) {
+ const keys = byAttempt.get(scope.attemptId);
+ if (keys === undefined) return;
+ byAttempt.delete(scope.attemptId);
+ for (const key of keys) {
+ const state = await input.stateStore.getAction(key);
+ if (state === undefined) continue;
+ const span = startSpan(state);
+ recordError(span, error);
+ span.end();
+ await input.stateStore.deleteAction(key);
+ }
+ },
+ events: {
+ "action.completed": onTerminal,
+ "action.failed": onTerminal,
+ "action.started": onStarted,
+ },
+ };
+
+ function forget(idempotencyKey: string): void {
+ for (const [attemptId, keys] of byAttempt) {
+ keys.delete(idempotencyKey);
+ if (keys.size === 0) byAttempt.delete(attemptId);
+ }
+ }
+}
+
+function actionContext(state: AgentActionTraceState): AgentActionContext {
+ const spanContext = {
+ isRemote: false,
+ spanId: state.spanId,
+ traceFlags: state.parent.traceFlags,
+ traceId: state.parent.traceId,
+ };
+ return {
+ context: trace.setSpan(ROOT_CONTEXT, trace.wrapSpanContext(spanContext)),
+ spanContext,
+ };
+}
+
+function contextFromActionState(state: AgentActionTraceState): Context {
+ return trace.setSpan(ROOT_CONTEXT, trace.wrapSpanContext({ ...state.parent, isRemote: false }));
+}
+
+function recordError(span: Span, error: unknown): void {
+ if (error instanceof Error) {
+ span.recordException(error);
+ span.setStatus({ code: SpanStatusCode.ERROR, message: error.message });
+ } else span.setStatus({ code: SpanStatusCode.ERROR });
+}
diff --git a/packages/eve/src/tracing/agent-approval-instrumentation.ts b/packages/eve/src/tracing/agent-approval-instrumentation.ts
new file mode 100644
index 000000000..7ab1e2de6
--- /dev/null
+++ b/packages/eve/src/tracing/agent-approval-instrumentation.ts
@@ -0,0 +1,182 @@
+import {
+ ROOT_CONTEXT,
+ SpanStatusCode,
+ type Span,
+ type SpanContext,
+ type Tracer,
+ trace,
+} from "#compiled/@opentelemetry/api/index.js";
+
+import type {
+ InstrumentationHandlerContext,
+ InstrumentationInputRequestedEvent,
+ InstrumentationInputResolvedEvent,
+ InstrumentationProviderDefinition,
+} from "#harness/instrumentation-lifecycle.js";
+import type { JsonValue } from "#shared/json.js";
+import { contentAttribute } from "#tracing/agent-otel-content.js";
+import type { AgentActionContext } from "#tracing/agent-action-instrumentation.js";
+import type { AgentSpanIdGenerator } from "#tracing/agent-span-id-generator.js";
+
+interface AgentApprovalSpanState {
+ readonly actionCallId: string;
+ readonly actionName: string;
+ readonly attemptIndex: number;
+ readonly parent: SpanContext;
+ readonly requestAttribute?: string;
+ readonly requestId: string;
+ readonly rootSessionId: string;
+ readonly sessionId: string;
+ readonly startTimeMs: number;
+ readonly stepIndex: number;
+ readonly turnId: string;
+}
+
+/** Builds durable approval wait spans under their originating runtime action. */
+export function createAgentApprovalInstrumentation(input: {
+ readonly actionContextFor: (
+ sessionId: string,
+ turnId: string,
+ callId: string,
+ ) => Promise<AgentActionContext | undefined>;
+ readonly frameworkVersion: string;
+ readonly idGenerator: AgentSpanIdGenerator;
+ readonly recordInputs: boolean;
+ readonly recordOutputs: boolean;
+ readonly tracer: Tracer;
+}): Pick<
+ NonNullable<InstrumentationProviderDefinition["events"]>,
+ "input.requested" | "input.resolved"
+> {
+ const onRequested = async (
+ event: InstrumentationInputRequestedEvent,
+ ctx: InstrumentationHandlerContext,
+ ): Promise<void> => {
+ if (event.kind !== "tool-approval" || readState(ctx.state.get()) !== undefined) return;
+ const parent = await input.actionContextFor(
+ event.scope.sessionId,
+ event.scope.turnId,
+ event.action.callId,
+ );
+ if (parent === undefined) return;
+ const state: Record<string, JsonValue> = {
+ actionCallId: event.action.callId,
+ actionName: event.action.name,
+ attemptIndex: event.scope.attemptIndex,
+ parent: {
+ spanId: parent.spanContext.spanId,
+ traceFlags: parent.spanContext.traceFlags,
+ traceId: parent.spanContext.traceId,
+ },
+ requestId: event.requestId,
+ rootSessionId: event.scope.rootSessionId ?? event.scope.sessionId,
+ sessionId: event.scope.sessionId,
+ startTimeMs: Date.now(),
+ stepIndex: event.scope.stepIndex,
+ turnId: event.scope.turnId,
+ };
+ const requestAttribute = input.recordInputs
+ ? contentAttribute(event.request, false)
+ : undefined;
+ if (requestAttribute !== undefined) state["requestAttribute"] = requestAttribute;
+ ctx.state.set(state);
+ };
+
+ const onResolved = (
+ event: InstrumentationInputResolvedEvent,
+ ctx: InstrumentationHandlerContext,
+ ): void => {
+ const state = readState(ctx.state.get());
+ if (state === undefined) return;
+ const span = input.idGenerator.withSpanId(
+ input.idGenerator.deriveSpanId(`approval:${event.idempotencyKey}`),
+ () =>
+ input.tracer.startSpan(
+ "agent.approval",
+ {
+ attributes: {
+ "agent.action.call_id": state.actionCallId,
+ "agent.action.name": state.actionName,
+ "agent.approval.kind": "tool-approval",
+ "agent.approval.outcome": event.outcome,
+ "agent.approval.request_id": state.requestId,
+ "agent.framework.name": "eve",
+ "agent.framework.version": input.frameworkVersion,
+ "agent.root.session.id": state.rootSessionId,
+ "agent.session.id": state.sessionId,
+ "agent.step.attempt": state.attemptIndex,
+ "agent.step.index": state.stepIndex,
+ "agent.turn.id": state.turnId,
+ },
+ startTime: state.startTimeMs,
+ },
+ trace.setSpan(ROOT_CONTEXT, trace.wrapSpanContext({ ...state.parent, isRemote: false })),
+ ),
+ );
+ if (state.requestAttribute !== undefined) {
+ span.setAttribute("agent.approval.request", state.requestAttribute);
+ }
+ if (input.recordOutputs && event.response !== undefined) {
+ const response = contentAttribute(event.response, false);
+ if (response !== undefined) span.setAttribute("agent.approval.response", response);
+ }
+ if (event.outcome === "failed") recordError(span, event.error);
+ span.end();
+ };
+
+ return {
+ "input.requested": onRequested,
+ "input.resolved": onResolved,
+ };
+}
+
+function readState(value: unknown): AgentApprovalSpanState | undefined {
+ if (typeof value !== "object" || value === null || Array.isArray(value)) return undefined;
+ const state = value as Record<string, unknown>;
+ const parent = state["parent"];
+ if (typeof parent !== "object" || parent === null || Array.isArray(parent)) return undefined;
+ const parentRecord = parent as Record<string, unknown>;
+ if (
+ typeof state["actionCallId"] !== "string" ||
+ typeof state["actionName"] !== "string" ||
+ typeof state["attemptIndex"] !== "number" ||
+ typeof parentRecord["spanId"] !== "string" ||
+ typeof parentRecord["traceFlags"] !== "number" ||
+ typeof parentRecord["traceId"] !== "string" ||
+ typeof state["requestId"] !== "string" ||
+ typeof state["rootSessionId"] !== "string" ||
+ typeof state["sessionId"] !== "string" ||
+ typeof state["startTimeMs"] !== "number" ||
+ typeof state["stepIndex"] !== "number" ||
+ typeof state["turnId"] !== "string"
+ ) {
+ return undefined;
+ }
+ const requestAttribute = state["requestAttribute"];
+ if (requestAttribute !== undefined && typeof requestAttribute !== "string") return undefined;
+ return {
+ actionCallId: state["actionCallId"],
+ actionName: state["actionName"],
+ attemptIndex: state["attemptIndex"],
+ parent: {
+ isRemote: false,
+ spanId: parentRecord["spanId"],
+ traceFlags: parentRecord["traceFlags"],
+ traceId: parentRecord["traceId"],
+ },
+ requestAttribute,
+ requestId: state["requestId"],
+ rootSessionId: state["rootSessionId"],
+ sessionId: state["sessionId"],
+ startTimeMs: state["startTimeMs"],
+ stepIndex: state["stepIndex"],
+ turnId: state["turnId"],
+ };
+}
+
+function recordError(span: Span, error: unknown): void {
+ if (error instanceof Error) {
+ span.recordException(error);
+ span.setStatus({ code: SpanStatusCode.ERROR, message: error.message });
+ } else span.setStatus({ code: SpanStatusCode.ERROR });
+}
diff --git a/packages/eve/src/tracing/agent-otel-content.test.ts b/packages/eve/src/tracing/agent-otel-content.test.ts
index 0984747d6..b81c3a695 100644
--- a/packages/eve/src/tracing/agent-otel-content.test.ts
+++ b/packages/eve/src/tracing/agent-otel-content.test.ts
@@ -2,9 +2,54 @@ import { describe, expect, it } from "vitest";
import {
CONTENT_ATTRIBUTE_LIMIT,
+ genAiInputMessagesAttribute,
+ genAiOutputMessagesAttribute,
+ genAiSystemInstructionsAttribute,
toolResultsContentAttribute,
} from "#tracing/agent-otel-content.js";
+describe("GenAI message attributes", () => {
+ it("formats model input, output, and system instructions for inspectors", () => {
+ expect(
+ genAiInputMessagesAttribute([
+ { content: "hello", role: "user" },
+ {
+ content: [
+ {
+ input: { message: "echo" },
+ toolCallId: "call-1",
+ toolName: "delegate",
+ type: "tool-call",
+ },
+ ],
+ role: "assistant",
+ },
+ ]),
+ ).toBe(
+ '[{"parts":[{"content":"hello","type":"text"}],"role":"user"},{"parts":[{"arguments":{"message":"echo"},"id":"call-1","name":"delegate","type":"tool_call"}],"role":"assistant"}]',
+ );
+ expect(genAiSystemInstructionsAttribute("Be concise.")).toBe(
+ '[{"content":"Be concise.","type":"text"}]',
+ );
+ expect(
+ genAiOutputMessagesAttribute(
+ [
+ { text: "Working.", type: "text" },
+ {
+ callId: "call-1",
+ input: { message: "echo" },
+ toolName: "delegate",
+ type: "tool-call",
+ },
+ ],
+ "tool-calls",
+ ),
+ ).toBe(
+ '[{"finish_reason":"tool_call","parts":[{"content":"Working.","type":"text"},{"arguments":{"message":"echo"},"id":"call-1","name":"delegate","type":"tool_call"}],"role":"assistant"}]',
+ );
+ });
+});
+
describe("toolResultsContentAttribute", () => {
it("returns the full payload when it fits", () => {
const json = toolResultsContentAttribute([
diff --git a/packages/eve/src/tracing/agent-otel-content.ts b/packages/eve/src/tracing/agent-otel-content.ts
index 12559c47e..9737f2c93 100644
--- a/packages/eve/src/tracing/agent-otel-content.ts
+++ b/packages/eve/src/tracing/agent-otel-content.ts
@@ -53,6 +53,44 @@ export function messagesContentAttribute(messages: unknown): string | undefined
return truncateSingleMessage(stripped);
}
+/** Serializes model messages using the OpenTelemetry GenAI message schema. */
+export function genAiInputMessagesAttribute(messages: unknown): string | undefined {
+ if (!Array.isArray(messages)) return undefined;
+ const formatted = messages.flatMap((message) => {
+ if (!isRecord(message) || message.role === "system" || typeof message.role !== "string") {
+ return [];
+ }
+ const parts = semanticParts(message.content);
+ return parts.length === 0 ? [] : [{ parts, role: message.role }];
+ });
+ for (let start = 0; start < formatted.length; start += 1) {
+ const json = semanticJsonAttribute(formatted.slice(start));
+ if (json !== undefined) return json;
+ }
+ return semanticJsonAttribute([]);
+}
+
+/** Serializes the system prompt using the OpenTelemetry GenAI instruction schema. */
+export function genAiSystemInstructionsAttribute(instructions: unknown): string | undefined {
+ const text = systemPromptAttribute(instructions);
+ return text === undefined ? undefined : semanticJsonAttribute([{ content: text, type: "text" }]);
+}
+
+/** Serializes one model response using the OpenTelemetry GenAI message schema. */
+export function genAiOutputMessagesAttribute(
+ content: readonly unknown[],
+ finishReason: string,
+): string | undefined {
+ const parts = semanticParts(content);
+ return semanticJsonAttribute([
+ {
+ finish_reason: finishReason === "tool-calls" ? "tool_call" : finishReason,
+ parts,
+ role: "assistant",
+ },
+ ]);
+}
+
/**
* Serializes provider-executed tool results, keeping the attribute valid
* JSON under the cap: when the full payload is too big, each entry's input
@@ -146,6 +184,75 @@ function stringifyContent(value: unknown): string | undefined {
}
}
+function semanticJsonAttribute(value: unknown): string | undefined {
+ const json = stringifyContent(value);
+ return json !== undefined && json.length <= CONTENT_ATTRIBUTE_LIMIT ? json : undefined;
+}
+
+function semanticParts(content: unknown): Record<string, unknown>[] {
+ if (typeof content === "string") return [{ content, type: "text" }];
+ if (!Array.isArray(content)) return [];
+ const parts: Record<string, unknown>[] = [];
+ for (const part of content) {
+ const formatted = semanticPart(part);
+ if (formatted !== undefined) parts.push(formatted);
+ }
+ return parts;
+}
+
+function semanticPart(value: unknown): Record<string, unknown> | undefined {
+ if (!isRecord(value) || typeof value.type !== "string") return undefined;
+ switch (value.type) {
+ case "text":
+ case "reasoning": {
+ const content = typeof value.text === "string" ? value.text : value.content;
+ return typeof content === "string" ? { content, type: value.type } : undefined;
+ }
+ case "tool-call":
+ return {
+ arguments: value.input,
+ id: stringValue(value.toolCallId) ?? stringValue(value.callId) ?? null,
+ name: value.toolName,
+ type: "tool_call",
+ };
+ case "tool-result":
+ return {
+ id: stringValue(value.toolCallId) ?? stringValue(value.callId) ?? null,
+ response: toolResponse(value.output),
+ type: "tool_call_response",
+ };
+ case "tool-error":
+ return {
+ id: stringValue(value.toolCallId) ?? stringValue(value.callId) ?? null,
+ response: value.error instanceof Error ? value.error.message : value.error,
+ type: "tool_call_response",
+ };
+ default:
+ return { type: value.type };
+ }
+}
+
+function toolResponse(output: unknown): unknown {
+ if (!isRecord(output)) return output;
+ if (
+ (output.type === "text" ||
+ output.type === "error-text" ||
+ output.type === "json" ||
+ output.type === "error-json") &&
+ "value" in output
+ ) {
+ return output.value;
+ }
+ if (output.type === "execution-denied") {
+ return { denied: true, reason: output.reason };
+ }
+ return output;
+}
+
+function stringValue(value: unknown): string | undefined {
+ return typeof value === "string" ? value : undefined;
+}
+
function isRecord(value: unknown): value is Record<string, unknown> {
return typeof value === "object" && value !== null && !Array.isArray(value);
}
diff --git a/packages/eve/src/tracing/agent-otel-provider.test.ts b/packages/eve/src/tracing/agent-otel-provider.test.ts
index a5b920303..e5611b1cc 100644
--- a/packages/eve/src/tracing/agent-otel-provider.test.ts
+++ b/packages/eve/src/tracing/agent-otel-provider.test.ts
@@ -7,24 +7,39 @@ import {
} from "@opentelemetry/sdk-trace-base";
import { describe, expect, it, vi } from "vitest";
-import { context } from "#compiled/@opentelemetry/api/index.js";
+import {
+ ROOT_CONTEXT,
+ context,
+ trace as runtimeTrace,
+} from "#compiled/@opentelemetry/api/index.js";
+import { ContextContainer, contextStorage } from "#context/container.js";
+import { deserializeContext, serializeContext } from "#context/serialize.js";
import { createAiSdkHookBridge } from "#harness/ai-sdk-hook-bridge.js";
import { createAgentOtelInstrumentation } from "#tracing/agent-otel-provider.js";
import { AgentSpanIdGenerator } from "#tracing/agent-span-id-generator.js";
+import { ContextAgentTraceStateStore } from "#tracing/agent-trace-context-store.js";
import {
+ type AgentTraceStateStore,
InMemoryAgentTraceStateStore,
SESSION_WINDOW_TURN_LIMIT,
} from "#tracing/agent-trace-state.js";
import {
- attemptIdempotencyKey,
createInstrumentationHooks,
- sessionIdempotencyKey,
- turnIdempotencyKey,
+ type InstrumentationActionKind,
type InstrumentationAttemptScope,
type InstrumentationContextRunner,
type InstrumentationHooks,
type InstrumentationParentLineage,
type InstrumentationTraceContext,
+ type InstrumentationUsage,
+} from "#harness/instrumentation-lifecycle.js";
+import {
+ actionIdempotencyKey,
+ attemptIdempotencyKey,
+ inputIdempotencyKey,
+ modelCallIdempotencyKey,
+ sessionIdempotencyKey,
+ turnIdempotencyKey,
} from "#harness/instrumentation-lifecycle.js";
interface TestRuntime {
@@ -32,31 +47,38 @@ interface TestRuntime {
readonly hooks: InstrumentationHooks;
readonly provider: BasicTracerProvider;
readonly runInContext: InstrumentationContextRunner;
+ readonly tracer: ReturnType<BasicTracerProvider["getTracer"]>;
}
-function createRuntime(stateStore = new InMemoryAgentTraceStateStore()): TestRuntime {
+function createRuntime(
+ stateStore: AgentTraceStateStore = new InMemoryAgentTraceStateStore(),
+): TestRuntime {
const exporter = new InMemorySpanExporter();
const idGenerator = new AgentSpanIdGenerator();
const provider = new BasicTracerProvider({
idGenerator,
spanProcessors: [new SimpleSpanProcessor(exporter)],
});
+ const tracer = provider.getTracer("eve.agent");
const agentOtel = createAgentOtelInstrumentation({
frameworkVersion: "test",
idGenerator,
stateStore,
- tracer: provider.getTracer("eve.agent"),
+ tracer,
});
const hooks = createInstrumentationHooks([agentOtel.hook]);
- return { exporter, hooks, provider, runInContext: agentOtel.runInContext };
+ return { exporter, hooks, provider, runInContext: agentOtel.runInContext, tracer };
}
async function emitAttempt(input: {
+ readonly actionUsage?: InstrumentationUsage;
readonly attemptIndex?: number;
readonly attemptError?: Error;
readonly hooks: InstrumentationHooks;
+ readonly parentTraceContext?: InstrumentationTraceContext;
readonly runInContext: InstrumentationContextRunner;
readonly providerMetadata?: Readonly<Record<string, unknown>>;
+ readonly actionKind?: InstrumentationActionKind;
readonly sessionId: string;
readonly skipModelTerminal?: boolean;
readonly skipToolTerminal?: boolean;
@@ -139,6 +161,16 @@ async function emitAttempt(input: {
},
]);
}
+ const actionKey = actionIdempotencyKey(input.sessionId, input.turnId, "tool-1");
+ await input.hooks.publish({
+ callId: "tool-1",
+ idempotencyKey: actionKey,
+ input: { secret: "value" },
+ kind: input.actionKind ?? "tool-call",
+ name: "weather",
+ scope,
+ type: "action.started",
+ });
await Reflect.apply(bridge.onToolExecutionStart!, bridge, [
{
callId: "call-1",
@@ -163,6 +195,25 @@ async function emitAttempt(input: {
: { error: input.toolError, type: "tool-error" },
},
]);
+ await input.hooks.publish(
+ input.toolError === undefined
+ ? {
+ idempotencyKey: actionKey,
+ outcome: "completed",
+ output: { output: { temperature: 72 }, type: "result" },
+ scope,
+ type: "action.completed",
+ usage: input.actionUsage,
+ }
+ : {
+ error: input.toolError,
+ errorCode: "TOOL_CALL_FAILED",
+ idempotencyKey: actionKey,
+ outcome: "failed",
+ scope,
+ type: "action.failed",
+ },
+ );
}
if (input.providerMetadata !== undefined) {
@@ -264,14 +315,21 @@ function nanos(hrTime: readonly [number, number]): bigint {
describe("createAgentOtelInstrumentation", () => {
it("emits the agent hierarchy in one session trace", async () => {
const runtime = createRuntime();
+ const delivery = runtime.tracer.startSpan("workflow.delivery");
+ const activeContext = runtimeTrace.setSpan(ROOT_CONTEXT, delivery);
+ const contextActive = vi.spyOn(context, "active").mockReturnValue(activeContext);
const contextWith = vi.spyOn(context, "with");
- await emitAttempt({
- hooks: runtime.hooks,
- runInContext: runtime.runInContext,
- sessionId: "session-1",
- turnId: "turn-1",
- turnSequence: 0,
- });
+ await context.with(activeContext, () =>
+ emitAttempt({
+ hooks: runtime.hooks,
+ runInContext: runtime.runInContext,
+ sessionId: "session-1",
+ turnId: "turn-1",
+ turnSequence: 0,
+ }),
+ );
+ contextActive.mockRestore();
+ delivery.end();
const executionParents = contextWith.mock.calls.map(([parent]) => parent);
contextWith.mockRestore();
await runtime.provider.forceFlush();
@@ -280,7 +338,7 @@ describe("createAgentOtelInstrumentation", () => {
const turn = byName(spans, "agent.turn")[0]!;
const step = byName(spans, "agent.step")[0]!;
const operation = byName(spans, "ai.streamText")[0]!;
- const model = byName(spans, "ai.streamText.doStream")[0]!;
+ const model = byName(spans, "chat claude-test")[0]!;
const action = byName(spans, "agent.action")[0]!;
const tool = byName(spans, "ai.toolCall")[0]!;
@@ -295,6 +353,12 @@ describe("createAgentOtelInstrumentation", () => {
expect(turn.parentSpanContext?.spanId).toBe(session.spanContext().spanId);
expect(step.parentSpanContext?.spanId).toBe(turn.spanContext().spanId);
expect(operation.parentSpanContext?.spanId).toBe(step.spanContext().spanId);
+ expect(step.links).toEqual([
+ expect.objectContaining({
+ attributes: { "eve.link.type": "workflow.delivery" },
+ context: delivery.spanContext(),
+ }),
+ ]);
expect(model.parentSpanContext?.spanId).toBe(operation.spanContext().spanId);
expect(
executionParents.some(
@@ -302,14 +366,16 @@ describe("createAgentOtelInstrumentation", () => {
apiTrace.getSpan(parent as never)?.spanContext().spanId === model.spanContext().spanId,
),
).toBe(true);
- expect(action.parentSpanContext?.spanId).toBe(step.spanContext().spanId);
+ expect(action.parentSpanContext?.spanId).toBe(turn.spanContext().spanId);
expect(tool.parentSpanContext?.spanId).toBe(action.spanContext().spanId);
- expect(new Set(spans.map((span) => span.spanContext().traceId))).toHaveLength(1);
- expect(turn.events.map((event) => event.name)).toEqual([
- "turn.started",
- "turn.completed",
- "session.waiting",
- ]);
+ expect(
+ new Set(
+ spans
+ .filter((span) => span.name !== "workflow.delivery")
+ .map((span) => span.spanContext().traceId),
+ ),
+ ).toHaveLength(1);
+ expect(turn.events.map((event) => event.name)).toEqual(["turn.started", "turn.completed"]);
// Turn timestamps are millisecond-quantized (`Date.now`), so the end
// comparison against the step's sub-millisecond clock gets 1ms of slack.
expect(turn.attributes).toMatchObject({ "agent.name": "weather", "agent.session.window": 0 });
@@ -326,14 +392,346 @@ describe("createAgentOtelInstrumentation", () => {
"gen_ai.usage.cache_read.input_tokens": 4,
});
expect(action.attributes).toMatchObject({
- "agent.action.kind": "tool",
+ "agent.action.kind": "tool-call",
"agent.action.name": "weather",
"agent.framework.name": "eve",
"agent.root.session.id": "session-1",
});
});
- it("ends model and tool spans still open when the step attempt terminates", async () => {
+ it("keeps logical ids stable but separates physical redeliveries", async () => {
+ const first = createRuntime();
+ const redelivery = createRuntime();
+ const parentTraceContext: InstrumentationTraceContext = {
+ spanId: "b".repeat(16),
+ traceFlags: 1,
+ traceId: "a".repeat(32),
+ };
+
+ for (const runtime of [first, redelivery]) {
+ await emitAttempt({
+ hooks: runtime.hooks,
+ parentTraceContext,
+ runInContext: runtime.runInContext,
+ sessionId: "session-1",
+ turnId: "turn-1",
+ turnSequence: 0,
+ });
+ await runtime.provider.forceFlush();
+ }
+
+ const firstSpans = first.exporter.getFinishedSpans();
+ const redeliverySpans = redelivery.exporter.getFinishedSpans();
+ for (const name of ["agent.turn", "agent.action", "ai.toolCall"]) {
+ expect(byName(redeliverySpans, name)[0]!.spanContext().spanId).toBe(
+ byName(firstSpans, name)[0]!.spanContext().spanId,
+ );
+ }
+ for (const name of ["agent.step", "ai.streamText", "chat claude-test"]) {
+ expect(byName(redeliverySpans, name)[0]!.spanContext().spanId).not.toBe(
+ byName(firstSpans, name)[0]!.spanContext().spanId,
+ );
+ }
+ });
+
+ it("parents a tool to its action when SDK telemetry arrives first", async () => {
+ const runtime = createRuntime();
+ const scope: InstrumentationAttemptScope = {
+ attemptId: "session-1:turn-1:0:0",
+ attemptIndex: 0,
+ sessionId: "session-1",
+ stepIndex: 0,
+ turnId: "turn-1",
+ };
+ const actionKey = actionIdempotencyKey(scope.sessionId, scope.turnId, "tool-1");
+ const toolKey = `tool:${scope.attemptId}:tool-1:0`;
+
+ await publishTurnStarted({
+ hooks: runtime.hooks,
+ sessionId: scope.sessionId,
+ turnId: scope.turnId,
+ turnSequence: 0,
+ });
+ await runtime.hooks.publish({
+ idempotencyKey: attemptIdempotencyKey(scope),
+ operation: { modelId: "model", operationId: "ai.streamText", provider: "test" },
+ scope,
+ type: "step.attempt.started",
+ });
+ await runtime.hooks.publish({
+ callId: "tool-1",
+ idempotencyKey: toolKey,
+ input: { secret: "value" },
+ scope,
+ toolName: "weather",
+ type: "tool.call.started",
+ });
+ await runtime.hooks.publish({
+ idempotencyKey: toolKey,
+ output: { output: "sunny", type: "result" },
+ scope,
+ type: "tool.call.completed",
+ });
+ await runtime.hooks.publish({
+ callId: "tool-1",
+ idempotencyKey: actionKey,
+ input: { secret: "value" },
+ kind: "tool-call",
+ name: "weather",
+ scope,
+ type: "action.started",
+ });
+ await runtime.hooks.publish({
+ idempotencyKey: actionKey,
+ outcome: "completed",
+ output: { output: "sunny", type: "result" },
+ scope,
+ type: "action.completed",
+ });
+ const uncorrelatedToolKey = `tool:${scope.attemptId}:tool-2:0`;
+ await runtime.hooks.publish({
+ callId: "tool-2",
+ idempotencyKey: uncorrelatedToolKey,
+ input: {},
+ scope,
+ toolName: "final_output",
+ type: "tool.call.started",
+ });
+ await runtime.hooks.publish({
+ idempotencyKey: uncorrelatedToolKey,
+ output: { output: "done", type: "result" },
+ scope,
+ type: "tool.call.completed",
+ });
+ await runtime.hooks.publish({
+ idempotencyKey: attemptIdempotencyKey(scope),
+ scope,
+ type: "step.attempt.completed",
+ });
+ await runtime.provider.forceFlush();
+
+ const spans = runtime.exporter.getFinishedSpans();
+ const action = byName(spans, "agent.action")[0]!;
+ const [tool, uncorrelatedTool] = byName(spans, "ai.toolCall");
+ expect(tool!.parentSpanContext?.spanId).toBe(action.spanContext().spanId);
+ expect(uncorrelatedTool!.parentSpanContext?.spanId).toBe(
+ byName(spans, "agent.step")[0]!.spanContext().spanId,
+ );
+ });
+
+ it("reconstructs a durable action span in a replacement worker", async () => {
+ const first = createRuntime(new ContextAgentTraceStateStore());
+ const context = new ContextContainer();
+ const scope: InstrumentationAttemptScope = {
+ attemptId: "session-1:turn-1:0:0",
+ attemptIndex: 0,
+ sessionId: "session-1",
+ stepIndex: 0,
+ turnId: "turn-1",
+ };
+ const actionKey = actionIdempotencyKey(scope.sessionId, scope.turnId, "tool-1");
+ const toolKey = `tool:${scope.attemptId}:tool-1:0`;
+
+ await contextStorage.run(context, async () => {
+ await publishTurnStarted({
+ hooks: first.hooks,
+ sessionId: scope.sessionId,
+ turnId: scope.turnId,
+ turnSequence: 0,
+ });
+ await first.hooks.publish({
+ idempotencyKey: attemptIdempotencyKey(scope),
+ operation: { modelId: "model", operationId: "ai.streamText", provider: "test" },
+ scope,
+ type: "step.attempt.started",
+ });
+ await first.hooks.publish({
+ callId: "tool-1",
+ idempotencyKey: actionKey,
+ input: { secret: "value" },
+ kind: "tool-call",
+ name: "weather",
+ scope,
+ type: "action.started",
+ });
+ await first.hooks.publish({
+ idempotencyKey: attemptIdempotencyKey(scope),
+ scope,
+ type: "step.attempt.completed",
+ });
+ });
+ await first.provider.forceFlush();
+ const firstSpans = first.exporter.getFinishedSpans();
+ const step = byName(firstSpans, "agent.step")[0]!;
+
+ await new Promise((resolve) => setTimeout(resolve, 2));
+ const acceptedAtMs = Date.now() + 1_000;
+ const restored = await deserializeContext(await serializeContext(context));
+ const replacement = createRuntime(new ContextAgentTraceStateStore());
+ const replacementScope = {
+ ...scope,
+ attemptId: "session-1:turn-2:0:0",
+ turnId: "turn-2",
+ };
+ await contextStorage.run(restored, async () => {
+ // Approval-resumed tools can execute before the replacement AI SDK emits
+ // a new step start. The persisted action context is still their parent.
+ await replacement.hooks.publish({
+ callId: "tool-1",
+ idempotencyKey: toolKey,
+ input: {},
+ scope: replacementScope,
+ toolName: "weather",
+ type: "tool.call.started",
+ });
+ await replacement.hooks.publish({
+ idempotencyKey: toolKey,
+ output: { output: "ok", type: "result" },
+ scope: replacementScope,
+ type: "tool.call.completed",
+ });
+ await replacement.hooks.publish({
+ acceptedAtMs,
+ idempotencyKey: actionKey,
+ outcome: "completed",
+ output: { output: { temperature: 72 }, type: "result" },
+ scope,
+ type: "action.completed",
+ });
+ });
+ await replacement.provider.forceFlush();
+
+ const replacementSpans = replacement.exporter.getFinishedSpans();
+ const action = byName(replacementSpans, "agent.action")[0]!;
+ const tool = byName(replacementSpans, "ai.toolCall")[0]!;
+ expect(action.spanContext().spanId).toBe(tool.parentSpanContext?.spanId);
+ expect(action.parentSpanContext?.spanId).toBe(step.parentSpanContext?.spanId);
+ expect(action.attributes).toMatchObject({
+ "agent.action.kind": "tool-call",
+ "agent.action.name": "weather",
+ "gen_ai.tool.call.arguments": expect.stringContaining("secret"),
+ "gen_ai.tool.call.result": expect.stringContaining("temperature"),
+ });
+ expect(nanos(action.duration)).toBeGreaterThan(0n);
+ expect(nanos(action.endTime)).toBe(BigInt(acceptedAtMs) * 1_000_000n);
+ });
+
+ it("reconstructs a durable approval span in a replacement worker", async () => {
+ const first = createRuntime(new ContextAgentTraceStateStore());
+ const context = new ContextContainer();
+ const scope: InstrumentationAttemptScope = {
+ attemptId: "session-1:turn-1:0:0",
+ attemptIndex: 0,
+ sessionId: "session-1",
+ stepIndex: 0,
+ turnId: "turn-1",
+ };
+ const actionKey = actionIdempotencyKey(scope.sessionId, scope.turnId, "tool-1");
+ const inputKey = inputIdempotencyKey(scope.sessionId, scope.turnId, "approval-1");
+
+ await contextStorage.run(context, async () => {
+ await publishTurnStarted({
+ hooks: first.hooks,
+ sessionId: scope.sessionId,
+ turnId: scope.turnId,
+ turnSequence: 0,
+ });
+ await first.hooks.publish({
+ callId: "tool-1",
+ idempotencyKey: actionKey,
+ input: { city: "SF" },
+ kind: "tool-call",
+ name: "weather",
+ scope,
+ type: "action.started",
+ });
+ await first.hooks.publish({
+ action: { callId: "tool-1", name: "weather" },
+ idempotencyKey: inputKey,
+ kind: "tool-approval",
+ request: { prompt: "Approve weather?" },
+ requestId: "approval-1",
+ scope,
+ type: "input.requested",
+ });
+ });
+
+ await new Promise((resolve) => setTimeout(resolve, 2));
+ const restored = await deserializeContext(await serializeContext(context));
+ const replacement = createRuntime(new ContextAgentTraceStateStore());
+ await contextStorage.run(restored, async () => {
+ await replacement.hooks.publish({
+ idempotencyKey: inputKey,
+ kind: "tool-approval",
+ outcome: "approved",
+ requestId: "approval-1",
+ response: { optionId: "approve" },
+ scope,
+ type: "input.resolved",
+ });
+ await replacement.hooks.publish({
+ idempotencyKey: actionKey,
+ outcome: "completed",
+ output: { output: { temperature: 72 }, type: "result" },
+ scope,
+ type: "action.completed",
+ });
+ });
+ await replacement.provider.forceFlush();
+
+ const spans = replacement.exporter.getFinishedSpans();
+ const approval = byName(spans, "agent.approval")[0]!;
+ const action = byName(spans, "agent.action")[0]!;
+ expect(approval.parentSpanContext?.spanId).toBe(action.spanContext().spanId);
+ expect(approval.status.code).toBe(SpanStatusCode.UNSET);
+ expect(approval.attributes).toMatchObject({
+ "agent.action.call_id": "tool-1",
+ "agent.action.name": "weather",
+ "agent.approval.kind": "tool-approval",
+ "agent.approval.outcome": "approved",
+ "agent.approval.request": expect.stringContaining("Approve weather?"),
+ "agent.approval.request_id": "approval-1",
+ "agent.approval.response": expect.stringContaining("approve"),
+ "agent.session.id": "session-1",
+ "agent.step.index": 0,
+ "agent.turn.id": "turn-1",
+ });
+ expect(nanos(approval.duration)).toBeGreaterThan(0n);
+ });
+
+ it("does not create approval spans for other input requests", async () => {
+ const runtime = createRuntime();
+ const scope: InstrumentationAttemptScope = {
+ attemptId: "session-1:turn-1:0:0",
+ attemptIndex: 0,
+ sessionId: "session-1",
+ stepIndex: 0,
+ turnId: "turn-1",
+ };
+ const key = inputIdempotencyKey(scope.sessionId, scope.turnId, "question-1");
+ await runtime.hooks.publish({
+ action: { callId: "question-1", name: "ask_question" },
+ idempotencyKey: key,
+ kind: "question",
+ request: { prompt: "Which region?" },
+ requestId: "question-1",
+ scope,
+ type: "input.requested",
+ });
+ await runtime.hooks.publish({
+ idempotencyKey: key,
+ kind: "question",
+ outcome: "answered",
+ requestId: "question-1",
+ response: { text: "west" },
+ scope,
+ type: "input.resolved",
+ });
+ await runtime.provider.forceFlush();
+ expect(byName(runtime.exporter.getFinishedSpans(), "agent.approval")).toHaveLength(0);
+ });
+
+ it("ends SDK spans but leaves durable actions for their own terminal", async () => {
const runtime = createRuntime();
await emitAttempt({
hooks: runtime.hooks,
@@ -347,8 +745,8 @@ describe("createAgentOtelInstrumentation", () => {
await runtime.provider.forceFlush();
const spans = runtime.exporter.getFinishedSpans();
- expect(byName(spans, "ai.streamText.doStream")).toHaveLength(1);
- expect(byName(spans, "agent.action")).toHaveLength(1);
+ expect(byName(spans, "chat claude-test")).toHaveLength(1);
+ expect(byName(spans, "agent.action")).toHaveLength(0);
expect(byName(spans, "ai.toolCall")).toHaveLength(1);
expect(byName(spans, "agent.step")).toHaveLength(1);
});
@@ -369,7 +767,7 @@ describe("createAgentOtelInstrumentation", () => {
await runtime.provider.forceFlush();
const spans = runtime.exporter.getFinishedSpans();
- for (const name of ["ai.streamText.doStream", "agent.action", "ai.toolCall"]) {
+ for (const name of ["chat claude-test", "agent.action", "ai.toolCall"]) {
const span = byName(spans, name)[0]!;
expect(span.status).toEqual({ code: SpanStatusCode.ERROR, message: error.message });
expect(span.events).toContainEqual(
@@ -381,6 +779,35 @@ describe("createAgentOtelInstrumentation", () => {
}
});
+ it("labels a subagent action by its kind, not as a plain tool", async () => {
+ const runtime = createRuntime();
+ await emitAttempt({
+ actionUsage: {
+ inputTokenDetails: { cacheReadTokens: 3, cacheWriteTokens: 4 },
+ inputTokens: 10,
+ outputTokens: 5,
+ },
+ hooks: runtime.hooks,
+ actionKind: "subagent-call",
+ runInContext: runtime.runInContext,
+ sessionId: "session-1",
+ turnId: "turn-1",
+ turnSequence: 0,
+ });
+ await runtime.provider.forceFlush();
+
+ const action = runtime.exporter.getFinishedSpans().find((span) => span.name === "agent.action");
+ expect(action?.attributes).toMatchObject({
+ "agent.action.kind": "subagent-call",
+ "agent.action.name": "weather",
+ "agent.action.outcome": "completed",
+ "agent.usage.input_tokens": 10,
+ "agent.usage.output_tokens": 5,
+ "gen_ai.usage.cache_creation.input_tokens": 4,
+ "gen_ai.usage.cache_read.input_tokens": 3,
+ });
+ });
+
it("captures model and tool inputs/outputs on the operation spans", async () => {
const runtime = createRuntime();
await emitAttempt({
@@ -393,7 +820,7 @@ describe("createAgentOtelInstrumentation", () => {
await runtime.provider.forceFlush();
const spans = runtime.exporter.getFinishedSpans();
- const model = byName(spans, "ai.streamText.doStream")[0]!;
+ const model = byName(spans, "chat claude-test")[0]!;
const tool = byName(spans, "ai.toolCall")[0]!;
// Provider transport noise (signatures et al.) is stripped at capture time.
expect(model.attributes["ai.prompt.messages"]).toBe(
@@ -405,6 +832,17 @@ describe("createAgentOtelInstrumentation", () => {
expect(model.attributes["ai.response.finish_reason"]).toBe("tool-calls");
expect(model.attributes["ai.response.reasoning"]).toBe("thinking about weather");
expect(model.attributes["ai.response.text"]).toBe("Checking the weather.");
+ expect(model.attributes).toMatchObject({
+ "gen_ai.agent.name": "weather",
+ "gen_ai.input.messages":
+ '[{"parts":[{"content":"real user text","type":"text"}],"role":"user"}]',
+ "gen_ai.operation.name": "chat",
+ "gen_ai.output.messages": expect.stringContaining('"finish_reason":"tool_call"'),
+ "gen_ai.response.finish_reasons": ["tool-calls"],
+ "gen_ai.system_instructions":
+ '[{"content":"You are a weather assistant (system prompt).","type":"text"}]',
+ });
+ expect(model.attributes["agent.input.messages.delta"]).toBeUndefined();
// Provider-executed tools never reach the tool loop; their calls and
// results are captured off the model response content.
expect(model.attributes["ai.response.tool_calls"]).toBe(
@@ -415,13 +853,13 @@ describe("createAgentOtelInstrumentation", () => {
);
expect(tool.attributes["gen_ai.tool.call.arguments"]).toBe('{"secret":"value"}');
expect(tool.attributes["gen_ai.tool.call.result"]).toBe('{"temperature":72}');
- // Structural spans stay structural: content lives only on the operation spans.
- const structural = byName(spans, "agent.action")[0]!;
- expect(JSON.stringify(structural.attributes)).not.toContain("secret");
- expect(JSON.stringify(structural.attributes)).not.toContain("temperature");
+ // Runtime action spans carry content for dispatches that have no SDK tool boundary.
+ const action = byName(spans, "agent.action")[0]!;
+ expect(action.attributes["gen_ai.tool.call.arguments"]).toContain("secret");
+ expect(action.attributes["gen_ai.tool.call.result"]).toContain("temperature");
});
- it("truncates long conversations from the front, keeping valid JSON and recent messages", async () => {
+ it("caps full model input while keeping valid message JSON", async () => {
const runtime = createRuntime();
const manyMessages = Array.from({ length: 200 }, (_, index) => ({
content: `message ${index} ${"x".repeat(200)}`,
@@ -478,16 +916,18 @@ describe("createAgentOtelInstrumentation", () => {
]);
await runtime.provider.forceFlush();
- const model = byName(runtime.exporter.getFinishedSpans(), "ai.streamText.doStream")[0]!;
+ const model = byName(runtime.exporter.getFinishedSpans(), "chat claude-test")[0]!;
const raw = model.attributes["ai.prompt.messages"];
expect(typeof raw).toBe("string");
expect((raw as string).length).toBeLessThanOrEqual(32 * 1024);
const parsed = JSON.parse(raw as string) as Array<Record<string, unknown>>;
- const marker = parsed[0]!["eve.truncated"] as { omittedMessages: number };
- expect(marker.omittedMessages).toBeGreaterThan(0);
- expect(marker.omittedMessages).toBeLessThan(200);
+ expect(parsed.length).toBeGreaterThan(1);
+ expect(parsed[0]).toMatchObject({
+ "eve.truncated": { omittedMessages: expect.any(Number) },
+ });
expect(JSON.stringify(parsed)).toContain("message 199");
expect(JSON.stringify(parsed)).not.toContain("message 0 ");
+ expect(model.attributes["agent.input.messages.delta"]).toBeUndefined();
});
it("captures nothing when content capture is off", async () => {
@@ -523,6 +963,56 @@ describe("createAgentOtelInstrumentation", () => {
expect(JSON.stringify(spans.map((span) => span.attributes))).not.toContain("system prompt");
});
+ it("resolves execution context by attempt identity across a scope snapshot", async () => {
+ const runtime = createRuntime();
+ const scope: InstrumentationAttemptScope = {
+ attemptId: "session-1:turn-1:0:0",
+ attemptIndex: 0,
+ functionId: "weather",
+ sessionId: "session-1",
+ stepIndex: 0,
+ turnId: "turn-1",
+ };
+ await runtime.hooks.publish({
+ agentName: "weather",
+ idempotencyKey: sessionIdempotencyKey("session-1"),
+ rootSessionId: "session-1",
+ sessionId: "session-1",
+ type: "session.started",
+ });
+ await runtime.hooks.publish({
+ idempotencyKey: turnIdempotencyKey("session-1", "turn-1"),
+ rootSessionId: "session-1",
+ sequence: 0,
+ sessionId: "session-1",
+ turnId: "turn-1",
+ type: "turn.started",
+ });
+ await runtime.hooks.publish({
+ idempotencyKey: attemptIdempotencyKey(scope),
+ operation: { modelId: "model", operationId: "ai.streamText", provider: "test" },
+ scope,
+ type: "step.attempt.started",
+ });
+ const idempotencyKey = modelCallIdempotencyKey(scope, 0);
+ await runtime.hooks.publish({
+ idempotencyKey,
+ input: { messages: [] },
+ model: { modelId: "model", provider: "test" },
+ scope,
+ type: "model.call.started",
+ });
+
+ const withSpy = vi.spyOn(context, "with");
+ await runtime.runInContext(
+ { idempotencyKey, scope: { ...scope }, type: "model.call" },
+ async () => undefined,
+ );
+
+ expect(withSpy).toHaveBeenCalledOnce();
+ withSpy.mockRestore();
+ });
+
it("writes gateway cost attributes on the step span when the gateway reports them", async () => {
const runtime = createRuntime();
await emitAttempt({
@@ -602,6 +1092,21 @@ describe("createAgentOtelInstrumentation", () => {
expect(turns).toHaveLength(2);
expect(turns[0]!.spanContext().traceId).toBe(turns[1]!.spanContext().traceId);
expect(turns.every((turn) => turn.attributes["agent.session.window"] === 0)).toBe(true);
+ const firstModel = byName(firstRuntime.exporter.getFinishedSpans(), "chat claude-test")[0]!;
+ const secondModel = byName(secondRuntime.exporter.getFinishedSpans(), "chat claude-test")[0]!;
+ expect(firstModel.attributes["ai.prompt.system"]).toBe(
+ "You are a weather assistant (system prompt).",
+ );
+ expect(secondModel.attributes["ai.prompt.system"]).toBe(
+ "You are a weather assistant (system prompt).",
+ );
+ expect(secondModel.attributes["gen_ai.system_instructions"]).toBe(
+ '[{"content":"You are a weather assistant (system prompt).","type":"text"}]',
+ );
+ expect(secondModel.attributes["agent.input.messages.delta"]).toBeUndefined();
+ expect(secondModel.attributes["ai.prompt.messages"]).toBe(
+ '[{"content":"real user text","role":"user"}]',
+ );
expect([
...byName(firstRuntime.exporter.getFinishedSpans(), "agent.session"),
...byName(secondRuntime.exporter.getFinishedSpans(), "agent.session"),
@@ -639,7 +1144,7 @@ describe("createAgentOtelInstrumentation", () => {
const action = byName(replacementSpans, "agent.action")[0]!;
expect(step.parentSpanContext?.spanId).toBe(turn.spanContext().spanId);
- expect(action.parentSpanContext?.spanId).toBe(step.spanContext().spanId);
+ expect(action.parentSpanContext?.spanId).toBe(turn.spanContext().spanId);
expect(step.spanContext().traceId).toBe(turn.spanContext().traceId);
});
@@ -740,6 +1245,31 @@ describe("createAgentOtelInstrumentation", () => {
expect(byName(spans, "agent.session")).toHaveLength(1);
});
+ it("preserves a remote action as the parent of a remote child turn", async () => {
+ const runtime = createRuntime();
+ const parentTraceContext: InstrumentationTraceContext & { readonly isRemote: true } = {
+ isRemote: true,
+ spanId: "2".repeat(16),
+ traceFlags: 1,
+ traceId: "1".repeat(32),
+ };
+ await publishTurnStarted({
+ hooks: runtime.hooks,
+ parentTraceContext,
+ rootSessionId: "parent-session",
+ sessionId: "remote-child",
+ turnId: "child-turn-1",
+ turnSequence: 0,
+ });
+ await completeTurn(runtime.hooks, "remote-child", "child-turn-1");
+ await runtime.provider.forceFlush();
+
+ const childTurn = byName(runtime.exporter.getFinishedSpans(), "agent.turn")[0]!;
+ expect(childTurn.spanContext().traceId).toBe(parentTraceContext.traceId);
+ expect(childTurn.parentSpanContext).toMatchObject(parentTraceContext);
+ expect(childTurn.attributes["agent.root.session.id"]).toBe("parent-session");
+ });
+
it("attributes a child turn to the exact call that dispatched it", async () => {
const runtime = createRuntime();
await publishTurnStarted({
@@ -869,7 +1399,13 @@ describe("createAgentOtelInstrumentation", () => {
await runtime.provider.forceFlush();
const spans = runtime.exporter.getFinishedSpans();
- expect(byName(spans, "agent.action")[0]!.status.code).toBe(SpanStatusCode.ERROR);
+ const action = byName(spans, "agent.action")[0]!;
+ expect(action.status.code).toBe(SpanStatusCode.ERROR);
+ expect(action.attributes).toMatchObject({
+ "agent.action.error.code": "TOOL_CALL_FAILED",
+ "agent.action.outcome": "failed",
+ "error.type": "TOOL_CALL_FAILED",
+ });
expect(byName(spans, "agent.turn")[0]!.status.code).toBe(SpanStatusCode.UNSET);
});
});
diff --git a/packages/eve/src/tracing/agent-otel-provider.ts b/packages/eve/src/tracing/agent-otel-provider.ts
index 245f576a7..9ff4b7280 100644
--- a/packages/eve/src/tracing/agent-otel-provider.ts
+++ b/packages/eve/src/tracing/agent-otel-provider.ts
@@ -16,12 +16,19 @@ import {
} from "#tracing/agent-trace-state.js";
import {
contentAttribute,
+ genAiInputMessagesAttribute,
+ genAiOutputMessagesAttribute,
+ genAiSystemInstructionsAttribute,
messagesContentAttribute,
systemPromptAttribute,
textContentAttribute,
toolResultsContentAttribute,
} from "#tracing/agent-otel-content.js";
import type { AgentSpanIdGenerator } from "#tracing/agent-span-id-generator.js";
+import { createAgentActionInstrumentation } from "#tracing/agent-action-instrumentation.js";
+import { createAgentApprovalInstrumentation } from "#tracing/agent-approval-instrumentation.js";
+import { createAgentToolInstrumentation } from "#tracing/agent-tool-instrumentation.js";
+import { setAgentUsage } from "#tracing/agent-otel-usage.js";
import type {
InstrumentationStepAttemptMetadataEvent,
InstrumentationAttemptScope,
@@ -35,11 +42,8 @@ import type {
InstrumentationParentLineage,
InstrumentationTraceContext,
InstrumentationSessionTransitionEvent,
- InstrumentationToolCallStartedEvent,
- InstrumentationToolCallTerminalEvent,
InstrumentationTurnStartedEvent,
InstrumentationTurnTerminalEvent,
- InstrumentationUsage,
} from "#harness/instrumentation-lifecycle.js";
import { sessionIdempotencyKey } from "#harness/instrumentation-lifecycle.js";
@@ -53,10 +57,6 @@ interface AttemptSpanState {
readonly step: SpanState;
}
-interface ToolSpanState extends SpanState {
- readonly toolSpan: Span;
-}
-
export interface AgentOtelInstrumentationInput {
/** Whether any destination requested model prompts and tool inputs. */
readonly recordInputs?: boolean;
@@ -80,17 +80,49 @@ export function createAgentOtelInstrumentation(
): AgentOtelInstrumentation {
const recordInputs = input.recordInputs ?? true;
const recordOutputs = input.recordOutputs ?? true;
- const executionContexts = new WeakMap<
- InstrumentationAttemptScope,
- { readonly models: Map<string, Context>; readonly tools: Map<string, Context> }
- >();
+ const executionContexts = new WeakMap<InstrumentationAttemptScope, Map<string, Context>>();
const attemptScopes = new Map<string, InstrumentationAttemptScope>();
// A serverless turn runs inside one `turnStep` "use step" invocation. If
// that worker is lost, Workflow retries the whole step from entry rather
// than resuming this callback sequence in a replacement process.
const steps = new WeakMap<InstrumentationAttemptScope, AttemptSpanState>();
const modelSpans = new WeakMap<InstrumentationAttemptScope, Map<string, SpanState>>();
- const toolSpans = new WeakMap<InstrumentationAttemptScope, Map<string, ToolSpanState>>();
+ const actions = createAgentActionInstrumentation({
+ frameworkVersion: input.frameworkVersion,
+ idGenerator: input.idGenerator,
+ recordInputs,
+ recordOutputs,
+ resolveParent: async (event) => {
+ const turn = await input.stateStore.getTurn(event.scope.sessionId, event.scope.turnId);
+ return turn === undefined
+ ? undefined
+ : { context: contextFromSpanContext(turn.context), spanContext: turn.context };
+ },
+ stateStore: input.stateStore,
+ tracer: input.tracer,
+ });
+ const approvals = createAgentApprovalInstrumentation({
+ actionContextFor: actions.contextFor,
+ frameworkVersion: input.frameworkVersion,
+ idGenerator: input.idGenerator,
+ recordInputs,
+ recordOutputs,
+ tracer: input.tracer,
+ });
+ const tools = createAgentToolInstrumentation({
+ actionContextFor: actions.contextFor,
+ idGenerator: input.idGenerator,
+ recordInputs,
+ recordOutputs,
+ resolveFallback: (event) => {
+ const scope = attemptScopes.get(event.scope.attemptId) ?? event.scope;
+ const step = steps.get(scope)?.step;
+ return step === undefined
+ ? undefined
+ : { context: step.context, spanContext: step.span.spanContext() };
+ },
+ tracer: input.tracer,
+ });
const onSessionStarted = async (event: InstrumentationSessionStartedEvent): Promise<void> => {
await ensureSessionContext(event);
@@ -114,7 +146,7 @@ export function createAgentOtelInstrumentation(
// itself is emitted at the turn's session transition.
const turnContext: SpanContext = {
isRemote: false,
- spanId: input.idGenerator.allocateSpanId(),
+ spanId: input.idGenerator.deriveSpanId(`turn:${event.idempotencyKey}`),
traceFlags: session.context.traceFlags,
traceId: session.context.traceId,
};
@@ -125,6 +157,7 @@ export function createAgentOtelInstrumentation(
await input.stateStore.setTurn(event.sessionId, event.turnId, {
context: turnContext,
lineage: event.parentLineage,
+ parentIsRemote: session.context.isRemote,
parentSpanId: session.context.spanId,
rootSessionId: event.rootSessionId,
sequence: event.sequence,
@@ -136,6 +169,7 @@ export function createAgentOtelInstrumentation(
const turn = await input.stateStore.getTurn(event.scope.sessionId, event.scope.turnId);
if (turn === undefined) return;
const turnContext = contextFromSpanContext(turn.context);
+ const activeSpanContext = trace.getSpan(context.active())?.spanContext();
const stepSpan = input.tracer.startSpan(
"agent.step",
{
@@ -149,6 +183,15 @@ export function createAgentOtelInstrumentation(
"agent.turn.id": event.scope.turnId,
"agent.name": event.scope.functionId,
},
+ links:
+ activeSpanContext === undefined || activeSpanContext.traceId === turn.context.traceId
+ ? undefined
+ : [
+ {
+ attributes: { "eve.link.type": "workflow.delivery" },
+ context: activeSpanContext,
+ },
+ ],
},
turnContext,
);
@@ -159,9 +202,9 @@ export function createAgentOtelInstrumentation(
operationName,
{
attributes: {
- "gen_ai.operation.name": operationName,
- "gen_ai.provider.name": event.operation.provider,
- "gen_ai.request.model": event.operation.modelId,
+ "ai.operation.name": operationName,
+ "ai.provider.name": event.operation.provider,
+ "ai.request.model": event.operation.modelId,
},
},
stepContext,
@@ -177,10 +220,17 @@ export function createAgentOtelInstrumentation(
attemptScopes.set(event.scope.attemptId, event.scope);
};
- const onStepTerminal = (event: InstrumentationStepAttemptTerminalEvent): void => {
+ const onStepTerminal = async (event: InstrumentationStepAttemptTerminalEvent): Promise<void> => {
const scope = attemptScopes.get(event.scope.attemptId) ?? event.scope;
executionContexts.delete(scope);
drainOpenSpans({ ...event, scope });
+ tools.drain(
+ event.scope.attemptId,
+ event.type === "step.attempt.failed" ? { error: event.error } : undefined,
+ );
+ if (event.type === "step.attempt.failed") {
+ await actions.failForAttempt(scope, event.error);
+ }
attemptScopes.delete(event.scope.attemptId);
const attempt = steps.get(scope);
if (attempt === undefined) return;
@@ -199,6 +249,9 @@ export function createAgentOtelInstrumentation(
};
const onTurnTerminal = async (event: InstrumentationTurnTerminalEvent): Promise<void> => {
+ if (event.type === "turn.cancelled" || event.type === "turn.failed") {
+ await actions.deleteForTurn(event.sessionId, event.turnId);
+ }
const turn = await input.stateStore.getTurn(event.sessionId, event.turnId);
if (turn === undefined) return;
await input.stateStore.setTurn(event.sessionId, event.turnId, {
@@ -235,7 +288,7 @@ export function createAgentOtelInstrumentation(
startTime: turn.startTimeMs,
},
contextFromSpanContext({
- isRemote: false,
+ isRemote: turn.parentIsRemote ?? false,
spanId: turn.parentSpanId,
traceFlags: turn.context.traceFlags,
traceId: turn.context.traceId,
@@ -249,7 +302,6 @@ export function createAgentOtelInstrumentation(
recordError(span, turn.terminal.error);
}
}
- span.addEvent(event.type);
span.end();
await input.stateStore.deleteTurn(event.sessionId, event.turnId);
}
@@ -258,6 +310,7 @@ export function createAgentOtelInstrumentation(
// turn that still needs its metadata — so only release session-scoped
// state on terminal transitions.
if (event.type === "session.completed" || event.type === "session.failed") {
+ await actions.deleteForSession(event.sessionId);
await input.stateStore.deleteSession(event.sessionId);
}
};
@@ -268,10 +321,11 @@ export function createAgentOtelInstrumentation(
attempt.step.span.setAttribute("agent.model.id", event.model.modelId);
attempt.step.span.setAttribute("agent.model.provider", event.model.provider);
const span = input.tracer.startSpan(
- modelSpanName(attempt.operation.name),
+ modelSpanName(event.model.modelId),
{
attributes: {
- "gen_ai.operation.name": attempt.operation.name,
+ "gen_ai.agent.name": event.scope.functionId,
+ "gen_ai.operation.name": "chat",
"gen_ai.provider.name": event.model.provider,
"gen_ai.request.model": event.model.modelId,
},
@@ -281,26 +335,38 @@ export function createAgentOtelInstrumentation(
if (recordInputs) {
const messages = messagesContentAttribute(event.input.messages);
if (messages !== undefined) span.setAttribute("ai.prompt.messages", messages);
+ const genAiMessages = genAiInputMessagesAttribute(event.input.messages);
+ if (genAiMessages !== undefined) span.setAttribute("gen_ai.input.messages", genAiMessages);
const system = systemPromptAttribute(event.input.instructions);
if (system !== undefined) span.setAttribute("ai.prompt.system", system);
+ const genAiSystem = genAiSystemInstructionsAttribute(event.input.instructions);
+ if (genAiSystem !== undefined) {
+ span.setAttribute("gen_ai.system_instructions", genAiSystem);
+ }
}
const state = { context: trace.setSpan(attempt.operation.context, span), span };
- getExecutionContexts(event.scope).models.set(event.idempotencyKey, state.context);
+ getExecutionContexts(event.scope).set(event.idempotencyKey, state.context);
getSpanStates(modelSpans, event.scope).set(event.idempotencyKey, state);
};
const onModelCallTerminal = (event: InstrumentationModelCallTerminalEvent): void => {
- executionContexts.get(event.scope)?.models.delete(event.idempotencyKey);
+ executionContexts.get(event.scope)?.delete(event.idempotencyKey);
const state = takeSpanState(modelSpans, event.scope, event.idempotencyKey);
if (state === undefined) return;
if (event.type === "model.call.failed") {
recordError(state.span, event.error);
} else {
- setUsage(state.span, event.usage);
+ setAgentUsage(state.span, event.usage);
+ state.span.setAttribute("gen_ai.response.finish_reasons", [event.finishReason]);
const attempt = steps.get(event.scope);
- if (attempt !== undefined) setUsage(attempt.step.span, event.usage);
+ if (attempt !== undefined) setAgentUsage(attempt.step.span, event.usage);
if (recordOutputs) {
state.span.setAttribute("ai.response.finish_reason", event.finishReason);
+ const content = event.content;
+ const outputMessages = genAiOutputMessagesAttribute(content, event.finishReason);
+ if (outputMessages !== undefined) {
+ state.span.setAttribute("gen_ai.output.messages", outputMessages);
+ }
const reasoning = textContentAttribute(
event.content
.filter((part) => part.type === "reasoning")
@@ -352,70 +418,6 @@ export function createAgentOtelInstrumentation(
state.span.end();
};
- const onToolCallStarted = (event: InstrumentationToolCallStartedEvent): void => {
- const attempt = steps.get(event.scope);
- if (attempt === undefined) return;
- const actionSpan = input.tracer.startSpan(
- "agent.action",
- {
- attributes: {
- "agent.action.call_id": event.callId,
- "agent.action.kind": "tool",
- "agent.action.name": event.toolName,
- "agent.framework.name": "eve",
- "agent.framework.version": input.frameworkVersion,
- "agent.root.session.id": event.scope.rootSessionId ?? event.scope.sessionId,
- "agent.session.id": event.scope.sessionId,
- "agent.step.attempt": event.scope.attemptIndex,
- "agent.step.index": event.scope.stepIndex,
- "agent.turn.id": event.scope.turnId,
- },
- },
- attempt.step.context,
- );
- const actionContext = trace.setSpan(attempt.step.context, actionSpan);
- const toolSpan = input.tracer.startSpan(
- "ai.toolCall",
- {
- attributes: {
- "gen_ai.operation.name": "execute_tool",
- "gen_ai.tool.call.id": event.callId,
- "gen_ai.tool.name": event.toolName,
- },
- },
- actionContext,
- );
- if (recordInputs) {
- const args = contentAttribute(event.input, false);
- if (args !== undefined) toolSpan.setAttribute("gen_ai.tool.call.arguments", args);
- }
- const state: ToolSpanState = {
- context: trace.setSpan(actionContext, toolSpan),
- span: actionSpan,
- toolSpan,
- };
- getExecutionContexts(event.scope).tools.set(event.idempotencyKey, state.context);
- getSpanStates(toolSpans, event.scope).set(event.idempotencyKey, state);
- };
-
- const onToolCallTerminal = (event: InstrumentationToolCallTerminalEvent): void => {
- executionContexts.get(event.scope)?.tools.delete(event.idempotencyKey);
- const state = takeSpanState(toolSpans, event.scope, event.idempotencyKey);
- if (state === undefined) return;
- if (event.type === "tool.call.failed") {
- recordError(state.toolSpan, event.error);
- recordError(state.span, event.error);
- } else if (event.output.type === "error") {
- recordError(state.toolSpan, event.output.error);
- recordError(state.span, event.output.error);
- } else if (recordOutputs) {
- const result = contentAttribute(event.output.output, false);
- if (result !== undefined) state.toolSpan.setAttribute("gen_ai.tool.call.result", result);
- }
- state.toolSpan.end();
- state.span.end();
- };
-
const openSessionWindow = (window: {
readonly agentName?: string;
readonly index: number;
@@ -509,6 +511,13 @@ export function createAgentOtelInstrumentation(
hook: {
name: "eve.otel",
events: {
+ "action.completed": actions.events["action.completed"],
+ "action.failed": actions.events["action.failed"],
+ async "action.started"(event, ctx) {
+ await actions.events["action.started"]!(event, ctx);
+ await tools.actionStarted(event);
+ },
+ ...approvals,
"step.attempt.completed": onStepTerminal,
"step.attempt.failed": onStepTerminal,
"step.attempt.metadata": onStepMetadata,
@@ -520,9 +529,7 @@ export function createAgentOtelInstrumentation(
"session.failed": onSessionTransition,
"session.started": onSessionStarted,
"session.waiting": onSessionTransition,
- "tool.call.completed": onToolCallTerminal,
- "tool.call.failed": onToolCallTerminal,
- "tool.call.started": onToolCallStarted,
+ ...tools.events,
"turn.cancelled": onTurnTerminal,
"turn.completed": onTurnTerminal,
"turn.failed": onTurnTerminal,
@@ -534,19 +541,16 @@ export function createAgentOtelInstrumentation(
const contexts = executionContexts.get(scope);
const parent =
operation.type === "model.call"
- ? contexts?.models.get(operation.idempotencyKey)
- : contexts?.tools.get(operation.idempotencyKey);
+ ? contexts?.get(operation.idempotencyKey)
+ : tools.contextFor(operation.scope.attemptId, operation.idempotencyKey);
return parent === undefined ? execute() : context.with(parent, execute);
},
};
- function getExecutionContexts(scope: InstrumentationAttemptScope): {
- readonly models: Map<string, Context>;
- readonly tools: Map<string, Context>;
- } {
+ function getExecutionContexts(scope: InstrumentationAttemptScope): Map<string, Context> {
let state = executionContexts.get(scope);
if (state === undefined) {
- state = { models: new Map(), tools: new Map() };
+ state = new Map();
executionContexts.set(scope, state);
}
return state;
@@ -558,15 +562,6 @@ export function createAgentOtelInstrumentation(
state.span.end();
}
modelSpans.delete(event.scope);
- for (const state of toolSpans.get(event.scope)?.values() ?? []) {
- if (event.type === "step.attempt.failed") {
- recordError(state.toolSpan, event.error);
- recordError(state.span, event.error);
- }
- state.toolSpan.end();
- state.span.end();
- }
- toolSpans.delete(event.scope);
}
}
@@ -612,9 +607,7 @@ function parentLineageAttributes(
function adoptedSpanContext(handed: InstrumentationTraceContext): SpanContext {
return {
- // Not remote: a subagent crosses the same worker boundary every restored
- // session context does, not the wire.
- isRemote: false,
+ isRemote: "isRemote" in handed && handed.isRemote === true,
spanId: handed.spanId,
traceFlags: handed.traceFlags,
traceId: handed.traceId,
@@ -662,29 +655,8 @@ function isRecord(value: unknown): value is Record<string, unknown> {
return typeof value === "object" && value !== null && !Array.isArray(value);
}
-function modelSpanName(operationName: string): string {
- return operationName === "ai.generateText"
- ? "ai.generateText.doGenerate"
- : "ai.streamText.doStream";
-}
-
-function setUsage(span: Span, usage: InstrumentationUsage): void {
- if (usage.inputTokens !== undefined) {
- span.setAttribute("agent.usage.input_tokens", usage.inputTokens);
- }
- if (usage.outputTokens !== undefined) {
- span.setAttribute("agent.usage.output_tokens", usage.outputTokens);
- }
- // Cached tokens price differently from plain input, so keep the split.
- // Named for the OTel GenAI semantic conventions; present only when the
- // provider reports details — others emit nothing.
- const details = usage.inputTokenDetails;
- if (details?.cacheReadTokens !== undefined) {
- span.setAttribute("gen_ai.usage.cache_read.input_tokens", details.cacheReadTokens);
- }
- if (details?.cacheWriteTokens !== undefined) {
- span.setAttribute("gen_ai.usage.cache_creation.input_tokens", details.cacheWriteTokens);
- }
+function modelSpanName(modelId: string): string {
+ return `chat ${modelId}`;
}
function errorText(error: unknown): unknown {
diff --git a/packages/eve/src/tracing/agent-otel-usage.ts b/packages/eve/src/tracing/agent-otel-usage.ts
new file mode 100644
index 000000000..b6889a9f2
--- /dev/null
+++ b/packages/eve/src/tracing/agent-otel-usage.ts
@@ -0,0 +1,20 @@
+import type { Span } from "#compiled/@opentelemetry/api/index.js";
+
+import type { InstrumentationUsage } from "#harness/instrumentation-lifecycle.js";
+
+/** Applies eve's structural token usage attributes to an agent span. */
+export function setAgentUsage(span: Span, usage: InstrumentationUsage): void {
+ if (usage.inputTokens !== undefined) {
+ span.setAttribute("agent.usage.input_tokens", usage.inputTokens);
+ }
+ if (usage.outputTokens !== undefined) {
+ span.setAttribute("agent.usage.output_tokens", usage.outputTokens);
+ }
+ const details = usage.inputTokenDetails;
+ if (details?.cacheReadTokens !== undefined) {
+ span.setAttribute("gen_ai.usage.cache_read.input_tokens", details.cacheReadTokens);
+ }
+ if (details?.cacheWriteTokens !== undefined) {
+ span.setAttribute("gen_ai.usage.cache_creation.input_tokens", details.cacheWriteTokens);
+ }
+}
diff --git a/packages/eve/src/tracing/agent-span-id-generator.ts b/packages/eve/src/tracing/agent-span-id-generator.ts
index 740ea8477..9e7ad0c92 100644
--- a/packages/eve/src/tracing/agent-span-id-generator.ts
+++ b/packages/eve/src/tracing/agent-span-id-generator.ts
@@ -1,3 +1,5 @@
+import { createHash } from "node:crypto";
+
/**
* Id generator shared by `registerOTel` and the agent OTel provider. A span
* whose lifetime crosses durable worker boundaries (`agent.turn`) is emitted
@@ -12,6 +14,12 @@ export class AgentSpanIdGenerator {
return randomHexId(16);
}
+ /** Derives one span id for a replay-stable instrumentation event. */
+ deriveSpanId(key: string): string {
+ const spanId = createHash("sha256").update(key).digest("hex").slice(0, 16);
+ return /^0+$/u.test(spanId) ? "0000000000000001" : spanId;
+ }
+
generateSpanId(): string {
const primed = this.#primedSpanId;
if (primed !== undefined) {
diff --git a/packages/eve/src/tracing/agent-tool-instrumentation.ts b/packages/eve/src/tracing/agent-tool-instrumentation.ts
new file mode 100644
index 000000000..9c865e220
--- /dev/null
+++ b/packages/eve/src/tracing/agent-tool-instrumentation.ts
@@ -0,0 +1,226 @@
+import {
+ ROOT_CONTEXT,
+ SpanStatusCode,
+ type Context,
+ type Span,
+ type SpanContext,
+ type Tracer,
+ trace,
+} from "#compiled/@opentelemetry/api/index.js";
+
+import type {
+ InstrumentationActionStartedEvent,
+ InstrumentationToolCallStartedEvent,
+ InstrumentationToolCallTerminalEvent,
+} from "#harness/instrumentation-lifecycle.js";
+import { actionIdempotencyKey } from "#harness/instrumentation-lifecycle.js";
+import { contentAttribute } from "#tracing/agent-otel-content.js";
+import type { AgentSpanIdGenerator } from "#tracing/agent-span-id-generator.js";
+import type { AgentActionContext } from "#tracing/agent-action-instrumentation.js";
+
+interface ToolSpanState {
+ readonly actionKey: string;
+ readonly attemptId: string;
+ readonly context: Context;
+ readonly event: InstrumentationToolCallStartedEvent;
+ readonly fallbackParent: Context;
+ readonly idempotencyKey: string;
+ readonly spanId: string;
+ readonly startTimeMs: number;
+ finished?: true;
+ span?: Span;
+ terminal?: InstrumentationToolCallTerminalEvent;
+}
+
+export interface AgentToolInstrumentation {
+ actionStarted(event: InstrumentationActionStartedEvent): Promise<void>;
+ contextFor(attemptId: string, idempotencyKey: string): Context | undefined;
+ drain(attemptId: string, failure?: { readonly error: unknown }): void;
+ readonly events: {
+ readonly "tool.call.completed": (event: InstrumentationToolCallTerminalEvent) => Promise<void>;
+ readonly "tool.call.failed": (event: InstrumentationToolCallTerminalEvent) => Promise<void>;
+ readonly "tool.call.started": (event: InstrumentationToolCallStartedEvent) => Promise<void>;
+ };
+}
+
+/** Keeps SDK tool spans parented to actions even when SDK telemetry wins the event race. */
+export function createAgentToolInstrumentation(input: {
+ readonly actionContextFor: (
+ sessionId: string,
+ turnId: string,
+ callId: string,
+ ) => Promise<AgentActionContext | undefined>;
+ readonly idGenerator: AgentSpanIdGenerator;
+ readonly recordInputs: boolean;
+ readonly recordOutputs: boolean;
+ readonly resolveFallback: (
+ event: InstrumentationToolCallStartedEvent,
+ ) => { readonly context: Context; readonly spanContext: SpanContext } | undefined;
+ readonly tracer: Tracer;
+}): AgentToolInstrumentation {
+ const byAction = new Map<string, ToolSpanState>();
+ const byAttempt = new Map<string, Map<string, ToolSpanState>>();
+
+ const onStarted = async (event: InstrumentationToolCallStartedEvent): Promise<void> => {
+ const actionKey = actionIdempotencyKey(event.scope.sessionId, event.scope.turnId, event.callId);
+ const fallback = input.resolveFallback(event);
+ let state = fallback === undefined ? undefined : reserve(event, actionKey, fallback);
+ const actionParent = await input.actionContextFor(
+ event.scope.sessionId,
+ event.scope.turnId,
+ event.callId,
+ );
+ if (state === undefined) {
+ if (actionParent === undefined) return;
+ state = reserve(event, actionKey, actionParent);
+ }
+ if (actionParent !== undefined) startSpan(state, actionParent.context);
+ };
+
+ const onTerminal = async (event: InstrumentationToolCallTerminalEvent): Promise<void> => {
+ const state = byAttempt.get(event.scope.attemptId)?.get(event.idempotencyKey);
+ if (state === undefined) return;
+ state.terminal = event;
+
+ if (state.span === undefined) {
+ const actionParent = await input.actionContextFor(
+ state.event.scope.sessionId,
+ state.event.scope.turnId,
+ state.event.callId,
+ );
+ if (actionParent !== undefined) startSpan(state, actionParent.context);
+ }
+ finishIfReady(state);
+ };
+
+ return {
+ async actionStarted(event) {
+ const state = byAction.get(event.idempotencyKey);
+ if (state === undefined || state.span !== undefined || state.finished === true) return;
+ const actionParent = await input.actionContextFor(
+ event.scope.sessionId,
+ event.scope.turnId,
+ event.callId,
+ );
+ if (actionParent === undefined) return;
+ startSpan(state, actionParent.context);
+ finishIfReady(state);
+ },
+ contextFor: (attemptId, idempotencyKey) =>
+ byAttempt.get(attemptId)?.get(idempotencyKey)?.context,
+ drain(attemptId, failure) {
+ const states = byAttempt.get(attemptId);
+ if (states === undefined) return;
+ for (const state of states.values()) {
+ if (state.finished === true) continue;
+ if (state.span === undefined) startSpan(state, state.fallbackParent);
+ finish(state, failure);
+ }
+ byAttempt.delete(attemptId);
+ },
+ events: {
+ "tool.call.completed": onTerminal,
+ "tool.call.failed": onTerminal,
+ "tool.call.started": onStarted,
+ },
+ };
+
+ function getAttemptStates(attemptId: string): Map<string, ToolSpanState> {
+ let states = byAttempt.get(attemptId);
+ if (states === undefined) {
+ states = new Map();
+ byAttempt.set(attemptId, states);
+ }
+ return states;
+ }
+
+ function reserve(
+ event: InstrumentationToolCallStartedEvent,
+ actionKey: string,
+ parent: { readonly context: Context; readonly spanContext: SpanContext },
+ ): ToolSpanState {
+ const spanId = input.idGenerator.deriveSpanId(`tool:${event.idempotencyKey}`);
+ const state: ToolSpanState = {
+ actionKey,
+ attemptId: event.scope.attemptId,
+ context: contextFromSpanContext({
+ isRemote: false,
+ spanId,
+ traceFlags: parent.spanContext.traceFlags,
+ traceId: parent.spanContext.traceId,
+ }),
+ event,
+ fallbackParent: parent.context,
+ idempotencyKey: event.idempotencyKey,
+ spanId,
+ startTimeMs: Date.now(),
+ };
+ getAttemptStates(event.scope.attemptId).set(event.idempotencyKey, state);
+ byAction.set(actionKey, state);
+ return state;
+ }
+
+ function startSpan(state: ToolSpanState, parent: Context): void {
+ if (state.span !== undefined || state.finished === true) return;
+ state.span = input.idGenerator.withSpanId(state.spanId, () =>
+ input.tracer.startSpan(
+ "ai.toolCall",
+ {
+ attributes: toolAttributes(state.event),
+ startTime: state.startTimeMs,
+ },
+ parent,
+ ),
+ );
+ if (input.recordInputs) {
+ const args = contentAttribute(state.event.input, false);
+ if (args !== undefined) state.span.setAttribute("gen_ai.tool.call.arguments", args);
+ }
+ }
+
+ function finishIfReady(state: ToolSpanState): void {
+ if (state.span === undefined || state.terminal === undefined) return;
+ finish(state);
+ }
+
+ function finish(state: ToolSpanState, failure?: { readonly error: unknown }): void {
+ const span = state.span;
+ if (span === undefined || state.finished === true) return;
+ state.finished = true;
+ const terminal = state.terminal;
+ if (failure !== undefined) {
+ recordError(span, failure.error);
+ } else if (terminal?.type === "tool.call.failed") {
+ recordError(span, terminal.error);
+ } else if (terminal?.output.type === "error") {
+ recordError(span, terminal.output.error);
+ } else if (terminal !== undefined && input.recordOutputs) {
+ const result = contentAttribute(terminal.output.output, false);
+ if (result !== undefined) span.setAttribute("gen_ai.tool.call.result", result);
+ }
+ span.end();
+ byAction.delete(state.actionKey);
+ const states = byAttempt.get(state.attemptId);
+ states?.delete(state.idempotencyKey);
+ if (states?.size === 0) byAttempt.delete(state.attemptId);
+ }
+}
+
+function toolAttributes(event: InstrumentationToolCallStartedEvent): Record<string, string> {
+ return {
+ "gen_ai.operation.name": "execute_tool",
+ "gen_ai.tool.call.id": event.callId,
+ "gen_ai.tool.name": event.toolName,
+ };
+}
+
+function contextFromSpanContext(spanContext: SpanContext): Context {
+ return trace.setSpan(ROOT_CONTEXT, trace.wrapSpanContext(spanContext));
+}
+
+function recordError(span: Span, error: unknown): void {
+ if (error instanceof Error) {
+ span.recordException(error);
+ span.setStatus({ code: SpanStatusCode.ERROR, message: error.message });
+ } else span.setStatus({ code: SpanStatusCode.ERROR });
+}
diff --git a/packages/eve/src/tracing/agent-trace-context-store.test.ts b/packages/eve/src/tracing/agent-trace-context-store.test.ts
index 51c36eb09..01c77b1d4 100644
--- a/packages/eve/src/tracing/agent-trace-context-store.test.ts
+++ b/packages/eve/src/tracing/agent-trace-context-store.test.ts
@@ -5,6 +5,7 @@ import { deserializeContext, serializeContext } from "#context/serialize.js";
import {
ContextAgentTraceStateStore,
preserveSerializedAgentTraceState,
+ readActionTraceContext,
readSessionTraceContext,
} from "#tracing/agent-trace-context-store.js";
@@ -28,6 +29,7 @@ describe("ContextAgentTraceStateStore", () => {
subagentName: "researcher",
turnId: "parent-turn",
},
+ parentIsRemote: true,
parentSpanId: "2".repeat(16),
rootSessionId: "session-1",
sequence: 0,
@@ -50,6 +52,7 @@ describe("ContextAgentTraceStateStore", () => {
subagentName: "researcher",
turnId: "parent-turn",
},
+ parentIsRemote: true,
parentSpanId: "2".repeat(16),
startTimeMs: 1_700_000_000_000,
});
@@ -124,6 +127,36 @@ describe("readSessionTraceContext", () => {
});
});
+describe("readActionTraceContext", () => {
+ it("reads the invoking action span out of a serialized context", async () => {
+ const context = new ContextContainer();
+ await contextStorage.run(context, () => {
+ new ContextAgentTraceStateStore().setAction("action-1", {
+ attemptIndex: 0,
+ callId: "call-1",
+ kind: "subagent-call",
+ name: "researcher",
+ parent: spanContext("1", "2"),
+ rootSessionId: "session-1",
+ sessionId: "session-1",
+ spanId: "3".repeat(16),
+ startTimeMs: 1_700_000_000_000,
+ stepIndex: 0,
+ turnId: "turn-1",
+ });
+ });
+ const serialized = await serializeContext(context);
+
+ expect(readActionTraceContext(serialized, "session-1", "turn-1", "call-1")).toEqual({
+ isRemote: false,
+ spanId: "3".repeat(16),
+ traceFlags: 1,
+ traceId: "1".repeat(32),
+ });
+ expect(readActionTraceContext(serialized, "session-1", "turn-1", "missing")).toBeUndefined();
+ });
+});
+
function spanContext(traceId: string, spanId: string) {
return { spanId: spanId.repeat(16), traceFlags: 1, traceId: traceId.repeat(32) };
}
diff --git a/packages/eve/src/tracing/agent-trace-context-store.ts b/packages/eve/src/tracing/agent-trace-context-store.ts
index 3c6111044..15c9c7d8b 100644
--- a/packages/eve/src/tracing/agent-trace-context-store.ts
+++ b/packages/eve/src/tracing/agent-trace-context-store.ts
@@ -4,12 +4,14 @@ import { contextStorage, loadContext } from "#context/container.js";
import { ContextKey } from "#context/key.js";
import type { InstrumentationParentLineage } from "#harness/instrumentation-lifecycle.js";
import type {
+ AgentActionTraceState,
AgentSessionTraceState,
AgentTraceStateStore,
AgentTurnTraceState,
} from "#tracing/agent-trace-state.js";
interface AgentTraceContextState {
+ readonly actions: Readonly<Record<string, AgentActionTraceState>>;
readonly sessions: Readonly<Record<string, AgentSessionTraceState>>;
readonly turns: Readonly<Record<string, AgentTurnTraceState>>;
}
@@ -46,8 +48,49 @@ export function readSessionTraceContext(
return deserializeState(raw).sessions[sessionId]?.context;
}
+/** Reads the durable action span that should parent a dispatched child agent. */
+export function readActionTraceContext(
+ serializedContext: Readonly<Record<string, unknown>>,
+ sessionId: string,
+ turnId: string,
+ callId: string,
+): SpanContext | undefined {
+ const raw = serializedContext[AgentTraceContextKey.name];
+ if (raw === undefined) return undefined;
+ const action = Object.values(deserializeState(raw).actions).find(
+ (state) => state.sessionId === sessionId && state.turnId === turnId && state.callId === callId,
+ );
+ if (action === undefined) return undefined;
+ return {
+ isRemote: false,
+ spanId: action.spanId,
+ traceFlags: action.parent.traceFlags,
+ traceId: action.parent.traceId,
+ };
+}
+
/** Durable trace state backed by eve's serialized Workflow context. */
export class ContextAgentTraceStateStore implements AgentTraceStateStore {
+ deleteAction(idempotencyKey: string): void {
+ updateState((state) => {
+ const actions = { ...state.actions };
+ delete actions[idempotencyKey];
+ return { ...state, actions };
+ });
+ }
+
+ deleteActions(sessionId: string, turnId?: string): void {
+ updateState((state) => {
+ const actions = { ...state.actions };
+ for (const [key, action] of Object.entries(actions)) {
+ if (action.sessionId === sessionId && (turnId === undefined || action.turnId === turnId)) {
+ delete actions[key];
+ }
+ }
+ return { ...state, actions };
+ });
+ }
+
deleteSession(sessionId: string): void {
updateState((state) => {
const sessions = { ...state.sessions };
@@ -64,6 +107,16 @@ export class ContextAgentTraceStateStore implements AgentTraceStateStore {
});
}
+ findAction(sessionId: string, callId: string): AgentActionTraceState | undefined {
+ return Object.values(contextStorage.getStore()?.get(AgentTraceContextKey)?.actions ?? {}).find(
+ (state) => state.sessionId === sessionId && state.callId === callId,
+ );
+ }
+
+ getAction(idempotencyKey: string): AgentActionTraceState | undefined {
+ return contextStorage.getStore()?.get(AgentTraceContextKey)?.actions[idempotencyKey];
+ }
+
getSession(sessionId: string): AgentSessionTraceState | undefined {
return contextStorage.getStore()?.get(AgentTraceContextKey)?.sessions[sessionId];
}
@@ -72,6 +125,13 @@ export class ContextAgentTraceStateStore implements AgentTraceStateStore {
return contextStorage.getStore()?.get(AgentTraceContextKey)?.turns[turnKey(sessionId, turnId)];
}
+ setAction(idempotencyKey: string, value: AgentActionTraceState): void {
+ updateState((state) => ({
+ ...state,
+ actions: { ...state.actions, [idempotencyKey]: value },
+ }));
+ }
+
setSession(sessionId: string, value: AgentSessionTraceState): void {
updateState((state) => ({
...state,
@@ -88,7 +148,9 @@ export class ContextAgentTraceStateStore implements AgentTraceStateStore {
}
function updateState(update: (state: AgentTraceContextState) => AgentTraceContextState): void {
- loadContext().set(AgentTraceContextKey, (state) => update(state ?? { sessions: {}, turns: {} }));
+ loadContext().set(AgentTraceContextKey, (state) =>
+ update(state ?? { actions: {}, sessions: {}, turns: {} }),
+ );
}
function turnKey(sessionId: string, turnId: string): string {
@@ -97,6 +159,7 @@ function turnKey(sessionId: string, turnId: string): string {
function serializeState(state: AgentTraceContextState): unknown {
return {
+ actions: state.actions,
sessions: Object.fromEntries(
Object.entries(state.sessions).map(([id, value]) => [
id,
@@ -122,7 +185,8 @@ function serializeState(state: AgentTraceContextState): unknown {
}
function deserializeState(data: unknown): AgentTraceContextState {
- if (!isRecord(data)) return { sessions: {}, turns: {} };
+ if (!isRecord(data)) return { actions: {}, sessions: {}, turns: {} };
+ const actions = deserializeRecord(data.actions, deserializeAction);
const sessions = deserializeRecord(data.sessions, (value) => {
if (!isRecord(value) || !isSpanContext(value.context)) return undefined;
return {
@@ -142,6 +206,7 @@ function deserializeState(data: unknown): AgentTraceContextState {
return {
context: value.context,
lineage: deserializeLineage(value.lineage),
+ parentIsRemote: typeof value.parentIsRemote === "boolean" ? value.parentIsRemote : undefined,
parentSpanId: value.parentSpanId,
rootSessionId: typeof value.rootSessionId === "string" ? value.rootSessionId : "",
sequence: typeof value.sequence === "number" ? value.sequence : 0,
@@ -149,7 +214,49 @@ function deserializeState(data: unknown): AgentTraceContextState {
terminal: deserializeTerminal(value.terminal),
} satisfies AgentTurnTraceState;
});
- return { sessions, turns };
+ return { actions, sessions, turns };
+}
+
+function deserializeAction(value: unknown): AgentActionTraceState | undefined {
+ if (
+ !isRecord(value) ||
+ typeof value.attemptIndex !== "number" ||
+ typeof value.callId !== "string" ||
+ !isActionKind(value.kind) ||
+ typeof value.name !== "string" ||
+ !isSpanContext(value.parent) ||
+ typeof value.rootSessionId !== "string" ||
+ typeof value.sessionId !== "string" ||
+ typeof value.spanId !== "string" ||
+ typeof value.startTimeMs !== "number" ||
+ typeof value.stepIndex !== "number" ||
+ typeof value.turnId !== "string"
+ ) {
+ return undefined;
+ }
+ return {
+ attemptIndex: value.attemptIndex,
+ callId: value.callId,
+ inputAttribute: typeof value.inputAttribute === "string" ? value.inputAttribute : undefined,
+ kind: value.kind,
+ name: value.name,
+ parent: value.parent,
+ rootSessionId: value.rootSessionId,
+ sessionId: value.sessionId,
+ spanId: value.spanId,
+ startTimeMs: value.startTimeMs,
+ stepIndex: value.stepIndex,
+ turnId: value.turnId,
+ };
+}
+
+function isActionKind(value: unknown): value is AgentActionTraceState["kind"] {
+ return (
+ value === "load-skill" ||
+ value === "remote-agent-call" ||
+ value === "subagent-call" ||
+ value === "tool-call"
+ );
}
function deserializeRecord<T>(
diff --git a/packages/eve/src/tracing/agent-trace-state.ts b/packages/eve/src/tracing/agent-trace-state.ts
index 93e9f20b6..641845db2 100644
--- a/packages/eve/src/tracing/agent-trace-state.ts
+++ b/packages/eve/src/tracing/agent-trace-state.ts
@@ -1,7 +1,9 @@
import type { SpanContext } from "#compiled/@opentelemetry/api/index.js";
import type {
+ InstrumentationActionKind,
InstrumentationParentLineage,
+ InstrumentationTraceContext,
InstrumentationTurnFailedEvent,
InstrumentationTurnSettledEvent,
} from "#harness/instrumentation-lifecycle.js";
@@ -21,6 +23,7 @@ export interface AgentSessionTraceState {
export interface AgentTurnTraceState {
readonly context: SpanContext;
readonly lineage?: InstrumentationParentLineage;
+ readonly parentIsRemote?: boolean;
readonly parentSpanId: string;
readonly rootSessionId: string;
readonly sequence: number;
@@ -30,10 +33,34 @@ export interface AgentTurnTraceState {
| { readonly type: InstrumentationTurnSettledEvent["type"] };
}
+export interface AgentActionTraceState {
+ readonly attemptIndex: number;
+ readonly callId: string;
+ readonly inputAttribute?: string;
+ readonly kind: InstrumentationActionKind;
+ readonly name: string;
+ readonly parent: InstrumentationTraceContext;
+ readonly rootSessionId: string;
+ readonly sessionId: string;
+ readonly spanId: string;
+ readonly startTimeMs: number;
+ readonly stepIndex: number;
+ readonly turnId: string;
+}
+
/** Provider-owned serializable storage for durable agent trace state. */
export interface AgentTraceStateStore {
+ deleteAction(idempotencyKey: string): void | PromiseLike<void>;
+ deleteActions(sessionId: string, turnId?: string): void | PromiseLike<void>;
deleteSession(sessionId: string): void | PromiseLike<void>;
deleteTurn(sessionId: string, turnId: string): void | PromiseLike<void>;
+ findAction(
+ sessionId: string,
+ callId: string,
+ ): AgentActionTraceState | undefined | PromiseLike<AgentActionTraceState | undefined>;
+ getAction(
+ idempotencyKey: string,
+ ): AgentActionTraceState | undefined | PromiseLike<AgentActionTraceState | undefined>;
getSession(
sessionId: string,
): AgentSessionTraceState | undefined | PromiseLike<AgentSessionTraceState | undefined>;
@@ -41,15 +68,29 @@ export interface AgentTraceStateStore {
sessionId: string,
turnId: string,
): AgentTurnTraceState | undefined | PromiseLike<AgentTurnTraceState | undefined>;
+ setAction(idempotencyKey: string, state: AgentActionTraceState): void | PromiseLike<void>;
setSession(sessionId: string, state: AgentSessionTraceState): void | PromiseLike<void>;
setTurn(sessionId: string, turnId: string, state: AgentTurnTraceState): void | PromiseLike<void>;
}
/** In-memory trace state used by tests and non-durable runtimes. */
export class InMemoryAgentTraceStateStore implements AgentTraceStateStore {
+ readonly #actions = new Map<string, AgentActionTraceState>();
readonly #sessions = new Map<string, AgentSessionTraceState>();
readonly #turns = new Map<string, AgentTurnTraceState>();
+ deleteAction(idempotencyKey: string): void {
+ this.#actions.delete(idempotencyKey);
+ }
+
+ deleteActions(sessionId: string, turnId?: string): void {
+ for (const [key, state] of this.#actions) {
+ if (state.sessionId === sessionId && (turnId === undefined || state.turnId === turnId)) {
+ this.#actions.delete(key);
+ }
+ }
+ }
+
deleteSession(sessionId: string): void {
this.#sessions.delete(sessionId);
}
@@ -58,6 +99,16 @@ export class InMemoryAgentTraceStateStore implements AgentTraceStateStore {
this.#turns.delete(turnKey(sessionId, turnId));
}
+ findAction(sessionId: string, callId: string): AgentActionTraceState | undefined {
+ return [...this.#actions.values()].find(
+ (state) => state.sessionId === sessionId && state.callId === callId,
+ );
+ }
+
+ getAction(idempotencyKey: string): AgentActionTraceState | undefined {
+ return this.#actions.get(idempotencyKey);
+ }
+
getSession(sessionId: string): AgentSessionTraceState | undefined {
return this.#sessions.get(sessionId);
}
@@ -66,6 +117,10 @@ export class InMemoryAgentTraceStateStore implements AgentTraceStateStore {
return this.#turns.get(turnKey(sessionId, turnId));
}
+ setAction(idempotencyKey: string, state: AgentActionTraceState): void {
+ this.#actions.set(idempotencyKey, state);
+ }
+
setSession(sessionId: string, state: AgentSessionTraceState): void {
this.#sessions.set(sessionId, state);
}
diff --git a/packages/eve/src/tracing/local-instrumentation-runtime.scenario.test.ts b/packages/eve/src/tracing/local-instrumentation-runtime.scenario.test.ts
index ec8755b3e..422920cd0 100644
--- a/packages/eve/src/tracing/local-instrumentation-runtime.scenario.test.ts
+++ b/packages/eve/src/tracing/local-instrumentation-runtime.scenario.test.ts
@@ -3,15 +3,21 @@ import { createRequire } from "node:module";
import { tmpdir } from "node:os";
import { join } from "node:path";
-import { ROOT_CONTEXT, trace as runtimeTrace } from "@opentelemetry/api";
+import { ROOT_CONTEXT, trace as apiTrace } from "@opentelemetry/api";
import { BasicTracerProvider } from "@opentelemetry/sdk-trace-base";
-import { afterEach, describe, expect, it } from "vitest";
+import { afterEach, describe, expect, it, vi } from "vitest";
+import {
+ ROOT_CONTEXT as COMPILED_ROOT_CONTEXT,
+ context as runtimeContext,
+ trace as runtimeTrace,
+} from "#compiled/@opentelemetry/api/index.js";
import { ContextContainer, contextStorage } from "#context/container.js";
import { createAiSdkHookBridge } from "#harness/ai-sdk-hook-bridge.js";
import { listLocalTraces } from "#tracing/local-trace-reader.js";
import type { InstrumentationAttemptScope } from "#harness/instrumentation-lifecycle.js";
import {
+ actionIdempotencyKey,
attemptIdempotencyKey,
sessionIdempotencyKey,
turnIdempotencyKey,
@@ -50,6 +56,9 @@ describe("local instrumentation runtime", () => {
stepIndex: 0,
turnId: "turn-1",
};
+ const delivery = runtimeTrace.getTracer("workflow").startSpan("workflow.delivery");
+ const activeContext = runtimeTrace.setSpan(COMPILED_ROOT_CONTEXT, delivery);
+ const contextActive = vi.spyOn(runtimeContext, "active").mockReturnValue(activeContext);
await contextStorage.run(new ContextContainer(), async () => {
await runtime.hooks.publish({
@@ -99,6 +108,16 @@ describe("local instrumentation runtime", () => {
usage: { inputTokens: 1, outputTokens: 1 },
},
]);
+ const actionKey = actionIdempotencyKey("session-1", "turn-1", "tool-1");
+ await runtime.hooks.publish({
+ callId: "tool-1",
+ idempotencyKey: actionKey,
+ input: {},
+ kind: "tool-call",
+ name: "weather",
+ scope,
+ type: "action.started",
+ });
await Reflect.apply(bridge.onToolExecutionStart!, bridge, [
{
callId: "call-1",
@@ -124,6 +143,13 @@ describe("local instrumentation runtime", () => {
toolOutput: { output: { temperature: 72 }, type: "tool-result" },
},
]);
+ await runtime.hooks.publish({
+ idempotencyKey: actionKey,
+ outcome: "completed",
+ output: { output: { temperature: 72 }, type: "result" },
+ scope,
+ type: "action.completed",
+ });
await runtime.hooks.publish({
idempotencyKey: attemptIdempotencyKey(scope),
scope,
@@ -143,6 +169,8 @@ describe("local instrumentation runtime", () => {
type: "session.waiting",
});
});
+ contextActive.mockRestore();
+ delivery.end();
await runtime.forceFlush();
const traceRoot = join(appRoot, ".eve", "traces", "v1");
@@ -164,7 +192,7 @@ describe("local instrumentation runtime", () => {
"agent.turn",
"agent.step",
"ai.streamText",
- "ai.streamText.doStream",
+ "chat model-1",
"agent.action",
"ai.toolCall",
"user.model-work",
@@ -172,14 +200,22 @@ describe("local instrumentation runtime", () => {
]),
);
expect(span(spans, "agent.step").parentSpanId).toBe(span(spans, "agent.turn").spanId);
+ expect(span(spans, "agent.step").links).toEqual([
+ expect.objectContaining({
+ attributes: expect.arrayContaining([
+ expect.objectContaining({
+ key: "eve.link.type",
+ value: { stringValue: "workflow.delivery" },
+ }),
+ ]),
+ spanId: delivery.spanContext().spanId,
+ traceId: delivery.spanContext().traceId,
+ }),
+ ]);
expect(span(spans, "ai.streamText").parentSpanId).toBe(span(spans, "agent.step").spanId);
- expect(span(spans, "ai.streamText.doStream").parentSpanId).toBe(
- span(spans, "ai.streamText").spanId,
- );
- expect(span(spans, "user.model-work").parentSpanId).toBe(
- span(spans, "ai.streamText.doStream").spanId,
- );
- expect(span(spans, "agent.action").parentSpanId).toBe(span(spans, "agent.step").spanId);
+ expect(span(spans, "chat model-1").parentSpanId).toBe(span(spans, "ai.streamText").spanId);
+ expect(span(spans, "user.model-work").parentSpanId).toBe(span(spans, "chat model-1").spanId);
+ expect(span(spans, "agent.action").parentSpanId).toBe(span(spans, "agent.turn").spanId);
expect(span(spans, "ai.toolCall").parentSpanId).toBe(span(spans, "agent.action").spanId);
expect(span(spans, "user.tool-work").parentSpanId).toBe(span(spans, "ai.toolCall").spanId);
const listed = await listLocalTraces(appRoot);
@@ -194,9 +230,9 @@ describe("local instrumentation runtime", () => {
const secondProcessor = new LocalTraceSpanProcessor(appRoot);
const firstProvider = new BasicTracerProvider({ spanProcessors: [firstProcessor] });
const secondProvider = new BasicTracerProvider({ spanProcessors: [secondProcessor] });
- const parent = runtimeTrace.setSpan(
+ const parent = apiTrace.setSpan(
ROOT_CONTEXT,
- runtimeTrace.wrapSpanContext({
+ apiTrace.wrapSpanContext({
spanId: "b".repeat(16),
traceFlags: 1,
traceId: "a".repeat(32),
@@ -215,6 +251,11 @@ describe("local instrumentation runtime", () => {
});
interface OtlpSpan {
+ readonly links?: ReadonlyArray<{
+ readonly attributes: ReadonlyArray<{ readonly key: string; readonly value: unknown }>;
+ readonly spanId: string;
+ readonly traceId: string;
+ }>;
readonly name: string;
readonly parentSpanId?: string;
readonly spanId: string;
diff --git a/packages/eve/src/tracing/otel-registration.test.ts b/packages/eve/src/tracing/otel-registration.test.ts
index 9e49ca976..cb53e16a1 100644
--- a/packages/eve/src/tracing/otel-registration.test.ts
+++ b/packages/eve/src/tracing/otel-registration.test.ts
@@ -92,6 +92,68 @@ describe("registerOtelPipeline", () => {
expect(configuration.spanProcessors[0]).toBe("auto");
});
+ it("exports only the first terminal for a replay-stable span", () => {
+ const downstream = {
+ forceFlush: vi.fn(async () => {}),
+ onEnd: vi.fn(),
+ onStart: vi.fn(),
+ shutdown: vi.fn(async () => {}),
+ };
+ registerOTel.mockImplementation(() => undefined);
+
+ expect(() =>
+ registerOtelPipeline({
+ pipeline: { spanProcessors: [downstream] },
+ serviceName: "weather",
+ }),
+ ).toThrow();
+
+ const processor = (
+ registerOTel.mock.calls.at(-1)?.[0] as {
+ spanProcessors: { onEnd(span: unknown): void }[];
+ }
+ ).spanProcessors[0]!;
+ const first = replaySpan("first");
+ const losingReplay = replaySpan("losing replay");
+ processor.onEnd(first);
+ processor.onEnd(losingReplay);
+
+ expect(downstream.onEnd).toHaveBeenCalledExactlyOnceWith(first);
+ });
+
+ it("holds a physical child until its started parent completes", () => {
+ const downstream = {
+ forceFlush: vi.fn(async () => {}),
+ onEnd: vi.fn(),
+ onStart: vi.fn(),
+ shutdown: vi.fn(async () => {}),
+ };
+ registerOTel.mockImplementation(() => undefined);
+ expect(() =>
+ registerOtelPipeline({
+ pipeline: { spanProcessors: [downstream] },
+ serviceName: "weather",
+ }),
+ ).toThrow();
+ const processor = (
+ registerOTel.mock.calls.at(-1)?.[0] as {
+ spanProcessors: {
+ onEnd(span: unknown): void;
+ onStart(span: unknown, parentContext: unknown): void;
+ }[];
+ }
+ ).spanProcessors[0]!;
+ const parent = physicalSpan("2");
+ const child = physicalSpan("3", "2");
+ processor.onStart(parent, {});
+ processor.onStart(child, {});
+ processor.onEnd(child);
+ expect(downstream.onEnd).not.toHaveBeenCalled();
+
+ processor.onEnd(parent);
+ expect(downstream.onEnd.mock.calls).toEqual([[parent], [child]]);
+ });
+
it("throws when the registration never reached a processor", () => {
registerOTel.mockImplementation(() => undefined);
@@ -100,3 +162,19 @@ describe("registerOtelPipeline", () => {
).toThrow(/another runtime already owns/u);
});
});
+
+function replaySpan(marker: string) {
+ return {
+ marker,
+ spanContext: () => ({ spanId: "2".repeat(16), traceId: "1".repeat(32) }),
+ };
+}
+
+function physicalSpan(spanId: string, parentSpanId?: string) {
+ const traceId = "1".repeat(32);
+ return {
+ parentSpanContext:
+ parentSpanId === undefined ? undefined : { spanId: parentSpanId.repeat(16), traceId },
+ spanContext: () => ({ spanId: spanId.repeat(16), traceId }),
+ };
+}
diff --git a/packages/eve/src/tracing/otel-registration.ts b/packages/eve/src/tracing/otel-registration.ts
index e2337f4fb..eb54545a5 100644
--- a/packages/eve/src/tracing/otel-registration.ts
+++ b/packages/eve/src/tracing/otel-registration.ts
@@ -10,6 +10,7 @@ import { AgentSpanIdGenerator } from "#tracing/agent-span-id-generator.js";
import type { OtelPipeline } from "#tracing/otel-declaration.js";
const REGISTRATION_SPAN_NAME = "eve.otel.registration";
+const REPLAY_DEDUPLICATION_LIMIT = 100_000;
class RegistrationMarkerPropagator {
#injected = false;
@@ -35,7 +36,11 @@ class RegistrationMarkerPropagator {
/** Keeps eve's ownership check out of every authored destination. */
class PrivateSpanFilteringProcessor implements SpanProcessor {
+ private readonly endedSpans = new Set<string>();
+ private readonly forwardedSpans = new Set<string>();
+ private readonly pendingByParent = new Map<string, unknown[]>();
private readonly processors: readonly SpanProcessor[];
+ private readonly startedSpans = new Set<string>();
constructor(processors: readonly SpanProcessor[]) {
this.processors = processors;
@@ -47,17 +52,45 @@ class PrivateSpanFilteringProcessor implements SpanProcessor {
onEnd(span: unknown): void {
if (isRegistrationSpan(span)) return;
- for (const processor of this.processors) processor.onEnd(span);
+ const identity = spanIdentity(span);
+ if (identity !== undefined) {
+ if (this.endedSpans.has(identity)) return;
+ if (this.endedSpans.size >= REPLAY_DEDUPLICATION_LIMIT) {
+ const oldest = this.endedSpans.values().next().value;
+ if (oldest !== undefined) this.endedSpans.delete(oldest);
+ }
+ this.endedSpans.add(identity);
+ }
+ const parent = parentIdentity(span);
+ if (parent !== undefined && this.startedSpans.has(parent) && !this.forwardedSpans.has(parent)) {
+ const pending = this.pendingByParent.get(parent) ?? [];
+ pending.push(span);
+ this.pendingByParent.set(parent, pending);
+ return;
+ }
+ this.forward(span, identity);
}
onStart(span: unknown, parentContext: unknown): void {
if (isRegistrationSpan(span)) return;
+ const identity = spanIdentity(span);
+ if (identity !== undefined) addBounded(this.startedSpans, identity);
for (const processor of this.processors) processor.onStart(span, parentContext);
}
async shutdown(): Promise<void> {
await Promise.all(this.processors.map((processor) => processor.shutdown()));
}
+
+ private forward(span: unknown, identity: string | undefined): void {
+ for (const processor of this.processors) processor.onEnd(span);
+ if (identity === undefined) return;
+ addBounded(this.forwardedSpans, identity);
+ const children = this.pendingByParent.get(identity);
+ if (children === undefined) return;
+ this.pendingByParent.delete(identity);
+ for (const child of children) this.forward(child, spanIdentity(child));
+ }
}
/**
@@ -181,6 +214,41 @@ function isRegistrationSpan(span: unknown): boolean {
);
}
+function spanIdentity(span: unknown): string | undefined {
+ if (
+ typeof span !== "object" ||
+ span === null ||
+ !("spanContext" in span) ||
+ typeof span.spanContext !== "function"
+ ) {
+ return undefined;
+ }
+ const context = span.spanContext() as { readonly spanId?: unknown; readonly traceId?: unknown };
+ return typeof context.traceId === "string" && typeof context.spanId === "string"
+ ? `${context.traceId}:${context.spanId}`
+ : undefined;
+}
+
+function parentIdentity(span: unknown): string | undefined {
+ if (typeof span !== "object" || span === null || !("parentSpanContext" in span)) {
+ return undefined;
+ }
+ const parent = span.parentSpanContext as
+ | { readonly spanId?: unknown; readonly traceId?: unknown }
+ | undefined;
+ return typeof parent?.traceId === "string" && typeof parent.spanId === "string"
+ ? `${parent.traceId}:${parent.spanId}`
+ : undefined;
+}
+
+function addBounded(values: Set<string>, value: string): void {
+ if (values.size >= REPLAY_DEDUPLICATION_LIMIT) {
+ const oldest = values.values().next().value;
+ if (oldest !== undefined) values.delete(oldest);
+ }
+ values.add(value);
+}
+
function isSpanProcessor(processor: SpanProcessorOrName): processor is SpanProcessor {
return processor !== "auto";
}